crawlo 1.0.1__py3-none-any.whl → 1.0.3__py3-none-any.whl

crawlo/network/request.py

@@ -1,155 +1,205 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-import hashlib
-from copy import deepcopy
-from w3lib.url import safe_url_string
-from typing import Dict, Optional, Callable, Union, Any
-
-from crawlo.utils.url import escape_ajax
-
-
-class Request(object):
-
-    def __init__(
-            self,
-            url: str,
-            *,
-            callback: Optional[Callable] = None,
-            headers: Optional[Dict[str, str]] = None,
-            body: Optional[Union[Dict, bytes, str]] = None,
-            method: Optional[str] = 'GET',
-            cookies: Optional[Dict[str, str]] = None,
-            priority: int = 0,
-            encoding: Optional[str] = 'utf-8',
-            meta: Optional[Dict[str, Any]] = None,
-            dont_filter: bool = False,
-            timeout: Optional[float] = None,
-            proxy: Optional[str] = None,
-            allow_redirects: bool = True,
-            auth: Optional[tuple] = None,
-            verify: bool = True
-    ):
-        # 先初始化基本属性
-        self.callback = callback
-        self.headers = headers if headers else {}
-        self.body = body
-        self.method = str(method).upper()
-        self.cookies = cookies if cookies else {}
-        self.priority = -priority
-        self.encoding = encoding
-        self.dont_filter = dont_filter
-        self._meta = meta if meta is not None else {}
-        self.timeout = timeout
-        self.proxy = proxy
-        self.allow_redirects = allow_redirects
-        self.auth = auth
-        self.verify = verify
-
-        # 最后处理URL，确保encoding等依赖属性已初始化
-        self._set_url(url)
-
-    def copy(self):
-        return deepcopy(self)
-
-    def set_meta(self, key: str, value: Any) -> None:
-        self._meta[key] = value
-
-    def _set_url(self, url: str) -> None:
-        if not isinstance(url, str):
-            raise TypeError(f"Request url must be str, got {type(url).__name__}")
-
-        s = safe_url_string(url, self.encoding)
-        self._url = escape_ajax(s)
-
-        if (
-                "://" not in self._url
-                and not self._url.startswith("about:")
-                and not self._url.startswith("data:")
-        ):
-            raise ValueError(f"Missing scheme in request url: {self._url}")
-
-    @property
-    def url(self) -> str:
-        return self._url
-
-    @property
-    def meta(self) -> Dict[str, Any]:
-        return self._meta
-
-    def __str__(self) -> str:
-        return f'<Request url={self.url} method={self.method}>'
-
-    def __repr__(self) -> str:
-        return self.__str__()
-
-    def __lt__(self, other) -> bool:
-        return self.priority < other.priority
-
-
-# #!/usr/bin/python
-# # -*- coding:UTF-8 -*-
-# import hashlib
-# from copy import deepcopy
-# from w3lib.url import safe_url_string
-# from typing import Dict, Optional, Callable, Union
-#
-# from crawlo.utils.url import escape_ajax
-#
-#
-# class Request(object):
-#
-#     def __init__(
-#             self,
-#             url: str,
-#             *,
-#             callback: Optional[Callable] = None,
-#             headers: Optional[Dict[str, str]] = None,
-#             body: Optional[Dict] = None,
-#             method: Optional[str] = 'GET',
-#             cookies: Optional[Dict[str, str]] = None,
-#             priority: int = 0,
-#             encoding: Optional[str] = 'UTF-8',
-#             meta: Optional[Dict[str, str]] = None,
-#             dont_filter: bool = False
-#
-#     ):
-#         self.url = url
-#         self.callback = callback
-#         self.headers = headers if headers else {}
-#         self.body = body
-#         self.method = str(method).upper()
-#         self.cookies = cookies
-#         self.priority = -priority
-#         self.encoding = encoding
-#         self.dont_filter = dont_filter
-#         self._meta = meta if meta is not None else {}
-#
-#     def copy(self):
-#         return deepcopy(self)
-#
-#     def set_meta(self, key: str, value: str):
-#         self._meta[key] = value
-#
-#     def _set_url(self, url: str) -> None:
-#         if not isinstance(url, str):
-#             raise TypeError(f"Request url must be str, got {type(url).__name__}")
-#
-#         s = safe_url_string(url, self.encoding)
-#         self._url = escape_ajax(s)
-#
-#         if (
-#             "://" not in self._url
-#             and not self._url.startswith("about:")
-#             and not self._url.startswith("data:")
-#         ):
-#             raise ValueError(f"Missing scheme in request url: {self._url}")
-#
-#     @property
-#     def meta(self):
-#         return self._meta
-#
-#     def __str__(self):
-#         return f'<Request url={self.url}> method={self.method}>'
-#
-#     def __lt__(self, other):
-#         return self.priority < other.priority
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+import json
+from copy import deepcopy
+from urllib.parse import urlencode
+from w3lib.url import safe_url_string
+from typing import Dict, Optional, Callable, Union, Any, TypeVar, List
+
+from crawlo.utils.url import escape_ajax
+
+
+_Request = TypeVar("_Request", bound="Request")
+
+
+class RequestPriority:
+    """请求优先级常量"""
+    HIGH = -100
+    NORMAL = 0
+    LOW = 100
+
+
+class Request:
+    """
+    封装一个 HTTP 请求对象，用于爬虫框架中表示一个待抓取的请求任务。
+    支持 JSON、表单、原始 body 提交，自动处理 Content-Type 与编码。
+    不支持文件上传（multipart/form-data），保持轻量。
+    """
+
+    __slots__ = (
+        '_url',
+        '_meta',
+        'callback',
+        'cb_kwargs',
+        'err_back',
+        'headers',
+        'body',
+        'method',
+        'cookies',
+        'priority',
+        'encoding',
+        'dont_filter',
+        'timeout',
+        'proxy',
+        'allow_redirects',
+        'auth',
+        'verify',
+        'flags',
+        # 保留高层参数用于 copy()
+        '_json_body',
+        '_form_data'
+    )
+
+    def __init__(
+        self,
+        url: str,
+        callback: Optional[Callable] = None,
+        method: Optional[str] = 'GET',
+        headers: Optional[Dict[str, str]] = None,
+        body: Optional[Union[bytes, str, Dict[Any, Any]]] = None,
+        form_data: Optional[Dict[Any, Any]] = None,
+        json_body: Optional[Dict[Any, Any]] = None,
+        cb_kwargs: Optional[Dict[str, Any]] = None,
+        cookies: Optional[Dict[str, str]] = None,
+        meta: Optional[Dict[str, Any]] = None,
+        priority: int = RequestPriority.NORMAL,
+        dont_filter: bool = False,
+        timeout: Optional[float] = None,
+        proxy: Optional[str] = None,
+        allow_redirects: bool = True,
+        auth: Optional[tuple] = None,
+        verify: bool = True,
+        flags: Optional[List[str]] = None,
+        encoding: str = 'utf-8'
+    ):
+        """
+        初始化请求对象。
+
+        :param url: 请求 URL（必须）
+        :param callback: 成功回调函数
+        :param method: HTTP 方法，默认 GET
+        :param headers: 请求头
+        :param body: 原始请求体（bytes/str），若为 dict 且未使用 json_body/form_data，则自动转为 JSON
+        :param form_data: 表单数据，自动转为 application/x-www-form-urlencoded
+        :param json_body: JSON 数据，自动序列化并设置 Content-Type
+        :param cb_kwargs: 传递给 callback 的额外参数
+        :param cookies: Cookies 字典
+        :param meta: 元数据（跨中间件传递数据）
+        :param priority: 优先级（数值越小越优先）
+        :param dont_filter: 是否跳过去重
+        :param timeout: 超时时间（秒）
+        :param proxy: 代理地址，如 http://127.0.0.1:8080
+        :param allow_redirects: 是否允许重定向
+        :param auth: 认证元组 (username, password)
+        :param verify: 是否验证 SSL 证书
+        :param flags: 标记（用于调试或分类）
+        :param encoding: 字符编码，默认 utf-8
+        """
+        self.callback = callback
+        self.method = str(method).upper()
+        self.headers = headers or {}
+        self.cookies = cookies or {}
+        self.priority = -priority  # 用于排序：值越小优先级越高
+        self._meta = deepcopy(meta) if meta is not None else {}
+        self.timeout = self._meta.get('download_timeout', timeout)
+        self.proxy = proxy
+        self.allow_redirects = allow_redirects
+        self.auth = auth
+        self.verify = verify
+        self.flags = flags or []
+        self.encoding = encoding
+        self.cb_kwargs = cb_kwargs or {}
+        self.body = body
+        # 保存高层语义参数（用于 copy）
+        self._json_body = json_body
+        self._form_data = form_data
+
+        # 构建 body
+        if json_body is not None:
+            if 'Content-Type' not in self.headers:
+                self.headers['Content-Type'] = 'application/json'
+            self.body = json.dumps(json_body, ensure_ascii=False).encode(encoding)
+            if self.method == 'GET':
+                self.method = 'POST'
+
+        elif form_data is not None:
+            if self.method == 'GET':
+                self.method = 'POST'
+            if 'Content-Type' not in self.headers:
+                self.headers['Content-Type'] = 'application/x-www-form-urlencoded'
+            query_str = urlencode(form_data)
+            self.body = query_str.encode(encoding)  # ✅ 显式编码为 bytes
+
+
+        else:
+            # 处理原始 body
+            if isinstance(self.body, dict):
+                if 'Content-Type' not in self.headers:
+                    self.headers['Content-Type'] = 'application/json'
+                self.body = json.dumps(self.body, ensure_ascii=False).encode(encoding)
+            elif isinstance(self.body, str):
+                self.body = self.body.encode(encoding)
+
+        self.dont_filter = dont_filter
+        self._set_url(url)
+
+    def copy(self: _Request) -> _Request:
+        """
+        创建当前请求的副本，保留所有高层语义（json_body/form_data）。
+        """
+        return type(self)(
+            url=self.url,
+            callback=self.callback,
+            method=self.method,
+            headers=self.headers.copy(),
+            body=None,  # 由 form_data/json_body 重新生成
+            form_data=self._form_data,
+            json_body=self._json_body,
+            cb_kwargs=deepcopy(self.cb_kwargs),
+            err_back=self.err_back,
+            cookies=self.cookies.copy(),
+            meta=deepcopy(self._meta),
+            priority=-self.priority,
+            dont_filter=self.dont_filter,
+            timeout=self.timeout,
+            proxy=self.proxy,
+            allow_redirects=self.allow_redirects,
+            auth=self.auth,
+            verify=self.verify,
+            flags=self.flags.copy(),
+            encoding=self.encoding
+        )
+
+    def set_meta(self, key: str, value: Any) -> None:
+        """设置 meta 中的某个键值。"""
+        self._meta[key] = value
+
+    def _set_url(self, url: str) -> None:
+        """安全设置 URL，确保格式正确。"""
+        if not isinstance(url, str):
+            raise TypeError(f"Request url 必须为字符串，当前类型: {type(url).__name__}")
+
+        s = safe_url_string(url, self.encoding)
+        escaped_url = escape_ajax(s)
+        self._url = escaped_url
+
+        if not self._url.startswith(('http://', 'https://')):
+            raise ValueError(f"URL 缺少 scheme: {self._url}")
+
+    @property
+    def url(self) -> str:
+        return self._url
+
+    @property
+    def meta(self) -> Dict[str, Any]:
+        return self._meta
+
+    def __str__(self) -> str:
+        return f'<Request url={self.url} method={self.method}>'
+
+    def __repr__(self) -> str:
+        return str(self)
+
+    def __lt__(self, other: _Request) -> bool:
+        """用于按优先级排序"""
+        return self.priority < other.priority

crawlo/network/response.py

@@ -1,93 +1,166 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-import re
-import ujson
-from typing import Dict
-from parsel import Selector
-from http.cookies import SimpleCookie
-from urllib.parse import urljoin as _urljoin
-
-from crawlo import Request
-from crawlo.exceptions import DecodeError
-
-
-class Response(object):
-
-    def __init__(
-            self,
-            url: str,
-            *,
-            headers: Dict,
-            body: bytes = b"",
-            method: str = 'GET',
-            request: Request = None,
-            status_code: int = 200,
-    ):
-        self.url = url
-        self.headers = headers
-        self.body = body
-        self.method = method
-        self.request = request
-        self.status_code = status_code
-        self.encoding = request.encoding
-        self._selector = None
-        self._text_cache = None
-
-    @property
-    def text(self):
-        # 请求缓存
-        if self._text_cache:
-            return self._text_cache
-        try:
-            self._text_cache = self.body.decode(self.encoding)
-        except UnicodeDecodeError:
-            try:
-                _encoding_re = re.compile(r"charset=([\w-]+)", flags=re.I)
-                _encoding_string = self.headers.get('Content-Type', '') or self.headers.get('content-type', '')
-                _encoding = _encoding_re.search(_encoding_string)
-                if _encoding:
-                    _encoding = _encoding.group(1)
-                    self._text_cache = self.body.decode(_encoding)
-                else:
-                    raise DecodeError(f"{self.request} {self.request.encoding} error.")
-            except UnicodeDecodeError as exp:
-                raise UnicodeDecodeError(
-                    exp.encoding, exp.object, exp.start, exp.end, f"{self.request} error."
-                )
-        return self._text_cache
-
-    def json(self):
-        return ujson.loads(self.text)
-
-    def urljoin(self, url):
-        return _urljoin(self.url, url)
-
-    def xpath(self, xpath_str):
-        if self._selector is None:
-            self._selector = Selector(self.text)
-        return self._selector.xpath(xpath_str)
-
-    def css(self, css_str):
-        if self._selector is None:
-            self._selector = Selector(self.text)
-        return self._selector.css(css_str)
-
-    def re_search(self, pattern, flags=re.DOTALL):
-        return re.search(pattern, self.text, flags=flags)
-
-    def re_findall(self, pattern, flags=re.DOTALL):
-        return re.findall(pattern, self.text, flags=flags)
-
-    def get_cookies(self):
-        cookie_headers = self.headers.getlist('Set-Cookie') or []
-        cookies = SimpleCookie()
-        for header in cookie_headers:
-            cookies.load(header)
-        return {k: v.value for k, v in cookies.items()}
-
-    @property
-    def meta(self):
-        return self.request.meta
-
-    def __str__(self):
-        return f"{self.url} {self.status_code} {self.request.encoding} "
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+import re
+import ujson
+from typing import Dict, Any, List, Optional
+from parsel import Selector, SelectorList
+from http.cookies import SimpleCookie
+from urllib.parse import urljoin as _urljoin
+
+from crawlo import Request
+from crawlo.exceptions import DecodeError
+
+
+class Response:
+    """
+    HTTP响应的封装，提供数据解析的便捷方法。
+    """
+
+    def __init__(
+        self,
+        url: str,
+        *,
+        headers: Dict[str, Any],
+        body: bytes = b"",
+        method: str = 'GET',
+        request: Request = None,
+        status_code: int = 200,
+    ):
+        self.url = url
+        self.headers = headers
+        self.body = body
+        self.method = method
+        self.request = request
+        self.status_code = status_code
+        self.encoding = self.request.encoding if self.request else None
+        self._text_cache = None
+        self._json_cache = None
+        self._selector_instance = None  # 修改变量名，避免与 @property 冲突
+
+    @property
+    def text(self) -> str:
+        """将响应体(body)以正确的编码解码为字符串，并缓存结果。"""
+        if self._text_cache is not None:
+            return self._text_cache
+
+        encoding = self.encoding
+        try:
+            # 优先使用 request 提供的编码
+            if encoding:
+                self._text_cache = self.body.decode(encoding)
+                return self._text_cache
+
+            # 从 Content-Type 中提取编码
+            content_type = self.headers.get("Content-Type", "")
+            charset_match = re.search(r"charset=([\w-]+)", content_type, re.I)
+            if charset_match:
+                encoding = charset_match.group(1)
+                self._text_cache = self.body.decode(encoding)
+                return self._text_cache
+
+            # 默认尝试 UTF-8
+            self._text_cache = self.body.decode("utf-8")
+            return self._text_cache
+
+        except UnicodeDecodeError as e:
+            raise DecodeError(f"Failed to decode response from {self.url}: {e}")
+
+    def json(self) -> Any:
+        """将响应文本解析为 JSON 对象。"""
+        if self._json_cache:
+            return self._json_cache
+        self._json_cache = ujson.loads(self.text)
+        return self._json_cache
+
+    def urljoin(self, url: str) -> str:
+        """拼接 URL，自动处理相对路径。"""
+        return _urljoin(self.url, url)
+
+    @property
+    def _selector(self) -> Selector:
+        """懒加载 Selector 实例"""
+        if self._selector_instance is None:
+            self._selector_instance = Selector(self.text)
+        return self._selector_instance
+
+    def xpath(self, query: str) -> SelectorList:
+        """使用 XPath 选择器查询文档。"""
+        return self._selector.xpath(query)
+
+    def css(self, query: str) -> SelectorList:
+        """使用 CSS 选择器查询文档。"""
+        return self._selector.css(query)
+
+    def xpath_text(self, query: str) -> str:
+        """使用 XPath 提取并返回纯文本。"""
+        fragments = self.xpath(f"{query}//text()").getall()
+        return " ".join(text.strip() for text in fragments if text.strip())
+
+    def css_text(self, query: str) -> str:
+        """使用 CSS 选择器提取并返回纯文本。"""
+        fragments = self.css(f"{query} ::text").getall()
+        return " ".join(text.strip() for text in fragments if text.strip())
+
+    def get_text(self, xpath_or_css: str, join_str: str = " ") -> str:
+        """
+        获取指定节点的纯文本(自动拼接子节点文本)
+
+        参数:
+            xpath_or_css: XPath或CSS选择器
+            join_str: 文本拼接分隔符(默认为空格)
+
+        返回:
+            拼接后的纯文本字符串
+        """
+        elements = self.xpath(xpath_or_css) if xpath_or_css.startswith(('/', '//', './')) else self.css(xpath_or_css)
+        texts = elements.xpath('.//text()').getall()
+        return join_str.join(text.strip() for text in texts if text.strip())
+
+    def get_all_text(self, xpath_or_css: str, join_str: str = " ") -> List[str]:
+        """
+        获取多个节点的纯文本列表
+
+        参数:
+            xpath_or_css: XPath或CSS选择器
+            join_str: 单个节点内文本拼接分隔符
+
+        返回:
+            纯文本列表(每个元素对应一个节点的文本)
+        """
+        elements = self.xpath(xpath_or_css) if xpath_or_css.startswith(('/', '//', './')) else self.css(xpath_or_css)
+        result = []
+        for element in elements:
+            texts = element.xpath('.//text()').getall()
+            clean_text = join_str.join(text.strip() for text in texts if text.strip())
+            if clean_text:
+                result.append(clean_text)
+        return result
+
+    def re_search(self, pattern: str, flags: int = re.DOTALL) -> Optional[re.Match]:
+        """在响应文本上执行正则表达式搜索。"""
+        if not isinstance(pattern, str):
+            raise TypeError("Pattern must be a string")
+        return re.search(pattern, self.text, flags=flags)
+
+    def re_findall(self, pattern: str, flags: int = re.DOTALL) -> List[Any]:
+        """在响应文本上执行正则表达式查找。"""
+        if not isinstance(pattern, str):
+            raise TypeError("Pattern must be a string")
+        return re.findall(pattern, self.text, flags=flags)
+
+    def get_cookies(self) -> Dict[str, str]:
+        """从响应头中解析并返回Cookies。"""
+        cookie_header = self.headers.get("Set-Cookie", "")
+        if isinstance(cookie_header, list):
+            cookie_header = ", ".join(cookie_header)
+        cookies = SimpleCookie()
+        cookies.load(cookie_header)
+        return {key: morsel.value for key, morsel in cookies.items()}
+
+    @property
+    def meta(self) -> Dict:
+        """获取关联的 Request 对象的 meta 字典。"""
+        return self.request.meta if self.request else {}
+
+    def __str__(self):
+        return f"<{self.status_code} {self.url}>"

crawlo/pipelines/__init__.py

@@ -1,13 +1,13 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-from crawlo.items.items import Item
-
-
-class BasePipeline:
-
-    def process_item(self, item: Item, spider):
-        raise NotImplementedError
-
-    @classmethod
-    def create_instance(cls, crawler):
-        return cls()
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+from crawlo.items.items import Item
+
+
+class BasePipeline:
+
+    def process_item(self, item: Item, spider):
+        raise NotImplementedError
+
+    @classmethod
+    def create_instance(cls, crawler):
+        return cls()