crawlo 1.0.1__py3-none-any.whl → 1.0.3__py3-none-any.whl

crawlo/items/__init__.py

@@ -1,62 +1,62 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-from abc import ABCMeta
-from typing import Any, Optional, Type
-
-
-class Field:
-    def __init__(
-        self,
-        nullable: bool = True,
-        *,
-        default: Any = None,
-        field_type: Optional[Type] = None,
-        max_length: Optional[int] = None,
-        description: str = ""
-    ):
-        self.nullable = nullable
-        self.default = default
-        self.field_type = field_type
-        self.max_length = max_length
-        self.description = description
-
-    def validate(self, value: Any, field_name: str = "") -> Any:
-        if value is None or (isinstance(value, str) and value.strip() == ""):
-            if self.default is not None:
-                return self.default
-            elif not self.nullable:
-                raise ValueError(
-                    f"字段 '{field_name}' 不允许为空。"
-                )
-
-        if value is not None and not (isinstance(value, str) and value.strip() == ""):
-            if self.field_type and not isinstance(value, self.field_type):
-                raise TypeError(
-                    f"字段 '{field_name}' 类型错误：期望类型 {self.field_type}, 得到 {type(value)}，值：{value!r}"
-                )
-            if self.max_length and len(str(value)) > self.max_length:
-                raise ValueError(
-                    f"字段 '{field_name}' 长度超限：最大长度 {self.max_length}，当前长度 {len(str(value))}，值：{value!r}"
-                )
-
-        return value
-
-    def __repr__(self):
-        return f"<Field required={self.nullable} type={self.field_type} default={self.default}>"
-
-
-class ItemMeta(ABCMeta):
-    """
-    元类
-    """
-    def __new__(mcs, name, bases, attrs):
-        field = {}
-        cls_attr = {}
-        for k, v in attrs.items():
-            if isinstance(v, Field):
-                field[k] = v
-            else:
-                cls_attr[k] = v
-        cls_instance = super().__new__(mcs, name, bases, attrs)
-        cls_instance.FIELDS = field
-        return cls_instance
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+from abc import ABCMeta
+from typing import Any, Optional, Type
+
+
+class Field:
+    def __init__(
+        self,
+        nullable: bool = True,
+        *,
+        default: Any = None,
+        field_type: Optional[Type] = None,
+        max_length: Optional[int] = None,
+        description: str = ""
+    ):
+        self.nullable = nullable
+        self.default = default
+        self.field_type = field_type
+        self.max_length = max_length
+        self.description = description
+
+    def validate(self, value: Any, field_name: str = "") -> Any:
+        if value is None or (isinstance(value, str) and value.strip() == ""):
+            if self.default is not None:
+                return self.default
+            elif not self.nullable:
+                raise ValueError(
+                    f"字段 '{field_name}' 不允许为空。"
+                )
+
+        if value is not None and not (isinstance(value, str) and value.strip() == ""):
+            if self.field_type and not isinstance(value, self.field_type):
+                raise TypeError(
+                    f"字段 '{field_name}' 类型错误：期望类型 {self.field_type}, 得到 {type(value)}，值：{value!r}"
+                )
+            if self.max_length and len(str(value)) > self.max_length:
+                raise ValueError(
+                    f"字段 '{field_name}' 长度超限：最大长度 {self.max_length}，当前长度 {len(str(value))}，值：{value!r}"
+                )
+
+        return value
+
+    def __repr__(self):
+        return f"<Field required={self.nullable} type={self.field_type} default={self.default}>"
+
+
+class ItemMeta(ABCMeta):
+    """
+    元类
+    """
+    def __new__(mcs, name, bases, attrs):
+        field = {}
+        cls_attr = {}
+        for k, v in attrs.items():
+            if isinstance(v, Field):
+                field[k] = v
+            else:
+                cls_attr[k] = v
+        cls_instance = super().__new__(mcs, name, bases, attrs)
+        cls_instance.FIELDS = field
+        return cls_instance

crawlo/items/items.py

@@ -1,119 +1,119 @@
-#!/usr/bin/python
-# -*- coding: UTF-8 -*-
-from copy import deepcopy
-from pprint import pformat
-from typing import Any, Iterator, Dict
-from collections.abc import MutableMapping
-
-from crawlo.items import ItemMeta, Field
-from crawlo.exceptions import ItemInitError, ItemAttributeError
-
-
-class Item(MutableMapping, metaclass=ItemMeta):
-    FIELDS: Dict[str, Any] = {}
-
-    def __init__(self, *args, **kwargs):
-        if args:
-            raise ItemInitError(f"{self.__class__.__name__} 不支持位置参数：{args}，请使用关键字参数初始化。")
-        if kwargs:
-            for key, value in kwargs.items():
-                self[key] = value
-
-        self._values: Dict[str, Any] = {}
-
-        # 初始化字段，默认值填充
-        for field_name, field_obj in self.FIELDS.items():
-            if field_obj.default is not None:
-                self._values[field_name] = field_obj.default
-
-        # 覆盖默认值或设置新值
-        for key, value in kwargs.items():
-            self[key] = value
-
-    def __getitem__(self, item: str) -> Any:
-        return self._values[item]
-
-    # def __setitem__(self, key: str, value: Any) -> None:
-    #     if key not in self.FIELDS:
-    #         raise KeyError(f"{self.__class__.__name__} 不包含字段：{key}")
-    #     self._values[key] = value
-
-    def __setitem__(self, key: str, value: Any) -> None:
-        if key not in self.FIELDS:
-            raise KeyError(f"{self.__class__.__name__} 不包含字段：{key}")
-
-        field = self.FIELDS[key]
-        try:
-            validated_value = field.validate(value, field_name=key)
-            self._values[key] = validated_value
-        except Exception as e:
-            error_lines = [
-                "",
-                "【字段校验失败】",
-                f"字段名称: {key}",
-                f"数据类型: {type(value)}",
-                f"原始值:   {repr(value)}",
-                f"是否允许空值: {field.nullable}",
-                f"错误原因: {str(e)}",
-                ""
-            ]
-            detailed_error = "\n".join(error_lines)
-            raise type(e)(detailed_error) from e
-
-    def __delitem__(self, key: str) -> None:
-        del self._values[key]
-
-    def __setattr__(self, key: str, value: Any) -> None:
-        if not key.startswith("_"):
-            raise AttributeError(
-                f"设置字段值请使用 item[{key!r}] = {value!r}"
-            )
-        super().__setattr__(key, value)
-
-    def __getattr__(self, item: str) -> Any:
-        # 当获取不到属性时触发
-        raise AttributeError(
-            f"{self.__class__.__name__} 不支持字段：{item}。"
-            f"请先在 `{self.__class__.__name__}` 中声明该字段，再通过 item[{item!r}] 获取。"
-        )
-
-    def __getattribute__(self, item: str) -> Any:
-        # 属性拦截器，只要访问属性就会进入该方法
-        try:
-            field = super().__getattribute__("FIELDS")
-            if isinstance(field, dict) and item in field:
-                raise ItemAttributeError(
-                    f"获取字段值请使用 item[{item!r}]"
-                )
-        except AttributeError:
-            pass  # 如果 FIELDS 尚未定义，继续执行后续逻辑
-        return super().__getattribute__(item)
-
-    def __repr__(self) -> str:
-        return pformat(dict(self))
-
-    __str__ = __repr__
-
-    def __iter__(self) -> Iterator[str]:
-        return iter(self._values)
-
-    def __len__(self) -> int:
-        return len(self._values)
-
-    def to_dict(self) -> Dict[str, Any]:
-        return dict(self)
-
-    def copy(self) -> "Item":
-        return deepcopy(self)
-
-
-if __name__ == '__main__':
-    class TestItem(Item):
-        url = Field(nullable=False, field_type=str, max_length=100)
-        title = Field(default="无标题", field_type=str)
-
-    test_item = TestItem()
-    test_item['title'] = '百度首页'
-    test_item['url'] = 'hhh'
-    # test_item.title = 'fffff'
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+from copy import deepcopy
+from pprint import pformat
+from typing import Any, Iterator, Dict
+from collections.abc import MutableMapping
+
+from crawlo.items import ItemMeta, Field
+from crawlo.exceptions import ItemInitError, ItemAttributeError
+
+
+class Item(MutableMapping, metaclass=ItemMeta):
+    FIELDS: Dict[str, Any] = {}
+
+    def __init__(self, *args, **kwargs):
+        if args:
+            raise ItemInitError(f"{self.__class__.__name__} 不支持位置参数：{args}，请使用关键字参数初始化。")
+        if kwargs:
+            for key, value in kwargs.items():
+                self[key] = value
+
+        self._values: Dict[str, Any] = {}
+
+        # 初始化字段，默认值填充
+        for field_name, field_obj in self.FIELDS.items():
+            if field_obj.default is not None:
+                self._values[field_name] = field_obj.default
+
+        # 覆盖默认值或设置新值
+        for key, value in kwargs.items():
+            self[key] = value
+
+    def __getitem__(self, item: str) -> Any:
+        return self._values[item]
+
+    # def __setitem__(self, key: str, value: Any) -> None:
+    #     if key not in self.FIELDS:
+    #         raise KeyError(f"{self.__class__.__name__} 不包含字段：{key}")
+    #     self._values[key] = value
+
+    def __setitem__(self, key: str, value: Any) -> None:
+        if key not in self.FIELDS:
+            raise KeyError(f"{self.__class__.__name__} 不包含字段：{key}")
+
+        field = self.FIELDS[key]
+        try:
+            validated_value = field.validate(value, field_name=key)
+            self._values[key] = validated_value
+        except Exception as e:
+            error_lines = [
+                "",
+                "【字段校验失败】",
+                f"字段名称: {key}",
+                f"数据类型: {type(value)}",
+                f"原始值:   {repr(value)}",
+                f"是否允许空值: {field.nullable}",
+                f"错误原因: {str(e)}",
+                ""
+            ]
+            detailed_error = "\n".join(error_lines)
+            raise type(e)(detailed_error) from e
+
+    def __delitem__(self, key: str) -> None:
+        del self._values[key]
+
+    def __setattr__(self, key: str, value: Any) -> None:
+        if not key.startswith("_"):
+            raise AttributeError(
+                f"设置字段值请使用 item[{key!r}] = {value!r}"
+            )
+        super().__setattr__(key, value)
+
+    def __getattr__(self, item: str) -> Any:
+        # 当获取不到属性时触发
+        raise AttributeError(
+            f"{self.__class__.__name__} 不支持字段：{item}。"
+            f"请先在 `{self.__class__.__name__}` 中声明该字段，再通过 item[{item!r}] 获取。"
+        )
+
+    def __getattribute__(self, item: str) -> Any:
+        # 属性拦截器，只要访问属性就会进入该方法
+        try:
+            field = super().__getattribute__("FIELDS")
+            if isinstance(field, dict) and item in field:
+                raise ItemAttributeError(
+                    f"获取字段值请使用 item[{item!r}]"
+                )
+        except AttributeError:
+            pass  # 如果 FIELDS 尚未定义，继续执行后续逻辑
+        return super().__getattribute__(item)
+
+    def __repr__(self) -> str:
+        return pformat(dict(self))
+
+    __str__ = __repr__
+
+    def __iter__(self) -> Iterator[str]:
+        return iter(self._values)
+
+    def __len__(self) -> int:
+        return len(self._values)
+
+    def to_dict(self) -> Dict[str, Any]:
+        return dict(self)
+
+    def copy(self) -> "Item":
+        return deepcopy(self)
+
+
+if __name__ == '__main__':
+    class TestItem(Item):
+        url = Field(nullable=False, field_type=str, max_length=100)
+        title = Field(default="无标题", field_type=str)
+
+    test_item = TestItem()
+    test_item['title'] = '百度首页'
+    test_item['url'] = 'hhh'
+    # test_item.title = 'fffff'
     print(test_item)

crawlo/middleware/__init__.py

@@ -1,21 +1,21 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-from crawlo import Request, Response
-
-
-class BaseMiddleware(object):
-    def process_request(self, request, spider) -> None | Request | Response:
-        # 请求预处理
-        pass
-
-    def process_response(self, request, response, spider) -> Request | Response:
-        # 响应预处理
-        pass
-
-    def process_exception(self, request, exp, spider) -> None | Request | Response:
-        # 异常预处理
-        pass
-
-    @classmethod
-    def create_instance(cls, crawler):
-        return cls()
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+from crawlo import Request, Response
+
+
+class BaseMiddleware(object):
+    def process_request(self, request, spider) -> None | Request | Response:
+        # 请求预处理
+        pass
+
+    def process_response(self, request, response, spider) -> Request | Response:
+        # 响应预处理
+        pass
+
+    def process_exception(self, request, exp, spider) -> None | Request | Response:
+        # 异常预处理
+        pass
+
+    @classmethod
+    def create_instance(cls, crawler):
+        return cls()

crawlo/middleware/default_header.py

@@ -1,32 +1,32 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-from crawlo.event import spider_opened
-
-
-class DefaultHeaderMiddleware(object):
-
-    def __init__(self, user_agent, headers, spider):
-        self.user_agent = user_agent
-        self.headers = headers
-        self.spider = spider
-
-    @classmethod
-    def create_instance(cls, crawler):
-        o = cls(
-            user_agent=crawler.settings.get('USER_AGENT'),
-            headers=crawler.settings.get_dict('DEFAULT_HEADERS'),
-            spider=crawler.spider
-        )
-        crawler.subscriber.subscribe(o.spider_opened, event=spider_opened)
-        return o
-
-    async def spider_opened(self):
-        self.user_agent = getattr(self.spider, 'user_agent', self.user_agent)
-        self.headers = getattr(self.spider, 'headers', self.headers)
-        if self.user_agent:
-            self.headers.setdefault('User-Agent', self.user_agent)
-
-    def process_request(self, request, _spider):
-        if self.headers:
-            for key, value in self.headers.items():
-                request.headers.setdefault(key, value)
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+from crawlo.event import spider_opened
+
+
+class DefaultHeaderMiddleware(object):
+
+    def __init__(self, user_agent, headers, spider):
+        self.user_agent = user_agent
+        self.headers = headers
+        self.spider = spider
+
+    @classmethod
+    def create_instance(cls, crawler):
+        o = cls(
+            user_agent=crawler.settings.get('USER_AGENT'),
+            headers=crawler.settings.get_dict('DEFAULT_HEADERS'),
+            spider=crawler.spider
+        )
+        crawler.subscriber.subscribe(o.spider_opened, event=spider_opened)
+        return o
+
+    async def spider_opened(self):
+        self.user_agent = getattr(self.spider, 'user_agent', self.user_agent)
+        self.headers = getattr(self.spider, 'headers', self.headers)
+        if self.user_agent:
+            self.headers.setdefault('User-Agent', self.user_agent)
+
+    def process_request(self, request, _spider):
+        if self.headers:
+            for key, value in self.headers.items():
+                request.headers.setdefault(key, value)

crawlo/middleware/download_delay.py

@@ -1,28 +1,28 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-from asyncio import sleep
-from random import uniform
-from crawlo.utils.log import get_logger
-from crawlo.exceptions import NotConfiguredError
-
-
-class DownloadDelayMiddleware(object):
-
-    def __init__(self, settings, log_level):
-        self.delay = settings.get_float("DOWNLOAD_DELAY")
-        if not self.delay:
-            raise NotConfiguredError
-        self.randomness = settings.get_bool("RANDOMNESS")
-        self.floor, self.upper = settings.get_list("RANDOM_RANGE")
-        self.logger = get_logger(self.__class__.__name__, log_level)
-
-    @classmethod
-    def create_instance(cls, crawler):
-        o = cls(settings=crawler.settings, log_level=crawler.settings.get('LOG_LEVEL'))
-        return o
-
-    async def process_request(self, _request, _spider):
-        if self.randomness:
-            await sleep(uniform(self.delay * self.floor, self.delay * self.upper))
-        else:
-            await sleep(self.delay)
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+from asyncio import sleep
+from random import uniform
+from crawlo.utils.log import get_logger
+from crawlo.exceptions import NotConfiguredError
+
+
+class DownloadDelayMiddleware(object):
+
+    def __init__(self, settings, log_level):
+        self.delay = settings.get_float("DOWNLOAD_DELAY")
+        if not self.delay:
+            raise NotConfiguredError
+        self.randomness = settings.get_bool("RANDOMNESS")
+        self.floor, self.upper = settings.get_list("RANDOM_RANGE")
+        self.logger = get_logger(self.__class__.__name__, log_level)
+
+    @classmethod
+    def create_instance(cls, crawler):
+        o = cls(settings=crawler.settings, log_level=crawler.settings.get('LOG_LEVEL'))
+        return o
+
+    async def process_request(self, _request, _spider):
+        if self.randomness:
+            await sleep(uniform(self.delay * self.floor, self.delay * self.upper))
+        else:
+            await sleep(self.delay)