crawlo 1.0.5__py3-none-any.whl → 1.0.6__py3-none-any.whl

crawlo/items/__init__.py

@@ -1,62 +1,22 @@
 #!/usr/bin/python
-# -*- coding:UTF-8 -*-
-from abc import ABCMeta
-from typing import Any, Optional, Type
+# -*- coding: UTF-8 -*-
+"""
+crawlo.items 包
+===============
+提供 Item 和 Field 类用于数据定义和验证。
+"""
+from .fields import Field
+from .items import Item
+from .base import ItemMeta
+from crawlo.exceptions import ItemInitError, ItemAttributeError
+
+__all__ = [
+    'Item',
+    'Field',
+    'ItemMeta',
+    'ItemInitError',
+    'ItemAttributeError'
+]
 
 
-class Field:
-    def __init__(
-        self,
-        nullable: bool = True,
-        *,
-        default: Any = None,
-        field_type: Optional[Type] = None,
-        max_length: Optional[int] = None,
-        description: str = ""
-    ):
-        self.nullable = nullable
-        self.default = default
-        self.field_type = field_type
-        self.max_length = max_length
-        self.description = description
 
-    def validate(self, value: Any, field_name: str = "") -> Any:
-        if value is None or (isinstance(value, str) and value.strip() == ""):
-            if self.default is not None:
-                return self.default
-            elif not self.nullable:
-                raise ValueError(
-                    f"字段 '{field_name}' 不允许为空。"
-                )
-
-        if value is not None and not (isinstance(value, str) and value.strip() == ""):
-            if self.field_type and not isinstance(value, self.field_type):
-                raise TypeError(
-                    f"字段 '{field_name}' 类型错误：期望类型 {self.field_type}, 得到 {type(value)}，值：{value!r}"
-                )
-            if self.max_length and len(str(value)) > self.max_length:
-                raise ValueError(
-                    f"字段 '{field_name}' 长度超限：最大长度 {self.max_length}，当前长度 {len(str(value))}，值：{value!r}"
-                )
-
-        return value
-
-    def __repr__(self):
-        return f"<Field required={self.nullable} type={self.field_type} default={self.default}>"
-
-
-class ItemMeta(ABCMeta):
-    """
-    元类
-    """
-    def __new__(mcs, name, bases, attrs):
-        field = {}
-        cls_attr = {}
-        for k, v in attrs.items():
-            if isinstance(v, Field):
-                field[k] = v
-            else:
-                cls_attr[k] = v
-        cls_instance = super().__new__(mcs, name, bases, attrs)
-        cls_instance.FIELDS = field
-        return cls_instance

crawlo/items/base.py

@@ -0,0 +1,31 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+基础元类定义
+"""
+from abc import ABCMeta
+
+from crawlo.items import Field
+
+
+class ItemMeta(ABCMeta):
+    """
+    元类，用于自动收集 Item 类中的 Field 定义
+    """
+
+    def __new__(mcs, name, bases, attrs):
+        fields = {}
+        cls_attrs = {}
+
+        # 收集所有 Field 实例
+        for attr_name, attr_value in attrs.items():
+            if isinstance(attr_value, Field):
+                fields[attr_name] = attr_value
+            else:
+                cls_attrs[attr_name] = attr_value
+
+        # 创建类实例
+        cls_instance = super().__new__(mcs, name, bases, cls_attrs)
+        cls_instance.FIELDS = fields
+
+        return cls_instance

crawlo/items/fields.py

@@ -0,0 +1,54 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+Field 类定义
+"""
+
+from typing import Any, Optional, Type
+
+
+class Field:
+    """
+    字段定义类，用于定义 Item 的字段属性和验证规则
+    """
+    def __init__(
+        self,
+        nullable: bool = True,
+        *,
+        default: Any = None,
+        field_type: Optional[Type] = None,
+        max_length: Optional[int] = None,
+        description: str = ""
+    ):
+        self.nullable = nullable
+        self.default = default
+        self.field_type = field_type
+        self.max_length = max_length
+        self.description = description
+
+    def validate(self, value: Any, field_name: str = "") -> Any:
+        """
+        验证字段值是否符合规则
+        """
+        if value is None or (isinstance(value, str) and value.strip() == ""):
+            if self.default is not None:
+                return self.default
+            elif not self.nullable:
+                raise ValueError(
+                    f"字段 '{field_name}' 不允许为空。"
+                )
+
+        if value is not None and not (isinstance(value, str) and value.strip() == ""):
+            if self.field_type and not isinstance(value, self.field_type):
+                raise TypeError(
+                    f"字段 '{field_name}' 类型错误：期望类型 {self.field_type}, 得到 {type(value)}，值：{value!r}"
+                )
+            if self.max_length and len(str(value)) > self.max_length:
+                raise ValueError(
+                    f"字段 '{field_name}' 长度超限：最大长度 {self.max_length}，当前长度 {len(str(value))}，值：{value!r}"
+                )
+
+        return value
+
+    def __repr__(self):
+        return f"<Field nullable={self.nullable} type={self.field_type} default={self.default}>"

crawlo/items/items.py

@@ -1,23 +1,26 @@
 #!/usr/bin/python
 # -*- coding: UTF-8 -*-
+"""
+Item 类定义
+"""
 from copy import deepcopy
 from pprint import pformat
 from typing import Any, Iterator, Dict
 from collections.abc import MutableMapping
 
-from crawlo.items import ItemMeta, Field
+from .base import ItemMeta
 from crawlo.exceptions import ItemInitError, ItemAttributeError
 
 
 class Item(MutableMapping, metaclass=ItemMeta):
+    """
+    数据项基类，用于定义结构化数据
+    """
     FIELDS: Dict[str, Any] = {}
 
     def __init__(self, *args, **kwargs):
         if args:
             raise ItemInitError(f"{self.__class__.__name__} 不支持位置参数：{args}，请使用关键字参数初始化。")
-        if kwargs:
-            for key, value in kwargs.items():
-                self[key] = value
 
         self._values: Dict[str, Any] = {}
 
@@ -66,14 +69,12 @@ class Item(MutableMapping, metaclass=ItemMeta):
         super().__setattr__(key, value)
 
     def __getattr__(self, item: str) -> Any:
-        # 当获取不到属性时触发
         raise AttributeError(
             f"{self.__class__.__name__} 不支持字段：{item}。"
             f"请先在 `{self.__class__.__name__}` 中声明该字段，再通过 item[{item!r}] 获取。"
         )
 
     def __getattribute__(self, item: str) -> Any:
-        # 属性拦截器，只要访问属性就会进入该方法
         try:
             field = super().__getattribute__("FIELDS")
             if isinstance(field, dict) and item in field:
@@ -96,20 +97,9 @@ class Item(MutableMapping, metaclass=ItemMeta):
         return len(self._values)
 
     def to_dict(self) -> Dict[str, Any]:
+        """转换为字典"""
         return dict(self)
 
     def copy(self) -> "Item":
-        return deepcopy(self)
-
-
-if __name__ == '__main__':
-    class TestItem(Item):
-        url = Field(nullable=False, field_type=str, max_length=100)
-        title = Field(default="无标题", field_type=str)
-
-
-    test_item = TestItem()
-    test_item['title'] = '百度首页'
-    test_item['url'] = 'hhh'
-    # test_item.title = 'fffff'
-    print(test_item)
+        """深拷贝当前 Item"""
+        return deepcopy(self)

crawlo/settings/default_settings.py

@@ -137,7 +137,7 @@ LOG_ENCODING = 'utf-8'
 
 # ============================== 代理配置 ==============================
 
-PROXY_ENABLED = True                       # 是否启用代理
+PROXY_ENABLED = False                       # 是否启用代理
 PROXY_API_URL = "https://api.proxyprovider.com/get"  # 代理获取接口（请替换为真实地址）
 
 # 代理提取方式（支持字段路径或函数）

crawlo/templates/crawlo.cfg.tmpl

@@ -0,0 +1,11 @@
+# crawlo.cfg
+# 项目的配置文件。
+
+[settings]
+# 指定 settings 模块的导入路径
+default = {{project_name}}.settings
+
+[deploy]
+# （可选）用于部署配置
+# url = http://localhost:6800/
+# project = {{project_name}}

crawlo/templates/project/__init__.py.tmpl

@@ -0,0 +1,4 @@
+# -*- coding: UTF-8 -*-
+"""
+{{project_name}} 项目包
+"""

crawlo/templates/project/items.py.tmpl

@@ -0,0 +1,18 @@
+# -*- coding: UTF-8 -*-
+"""
+{{project_name}}.items
+======================
+定义你抓取的数据结构。
+"""
+
+from crawlo.items import Item, Field
+
+
+class ExampleItem(Item):
+    """
+    一个示例数据项。
+    """
+    # name = Field()
+    # price = Field()
+    # description = Field()
+    pass

crawlo/templates/project/middlewares.py.tmpl

@@ -0,0 +1,76 @@
+# -*- coding: UTF-8 -*-
+"""
+{{project_name}}.middlewares
+============================
+自定义中间件，用于在请求/响应/异常处理过程中插入自定义逻辑。
+"""
+
+# 示例：下载器中间件
+class CustomDownloaderMiddleware:
+    """
+    下载器中间件示例。
+    """
+
+    def process_request(self, request, spider):
+        """
+        在请求被下载器执行前调用。
+        """
+        # request.headers['User-Agent'] = 'Custom UA'
+        # return None  # 继续处理
+        # return request  # 修改并返回
+        # return Response(...)  # 返回一个响应，停止下载
+        pass
+
+    def process_response(self, request, response, spider):
+        """
+        在响应被 Spider 处理前调用。
+        """
+        # return response  # 继续处理
+        # return request  # 重试请求
+        pass
+
+    def process_exception(self, request, exception, spider):
+        """
+        在下载或处理过程中发生异常时调用。
+        """
+        # return None  # 继续抛出异常
+        # return request  # 重试
+        # return Response(...) # 返回一个响应
+        pass
+
+
+# 示例：Spider 中间件
+class CustomSpiderMiddleware:
+    """
+    Spider 中间件示例。
+    """
+
+    def process_spider_input(self, response, spider):
+        """
+        在 Spider 的 parse 方法被调用前调用。
+        """
+        # 可以用来验证响应
+        pass
+
+    def process_spider_output(self, response, result, spider):
+        """
+        在 Spider 的 parse 方法返回结果后调用。
+        """
+        # 可以用来过滤或修改结果
+        # for item in result:
+        #     yield item
+        pass
+
+    def process_spider_exception(self, response, exception, spider):
+        """
+        在 Spider 的 parse 方法抛出异常时调用。
+        """
+        pass
+
+    def process_start_requests(self, start_requests, spider):
+        """
+        在 Spider 的 start_requests 生成器被消费时调用。
+        """
+        # for request in start_requests:
+        #     yield request
+        pass

crawlo/templates/project/pipelines.py.tmpl

@@ -0,0 +1,64 @@
+# -*- coding: UTF-8 -*-
+"""
+{{project_name}}.pipelines
+==========================
+数据管道，用于处理 Spider 返回的 Item。
+例如：清理、验证、去重、保存到数据库等。
+"""
+
+class PrintItemPipeline:
+    """
+    一个简单的管道，用于打印 Item。
+    """
+
+    def process_item(self, item, spider):
+        print(f"Pipeline received item: {dict(item)}")
+        return item
+
+
+class DuplicatesPipeline:
+    """
+    一个去重管道示例。
+    """
+    def __init__(self):
+        self.seen = set()
+
+    def process_item(self, item, spider):
+        identifier = item.get('id') or item.get('url')
+        if identifier in self.seen:
+            spider.logger.debug(f"Duplicate item found: {identifier}")
+            raise DropItem(f"Duplicate item: {identifier}")
+        self.seen.add(identifier)
+        return item
+
+
+# class MySQLPipeline:
+#     """
+#     将 Item 保存到 MySQL 的管道示例。
+#     """
+#     def __init__(self, mysql_uri, mysql_user, mysql_password, mysql_db):
+#         self.mysql_uri = mysql_uri
+#         self.mysql_user = mysql_user
+#         self.mysql_password = mysql_password
+#         self.mysql_db = mysql_db
+#         self.connection = None
+#
+#     @classmethod
+#     def from_settings(cls, settings):
+#         return cls(
+#             mysql_uri=settings.get('MYSQL_HOST'),
+#             mysql_user=settings.get('MYSQL_USER'),
+#             mysql_password=settings.get('MYSQL_PASSWORD'),
+#             mysql_db=settings.get('MYSQL_DB')
+#         )
+#
+#     def open_spider(self, spider):
+#         self.connection = pymysql.connect(...)
+#
+#     def close_spider(self, spider):
+#         if self.connection:
+#             self.connection.close()
+#
+#     def process_item(self, item, spider):
+#         # 执行 SQL 插入
+#         return item

crawlo/templates/project/settings.py.tmpl

@@ -0,0 +1,54 @@
+# -*- coding: UTF-8 -*-
+"""自动创建的 settings.py 文件"""
+
+PROJECT_NAME = '{{project_name}}'
+VERSION = '1.0'
+
+# ============================== 网络请求配置 ==============================
+DOWNLOADER = "crawlo.downloader.aiohttp_downloader.AioHttpDownloader"
+DOWNLOAD_TIMEOUT = 60
+VERIFY_SSL = True
+USE_SESSION = True
+
+DOWNLOAD_DELAY = 1.0
+RANDOMNESS = True
+
+MAX_RETRY_TIMES = 3
+RETRY_HTTP_CODES = [408, 429, 500, 502, 503, 504, 522, 524]
+IGNORE_HTTP_CODES = [403, 404]
+
+CONNECTION_POOL_LIMIT = 100
+
+# ============================== 并发与调度 ==============================
+CONCURRENCY = 8
+MAX_RUNNING_SPIDERS = 3
+
+# ============================== 数据存储 ==============================
+MYSQL_HOST = '127.0.0.1'
+MYSQL_PORT = 3306
+MYSQL_USER = 'root'
+MYSQL_PASSWORD = '123456'
+MYSQL_DB = '{{project_name}}'
+MYSQL_TABLE = 'crawled_data'
+
+# ============================== 去重过滤 ==============================
+FILTER_CLASS = 'crawlo.filters.memory_filter.MemoryFilter'
+
+# ============================== 中间件 & 管道 ==============================
+MIDDLEWARES = [
+    'crawlo.middleware.request_ignore.RequestIgnoreMiddleware',
+    'crawlo.middleware.download_delay.DownloadDelayMiddleware',
+    'crawlo.middleware.default_header.DefaultHeaderMiddleware',
+    'crawlo.middleware.proxy.ProxyMiddleware',
+    'crawlo.middleware.retry.RetryMiddleware',
+    'crawlo.middleware.response_code.ResponseCodeMiddleware',
+    'crawlo.middleware.response_filter.ResponseFilterMiddleware',
+]
+
+PIPELINES = [
+    'crawlo.pipelines.console_pipeline.ConsolePipeline',
+]
+
+# ============================== 日志 ==============================
+LOG_LEVEL = 'INFO'
+LOG_FILE = f'logs/{{{{project_name}}}}.log'

crawlo/templates/project/spiders/__init__.py.tmpl

@@ -0,0 +1,6 @@
+# -*- coding: UTF-8 -*-
+"""
+{{project_name}}.spiders
+========================
+存放所有的爬虫。
+"""

crawlo/templates/spider/spider.py.tmpl

@@ -0,0 +1,32 @@
+# -*- coding: UTF-8 -*-
+"""
+{{project_name}}.spiders.{{spider_name}}
+=======================================
+由 `crawlo genspider` 命令生成的爬虫。
+"""
+
+from crawlo.spider import Spider
+
+
+class {{class_name}}(Spider):
+    """
+    爬虫：{{spider_name}}
+    """
+    name = '{{spider_name}}'
+    allowed_domains = ['{{domain}}']
+    start_urls = ['https://{{domain}}/']
+
+    def parse(self, response):
+        """
+        解析响应的主方法。
+        """
+        # TODO: 在这里编写你的解析逻辑
+
+        # 示例：提取数据
+        # item = {{item_class}}()
+        # item['title'] = response.xpath('//title/text()').get()
+        # yield item
+
+        # 示例：提取链接并跟进
+        # for href in response.xpath('//a/@href').getall():
+        #     yield response.follow(href, callback=self.parse)