crawlo 1.0.9__py3-none-any.whl → 1.1.1__py3-none-any.whl

crawlo/templates/project/__init__.py.tmpl

@@ -1,4 +1,4 @@
-# -*- coding: UTF-8 -*-
-"""
-{{project_name}} 项目包
+# -*- coding: UTF-8 -*-
+"""
+{{project_name}} 项目包
 """

crawlo/templates/project/items.py.tmpl

@@ -1,18 +1,18 @@
-# -*- coding: UTF-8 -*-
-"""
-{{project_name}}.items
-======================
-定义你抓取的数据结构。
-"""
-
-from crawlo.items import Item, Field
-
-
-class ExampleItem(Item):
-    """
-    一个示例数据项。
-    """
-    # name = Field()
-    # price = Field()
-    # description = Field()
+# -*- coding: UTF-8 -*-
+"""
+{{project_name}}.items
+======================
+定义你抓取的数据结构。
+"""
+
+from crawlo.items import Item, Field
+
+
+class ExampleItem(Item):
+    """
+    一个示例数据项。
+    """
+    # name = Field()
+    # price = Field()
+    # description = Field()
     pass

crawlo/templates/project/middlewares.py.tmpl

@@ -1,76 +1,76 @@
-# -*- coding: UTF-8 -*-
-"""
-{{project_name}}.middlewares
-============================
-自定义中间件，用于在请求/响应/异常处理过程中插入自定义逻辑。
-"""
-
-# 示例：下载器中间件
-class CustomDownloaderMiddleware:
-    """
-    下载器中间件示例。
-    """
-
-    def process_request(self, request, spider):
-        """
-        在请求被下载器执行前调用。
-        """
-        # request.headers['User-Agent'] = 'Custom UA'
-        # return None  # 继续处理
-        # return request  # 修改并返回
-        # return Response(...)  # 返回一个响应，停止下载
-        pass
-
-    def process_response(self, request, response, spider):
-        """
-        在响应被 Spider 处理前调用。
-        """
-        # return response  # 继续处理
-        # return request  # 重试请求
-        pass
-
-    def process_exception(self, request, exception, spider):
-        """
-        在下载或处理过程中发生异常时调用。
-        """
-        # return None  # 继续抛出异常
-        # return request  # 重试
-        # return Response(...) # 返回一个响应
-        pass
-
-
-# 示例：Spider 中间件
-class CustomSpiderMiddleware:
-    """
-    Spider 中间件示例。
-    """
-
-    def process_spider_input(self, response, spider):
-        """
-        在 Spider 的 parse 方法被调用前调用。
-        """
-        # 可以用来验证响应
-        pass
-
-    def process_spider_output(self, response, result, spider):
-        """
-        在 Spider 的 parse 方法返回结果后调用。
-        """
-        # 可以用来过滤或修改结果
-        # for item in result:
-        #     yield item
-        pass
-
-    def process_spider_exception(self, response, exception, spider):
-        """
-        在 Spider 的 parse 方法抛出异常时调用。
-        """
-        pass
-
-    def process_start_requests(self, start_requests, spider):
-        """
-        在 Spider 的 start_requests 生成器被消费时调用。
-        """
-        # for request in start_requests:
-        #     yield request
+# -*- coding: UTF-8 -*-
+"""
+{{project_name}}.middlewares
+============================
+自定义中间件，用于在请求/响应/异常处理过程中插入自定义逻辑。
+"""
+
+# 示例：下载器中间件
+class CustomDownloaderMiddleware:
+    """
+    下载器中间件示例。
+    """
+
+    def process_request(self, request, spider):
+        """
+        在请求被下载器执行前调用。
+        """
+        # request.headers['User-Agent'] = 'Custom UA'
+        # return None  # 继续处理
+        # return request  # 修改并返回
+        # return Response(...)  # 返回一个响应，停止下载
+        pass
+
+    def process_response(self, request, response, spider):
+        """
+        在响应被 Spider 处理前调用。
+        """
+        # return response  # 继续处理
+        # return request  # 重试请求
+        pass
+
+    def process_exception(self, request, exception, spider):
+        """
+        在下载或处理过程中发生异常时调用。
+        """
+        # return None  # 继续抛出异常
+        # return request  # 重试
+        # return Response(...) # 返回一个响应
+        pass
+
+
+# 示例：Spider 中间件
+class CustomSpiderMiddleware:
+    """
+    Spider 中间件示例。
+    """
+
+    def process_spider_input(self, response, spider):
+        """
+        在 Spider 的 parse 方法被调用前调用。
+        """
+        # 可以用来验证响应
+        pass
+
+    def process_spider_output(self, response, result, spider):
+        """
+        在 Spider 的 parse 方法返回结果后调用。
+        """
+        # 可以用来过滤或修改结果
+        # for item in result:
+        #     yield item
+        pass
+
+    def process_spider_exception(self, response, exception, spider):
+        """
+        在 Spider 的 parse 方法抛出异常时调用。
+        """
+        pass
+
+    def process_start_requests(self, start_requests, spider):
+        """
+        在 Spider 的 start_requests 生成器被消费时调用。
+        """
+        # for request in start_requests:
+        #     yield request
         pass

crawlo/templates/project/pipelines.py.tmpl

@@ -1,64 +1,64 @@
-# -*- coding: UTF-8 -*-
-"""
-{{project_name}}.pipelines
-==========================
-数据管道，用于处理 Spider 返回的 Item。
-例如：清理、验证、去重、保存到数据库等。
-"""
-
-class PrintItemPipeline:
-    """
-    一个简单的管道，用于打印 Item。
-    """
-
-    def process_item(self, item, spider):
-        print(f"Pipeline received item: {dict(item)}")
-        return item
-
-
-class DuplicatesPipeline:
-    """
-    一个去重管道示例。
-    """
-    def __init__(self):
-        self.seen = set()
-
-    def process_item(self, item, spider):
-        identifier = item.get('id') or item.get('url')
-        if identifier in self.seen:
-            spider.logger.debug(f"Duplicate item found: {identifier}")
-            raise DropItem(f"Duplicate item: {identifier}")
-        self.seen.add(identifier)
-        return item
-
-
-# class MySQLPipeline:
-#     """
-#     将 Item 保存到 MySQL 的管道示例。
-#     """
-#     def __init__(self, mysql_uri, mysql_user, mysql_password, mysql_db):
-#         self.mysql_uri = mysql_uri
-#         self.mysql_user = mysql_user
-#         self.mysql_password = mysql_password
-#         self.mysql_db = mysql_db
-#         self.connection = None
-#
-#     @classmethod
-#     def from_settings(cls, settings):
-#         return cls(
-#             mysql_uri=settings.get('MYSQL_HOST'),
-#             mysql_user=settings.get('MYSQL_USER'),
-#             mysql_password=settings.get('MYSQL_PASSWORD'),
-#             mysql_db=settings.get('MYSQL_DB')
-#         )
-#
-#     def open_spider(self, spider):
-#         self.connection = pymysql.connect(...)
-#
-#     def close_spider(self, spider):
-#         if self.connection:
-#             self.connection.close()
-#
-#     def process_item(self, item, spider):
-#         # 执行 SQL 插入
+# -*- coding: UTF-8 -*-
+"""
+{{project_name}}.pipelines
+==========================
+数据管道，用于处理 Spider 返回的 Item。
+例如：清理、验证、去重、保存到数据库等。
+"""
+
+class PrintItemPipeline:
+    """
+    一个简单的管道，用于打印 Item。
+    """
+
+    def process_item(self, item, spider):
+        print(f"Pipeline received item: {dict(item)}")
+        return item
+
+
+class DuplicatesPipeline:
+    """
+    一个去重管道示例。
+    """
+    def __init__(self):
+        self.seen = set()
+
+    def process_item(self, item, spider):
+        identifier = item.get('id') or item.get('url')
+        if identifier in self.seen:
+            spider.logger.debug(f"Duplicate item found: {identifier}")
+            raise DropItem(f"Duplicate item: {identifier}")
+        self.seen.add(identifier)
+        return item
+
+
+# class MySQLPipeline:
+#     """
+#     将 Item 保存到 MySQL 的管道示例。
+#     """
+#     def __init__(self, mysql_uri, mysql_user, mysql_password, mysql_db):
+#         self.mysql_uri = mysql_uri
+#         self.mysql_user = mysql_user
+#         self.mysql_password = mysql_password
+#         self.mysql_db = mysql_db
+#         self.connection = None
+#
+#     @classmethod
+#     def from_settings(cls, settings):
+#         return cls(
+#             mysql_uri=settings.get('MYSQL_HOST'),
+#             mysql_user=settings.get('MYSQL_USER'),
+#             mysql_password=settings.get('MYSQL_PASSWORD'),
+#             mysql_db=settings.get('MYSQL_DB')
+#         )
+#
+#     def open_spider(self, spider):
+#         self.connection = pymysql.connect(...)
+#
+#     def close_spider(self, spider):
+#         if self.connection:
+#             self.connection.close()
+#
+#     def process_item(self, item, spider):
+#         # 执行 SQL 插入
 #         return item

crawlo/templates/project/settings.py.tmpl

@@ -1,54 +1,54 @@
-# -*- coding: UTF-8 -*-
-"""自动创建的 settings.py 文件"""
-
-PROJECT_NAME = '{{project_name}}'
-VERSION = '1.0'
-
-# ============================== 网络请求配置 ==============================
-DOWNLOADER = "crawlo.downloader.aiohttp_downloader.AioHttpDownloader"
-DOWNLOAD_TIMEOUT = 60
-VERIFY_SSL = True
-USE_SESSION = True
-
-DOWNLOAD_DELAY = 1.0
-RANDOMNESS = True
-
-MAX_RETRY_TIMES = 3
-RETRY_HTTP_CODES = [408, 429, 500, 502, 503, 504, 522, 524]
-IGNORE_HTTP_CODES = [403, 404]
-
-CONNECTION_POOL_LIMIT = 100
-
-# ============================== 并发与调度 ==============================
-CONCURRENCY = 8
-MAX_RUNNING_SPIDERS = 3
-
-# ============================== 数据存储 ==============================
-MYSQL_HOST = '127.0.0.1'
-MYSQL_PORT = 3306
-MYSQL_USER = 'root'
-MYSQL_PASSWORD = '123456'
-MYSQL_DB = '{{project_name}}'
-MYSQL_TABLE = 'crawled_data'
-
-# ============================== 去重过滤 ==============================
-FILTER_CLASS = 'crawlo.filters.memory_filter.MemoryFilter'
-
-# ============================== 中间件 & 管道 ==============================
-MIDDLEWARES = [
-    'crawlo.middleware.request_ignore.RequestIgnoreMiddleware',
-    'crawlo.middleware.download_delay.DownloadDelayMiddleware',
-    'crawlo.middleware.default_header.DefaultHeaderMiddleware',
-    'crawlo.middleware.proxy.ProxyMiddleware',
-    'crawlo.middleware.retry.RetryMiddleware',
-    'crawlo.middleware.response_code.ResponseCodeMiddleware',
-    'crawlo.middleware.response_filter.ResponseFilterMiddleware',
-]
-
-PIPELINES = [
-    'crawlo.pipelines.console_pipeline.ConsolePipeline',
-]
-
-# ============================== 日志 ==============================
-LOG_LEVEL = 'INFO'
-LOG_FILE = f'logs/{{{{project_name}}}}.log'
+# -*- coding: UTF-8 -*-
+"""自动创建的 settings.py 文件"""
+
+PROJECT_NAME = '{{project_name}}'
+VERSION = '1.0'
+
+# ============================== 网络请求配置 ==============================
+DOWNLOADER = "crawlo.downloader.aiohttp_downloader.AioHttpDownloader"
+DOWNLOAD_TIMEOUT = 60
+VERIFY_SSL = True
+USE_SESSION = True
+
+DOWNLOAD_DELAY = 1.0
+RANDOMNESS = True
+
+MAX_RETRY_TIMES = 3
+RETRY_HTTP_CODES = [408, 429, 500, 502, 503, 504, 522, 524]
+IGNORE_HTTP_CODES = [403, 404]
+
+CONNECTION_POOL_LIMIT = 100
+
+# ============================== 并发与调度 ==============================
+CONCURRENCY = 8
+MAX_RUNNING_SPIDERS = 3
+
+# ============================== 数据存储 ==============================
+MYSQL_HOST = '127.0.0.1'
+MYSQL_PORT = 3306
+MYSQL_USER = 'root'
+MYSQL_PASSWORD = '123456'
+MYSQL_DB = '{{project_name}}'
+MYSQL_TABLE = 'crawled_data'
+
+# ============================== 去重过滤 ==============================
+FILTER_CLASS = 'crawlo.filters.memory_filter.MemoryFilter'
+
+# ============================== 中间件 & 管道 ==============================
+MIDDLEWARES = [
+    'crawlo.middleware.request_ignore.RequestIgnoreMiddleware',
+    'crawlo.middleware.download_delay.DownloadDelayMiddleware',
+    'crawlo.middleware.default_header.DefaultHeaderMiddleware',
+    'crawlo.middleware.proxy.ProxyMiddleware',
+    'crawlo.middleware.retry.RetryMiddleware',
+    'crawlo.middleware.response_code.ResponseCodeMiddleware',
+    'crawlo.middleware.response_filter.ResponseFilterMiddleware',
+]
+
+PIPELINES = [
+    'crawlo.pipelines.console_pipeline.ConsolePipeline',
+]
+
+# ============================== 日志 ==============================
+LOG_LEVEL = 'INFO'
+LOG_FILE = f'logs/{{{project_name}}}.log'

crawlo/templates/project/spiders/__init__.py.tmpl

@@ -1,6 +1,6 @@
-# -*- coding: UTF-8 -*-
-"""
-{{project_name}}.spiders
-========================
-存放所有的爬虫。
+# -*- coding: UTF-8 -*-
+"""
+{{project_name}}.spiders
+========================
+存放所有的爬虫。
 """

crawlo/templates/spider/spider.py.tmpl

@@ -1,32 +1,32 @@
-# -*- coding: UTF-8 -*-
-"""
-{{project_name}}.spiders.{{spider_name}}
-=======================================
-由 `crawlo genspider` 命令生成的爬虫。
-"""
-
-from crawlo.spider import Spider
-
-
-class {{class_name}}(Spider):
-    """
-    爬虫：{{spider_name}}
-    """
-    name = '{{spider_name}}'
-    allowed_domains = ['{{domain}}']
-    start_urls = ['https://{{domain}}/']
-
-    def parse(self, response):
-        """
-        解析响应的主方法。
-        """
-        # TODO: 在这里编写你的解析逻辑
-
-        # 示例：提取数据
-        # item = {{item_class}}()
-        # item['title'] = response.xpath('//title/text()').get()
-        # yield item
-
-        # 示例：提取链接并跟进
-        # for href in response.xpath('//a/@href').getall():
+# -*- coding: UTF-8 -*-
+"""
+{{project_name}}.spiders.{{spider_name}}
+=======================================
+由 `crawlo genspider` 命令生成的爬虫。
+"""
+
+from crawlo.spider import Spider
+
+
+class {{class_name}}(Spider):
+    """
+    爬虫：{{spider_name}}
+    """
+    name = '{{spider_name}}'
+    allowed_domains = ['{{domain}}']
+    start_urls = ['https://{{domain}}/']
+
+    def parse(self, response):
+        """
+        解析响应的主方法。
+        """
+        # TODO: 在这里编写你的解析逻辑
+
+        # 示例：提取数据
+        # item = {{item_class}}()
+        # item['title'] = response.xpath('//title/text()').get()
+        # yield item
+
+        # 示例：提取链接并跟进
+        # for href in response.xpath('//a/@href').getall():
         #     yield response.follow(href, callback=self.parse)

crawlo/utils/__init__.py

@@ -1,7 +1,7 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-"""
-# @Time    :    2025-02-05 13:57
-# @Author  :   oscar
-# @Desc    :   None
-"""
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+# @Time    :    2025-02-05 13:57
+# @Author  :   oscar
+# @Desc    :   None
+"""