crawlo 1.0.1__tar.gz → 1.0.2__tar.gz

crawlo-1.0.2/MANIFEST.in

@@ -0,0 +1,17 @@
+include README.md
+include LICENSE
+include requirements.txt  # 如果根目录有全局requirements.txt
+include VERSION           # 如果根目录有全局VERSION文件
+
+# 包内文件包含
+recursive-include crawlo/utils/js *
+recursive-include crawlo/templates *
+
+# 测试文件（如果需要在分发包中包含测试）
+recursive-include tests *
+
+# 排除项
+global-exclude __pycache__ *.py[cod] .DS_Store *.so
+global-exclude *.bak *.swp *.orig *.rej
+prune samples              # 排除示例目录
+prune docs                 # 排除文档目录

{crawlo-1.0.1/crawlo.egg-info → crawlo-1.0.2}/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: crawlo
-Version: 1.0.1
-Summary: Crawlo是一款支持异步的python爬虫框架
+Version: 1.0.2
+Summary: Crawlo 是一款基于异步IO的高性能Python爬虫框架，支持分布式抓取。
 Home-page: https://github.com/crawl-coder/Crawlo.git
 Author: crawl-coder
 Author-email: crawlo@qq.com
@@ -11,13 +11,26 @@ Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.6
 Description-Content-Type: text/markdown
-License-File: LICENSE
-Requires-Dist: aiohttp>=3.12.6
-Requires-Dist: httpx>=0.28.1
-Requires-Dist: DBUtils>=2.0
-Requires-Dist: parsel>=1.10.0
-Requires-Dist: pymysql>=1.1.1
-Requires-Dist: ujson>=5.10.0
+Requires-Dist: aiohttp>=3.12.14
+Requires-Dist: aiomysql>=0.2.0
+Requires-Dist: aioredis>=2.0.1
+Requires-Dist: asyncmy>=0.2.10
+Requires-Dist: cssselect>=1.2.0
+Requires-Dist: dateparser>=1.2.2
+Requires-Dist: httpx>=0.27.0
+Requires-Dist: lxml>=5.2.1
+Requires-Dist: motor>=3.7.0
+Requires-Dist: parsel>=1.9.1
+Requires-Dist: pydantic>=2.11.7
+Requires-Dist: pymongo>=4.11
+Requires-Dist: PyMySQL>=1.1.1
+Requires-Dist: python-dateutil>=2.9.0.post0
+Requires-Dist: redis>=6.2.0
+Requires-Dist: requests>=2.32.4
+Requires-Dist: six>=1.17.0
+Requires-Dist: ujson>=5.9.0
+Requires-Dist: urllib3>=2.5.0
+Requires-Dist: w3lib>=2.1.2
 Provides-Extra: render
 Requires-Dist: webdriver-manager>=4.0.0; extra == "render"
 Requires-Dist: playwright; extra == "render"
@@ -30,7 +43,6 @@ Requires-Dist: redis-py-cluster>=2.1.0; extra == "all"
 Requires-Dist: webdriver-manager>=4.0.0; extra == "all"
 Requires-Dist: playwright; extra == "all"
 Requires-Dist: selenium>=3.141.0; extra == "all"
-Dynamic: license-file
 
 # Crawlo
-异步通用爬虫框架
+Crawlo 是一款基于异步IO的高性能Python爬虫框架，支持分布式抓取与数据管道。

crawlo-1.0.2/README.md

@@ -0,0 +1,2 @@
+# Crawlo
+Crawlo 是一款基于异步IO的高性能Python爬虫框架，支持分布式抓取与数据管道。

crawlo-1.0.2/crawlo/__version__.py

@@ -0,0 +1,2 @@
+
+__version__ = "1.0.2"

crawlo-1.0.2/crawlo/crawler.py

@@ -0,0 +1,222 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*
+import signal
+import asyncio
+from typing import Type, Final, Set, Optional
+
+from crawlo.spider import Spider
+from crawlo.core.engine import Engine
+from crawlo.utils.log import get_logger
+from crawlo.subscriber import Subscriber
+from crawlo.extension import ExtensionManager
+from crawlo.exceptions import SpiderTypeError
+from crawlo.stats_collector import StatsCollector
+from crawlo.event import spider_opened, spider_closed
+from crawlo.settings.setting_manager import SettingManager
+from crawlo.utils.project import merge_settings, get_settings
+from crawlo.utils.concurrency_manager import calculate_optimal_concurrency
+
+logger = get_logger(__name__)
+
+
+class Crawler:
+
+    def __init__(self, spider_cls, settings):
+        self.spider_cls = spider_cls
+        self.spider: Optional[Spider] = None
+        self.engine: Optional[Engine] = None
+        self.stats: Optional[StatsCollector] = None
+        self.subscriber: Optional[Subscriber] = None
+        self.extension: Optional[ExtensionManager] = None
+        self.settings: SettingManager = settings.copy()
+
+    async def crawl(self):
+        self.subscriber = self._create_subscriber()
+        self.spider = self._create_spider()
+        self.engine = self._create_engine()
+        self.stats = self._create_stats()
+        self.extension = self._create_extension()
+
+        await self.engine.start_spider(self.spider)
+
+    @staticmethod
+    def _create_subscriber():
+        return Subscriber()
+
+    def _create_spider(self) -> Spider:
+        spider = self.spider_cls.create_instance(self)
+        self._set_spider(spider)
+        return spider
+
+    def _create_engine(self) -> Engine:
+        engine = Engine(self)
+        engine.engine_start()
+        return engine
+
+    def _create_stats(self) -> StatsCollector:
+        stats = StatsCollector(self)
+        return stats
+
+    def _create_extension(self) -> ExtensionManager:
+        extension = ExtensionManager.create_instance(self)
+        return extension
+
+    def _set_spider(self, spider):
+        self.subscriber.subscribe(spider.spider_opened, event=spider_opened)
+        self.subscriber.subscribe(spider.spider_closed, event=spider_closed)
+        merge_settings(spider, self.settings)
+
+    async def close(self, reason='finished') -> None:
+        await asyncio.create_task(self.subscriber.notify(spider_closed))
+        self.stats.close_spider(spider_name=self.spider, reason=reason)
+
+
+class CrawlerProcess:
+    """爬虫处理类，支持跨平台动态并发控制和精细化日志"""
+
+    def __init__(self, settings=None, max_concurrency: Optional[int] = None, batch_size: int = 50):
+        self.crawlers: Final[Set] = set()
+        self._active_spiders: Final[Set] = set()
+        self.settings = settings or self._get_default_settings()
+        self.batch_size = batch_size
+
+        # 使用独立模块计算最大并发数
+        self.max_concurrency = calculate_optimal_concurrency(max_concurrency)
+        self.semaphore = asyncio.Semaphore(self.max_concurrency)
+
+        signal.signal(signal.SIGINT, self._shutdown)
+        logger.info(f"初始化爬虫处理进程，最大并发数: {self.max_concurrency}")
+
+    async def crawl(self, spiders):
+        """支持单个或多个爬虫的批量处理，优化日志输出"""
+        if not spiders:
+            raise ValueError("至少需要提供一个爬虫类")
+
+        # 统一转换为列表
+        if isinstance(spiders, type) and issubclass(spiders, Spider):
+            spiders = [spiders]
+        elif isinstance(spiders, (list, tuple)):
+            spiders = list(spiders)
+        else:
+            raise TypeError("spiders 必须是爬虫类或爬虫类列表/元组")
+
+        # 按爬虫类名首字母排序（升序）
+        spiders.sort(key=lambda x: x.__name__.lower())
+
+        if len(spiders) == 1:
+            logger.info(f"启动爬虫: {spiders[0].__name__}")
+        else:
+            logger.info(f"启动{len(spiders)}个爬虫，按名称排序后分批处理中")
+
+        batches = [spiders[i:i + self.batch_size] for i in range(0, len(spiders), self.batch_size)]
+
+        for batch_idx, batch in enumerate(batches):
+            batch_tasks = set()
+
+            for spider_cls in batch:
+                crawler = self._create_crawler(spider_cls)
+                self.crawlers.add(crawler)
+
+                await self.semaphore.acquire()
+                task = asyncio.create_task(self._run_crawler_with_semaphore(crawler))
+                batch_tasks.add(task)
+                self._active_spiders.add(task)
+
+            if len(spiders) > 1:  # 仅对多爬虫显示批次信息
+                logger.info(f"启动第 {batch_idx + 1}/{len(batches)} 批爬虫，共 {len(batch)} 个")
+
+            await asyncio.gather(*batch_tasks)
+
+            if len(spiders) > 1:  # 仅对多爬虫显示批次完成信息
+                logger.info(f"第 {batch_idx + 1} 批爬虫处理完成")
+
+    async def _run_crawler_with_semaphore(self, crawler):
+        """使用信号量控制的爬虫运行函数"""
+        try:
+            await crawler.crawl()
+        finally:
+            self.semaphore.release()  # 确保资源释放
+
+    async def start(self):
+        """启动所有爬虫任务"""
+        if self._active_spiders:
+            logger.info(f"启动 {len(self._active_spiders)} 个爬虫任务，并发限制: {self.max_concurrency}")
+            await asyncio.gather(*self._active_spiders)
+
+    def _create_crawler(self, spider_cls) -> Crawler:
+        """创建爬虫实例"""
+        if isinstance(spider_cls, str):
+            raise SpiderTypeError(f"{type(self)}.crawl args: String is not supported.")
+        crawler: Crawler = Crawler(spider_cls, self.settings)
+        return crawler
+
+    def _shutdown(self, _signum, _frame):
+        """优雅关闭所有爬虫"""
+        logger.warning(f"收到关闭信号，正在优雅关闭 {len(self.crawlers)} 个爬虫...")
+        for crawler in self.crawlers:
+            if crawler.engine:
+                crawler.engine.running = False
+                crawler.engine.normal = False
+                crawler.stats.close_spider(crawler.spider, 'shutdown signal')
+
+        # 等待所有任务完成
+        asyncio.create_task(self._wait_for_tasks())
+
+    async def _wait_for_tasks(self):
+        """等待所有活跃任务完成"""
+        pending = [task for task in self._active_spiders if not task.done()]
+        if pending:
+            logger.info(f"等待 {len(pending)} 个活跃任务完成...")
+            await asyncio.gather(*pending)
+        logger.info("所有爬虫已优雅关闭")
+
+    @classmethod
+    def _get_default_settings(cls):
+        """框架自动获取默认配置"""
+        try:
+            return get_settings()
+        except ImportError:
+            return {}
+
+# class CrawlerProcess:
+#
+#     def __init__(self, settings=None):
+#         self.crawlers: Final[Set] = set()
+#         self._active_spiders: Final[Set] = set()
+#         self.settings = settings or self._get_default_settings()
+#
+#         signal.signal(signal.SIGINT, self._shutdown)
+#
+#     async def crawl(self, spider: Type[Spider]):
+#         crawler: Crawler = self._create_crawler(spider)
+#         self.crawlers.add(crawler)
+#         task = await self._crawl(crawler)
+#         self._active_spiders.add(task)
+#
+#     @classmethod
+#     def _get_default_settings(cls):
+#         """自动获取默认配置"""
+#         try:
+#             return get_settings()
+#         except ImportError:
+#             return {}
+#
+#     @staticmethod
+#     async def _crawl(crawler):
+#         return asyncio.create_task(crawler.crawl())
+#
+#     async def start(self):
+#         await asyncio.gather(*self._active_spiders)
+#
+#     def _create_crawler(self, spider_cls) -> Crawler:
+#         if isinstance(spider_cls, str):
+#             raise SpiderTypeError(f"{type(self)}.crawl args: String is not supported.")
+#         crawler: Crawler = Crawler(spider_cls, self.settings)
+#         return crawler
+#
+#     def _shutdown(self, _signum, _frame):
+#         for crawler in self.crawlers:
+#             crawler.engine.running = False
+#             crawler.engine.normal = False
+#             crawler.stats.close_spider(crawler.spider, 'Ctrl C')
+#         logger.warning(f'spiders received: `Ctrl C` signal, closed.')

{crawlo-1.0.1 → crawlo-1.0.2}/crawlo/middleware/retry.py

@@ -80,7 +80,8 @@ class RetryMiddleware(object):
             retry_times += 1
             self.logger.info(f"{spider} {request} {reason} retrying {retry_times} time...")
             request.meta['retry_times'] = retry_times
-            request.dont_retry = True
+            # request.dont_retry = True
+            request.meta['dont_retry'] = True
             request.retry_priority = request.priority + self.retry_priority
             self.stats.inc_value("retry_count")
             return request

crawlo-1.0.2/crawlo/network/request.py

@@ -0,0 +1,234 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+import json
+from copy import deepcopy
+from urllib.parse import urlencode
+from w3lib.url import safe_url_string
+from typing import Dict, Optional, Callable, Union, Any, TypeVar, List
+
+from crawlo.utils.url import escape_ajax
+
+_Request = TypeVar("_Request", bound="Request")
+
+
+class RequestPriority:
+    HIGH = -100
+    NORMAL = 0
+    LOW = 100
+
+
+class Request:
+    """
+    封装一个 HTTP 请求对象，用于爬虫框架中表示一个待抓取的请求任务。
+    支持设置回调函数、请求头、请求体、优先级、元数据等。
+    """
+
+    __slots__ = (
+        '_url',
+        '_meta',
+        'callback',
+        'cb_kwargs',
+        'err_back',
+        'headers',
+        'body',
+        'method',
+        'cookies',
+        'priority',
+        'encoding',
+        'dont_filter',
+        'timeout',
+        'proxy',
+        'allow_redirects',
+        'auth',
+        'verify',
+        'flags'
+    )
+
+    def __init__(
+        self,
+        url: str,
+        callback: Optional[Callable] = None,
+        method: Optional[str] = 'GET',
+        headers: Optional[Dict[str, str]] = None,
+        body: Optional[Union[Dict, bytes, str]] = None,
+        form_data: Optional[Dict] = None,
+        json_body: Optional[Dict] = None,   # ✅ 参数名从 json 改为 json_body
+        cb_kwargs: Optional[Dict[str, Any]] = None,
+        err_back: Optional[Callable] = None,
+        cookies: Optional[Dict[str, str]] = None,
+        meta: Optional[Dict[str, Any]] = None,
+        priority: int = RequestPriority.NORMAL,
+        dont_filter: bool = False,
+        timeout: Optional[float] = None,
+        proxy: Optional[str] = None,
+        allow_redirects: bool = True,
+        auth: Optional[tuple] = None,
+        verify: bool = True,
+        flags: Optional[List[str]] = None,
+        encoding: str = 'utf-8'
+    ):
+        """
+        初始化请求对象。
+
+        参数说明:
+        :param url: 请求的 URL 地址（必须）
+        :param callback: 响应处理回调函数（可选）
+        :param method: HTTP 请求方法，默认为 GET
+        :param headers: 请求头（可选）
+        :param body: 请求体（可为 dict、bytes 或 str）
+        :param form_data 表单数据，自动设置为 POST 并构造 x-www-form-urlencoded 请求体
+        :param json_body: 用于构造 JSON 请求体，自动设置 Content-Type 为 application/json
+        :param cb_kwargs: 传递给回调函数的额外参数（可选）
+        :param err_back: 请求失败时的错误回调函数（可选）
+        :param cookies: 请求 cookies（可选）
+        :param meta: 元数据字典，用于在请求间传递数据
+        :param priority: 请求优先级，数值越小优先级越高（默认为 0）
+        :param dont_filter: 是否跳过去重过滤（默认为 False）
+        :param timeout: 请求超时时间（秒）
+        :param proxy: 代理地址（如：http://127.0.0.1:8080）
+        :param allow_redirects: 是否允许重定向（默认为 True）
+        :param auth: 认证信息，格式为 (username, password)
+        :param verify: 是否验证 SSL 证书（默认为 True）
+        :param flags: 请求标记（调试、重试等用途）
+        """
+        self.callback = callback
+        self.method = str(method).upper()
+        self.headers = headers or {}
+        self.body = body
+        self.cb_kwargs = cb_kwargs or {}
+        self.err_back = err_back
+        self.cookies = cookies or {}
+        self.priority = -priority  # 高优先级值更小，便于排序
+        self._meta = deepcopy(meta) if meta is not None else {}
+        self.timeout = self._meta.get('download_timeout', timeout)
+        self.proxy = proxy
+        self.allow_redirects = allow_redirects
+        self.auth = auth
+        self.verify = verify
+        self.flags = flags or []
+
+        # 默认编码
+        self.encoding = encoding
+
+        # 优先使用 json_body 参数
+        if json_body is not None:
+            if 'Content-Type' not in self.headers:
+                self.headers['Content-Type'] = 'application/json'
+            self.body = json.dumps(json_body, ensure_ascii=False).encode(self.encoding)
+            if self.method == 'GET':
+                self.method = 'POST'
+
+        # 其次使用 form_data
+        elif form_data is not None:
+            if self.method == 'GET':
+                self.method = 'POST'
+            if 'Content-Type' not in self.headers:
+                self.headers['Content-Type'] = 'application/x-www-form-urlencoded'
+            self.body = urlencode(form_data)
+
+        # 最后处理 body 为 dict 的情况
+        elif isinstance(self.body, dict):
+            if 'Content-Type' not in self.headers:
+                self.headers['Content-Type'] = 'application/json'
+            self.body = json.dumps(self.body, ensure_ascii=False).encode(self.encoding)
+
+        self.dont_filter = dont_filter
+        self._set_url(url)
+
+    def copy(self: _Request) -> _Request:
+        """
+        创建当前 Request 的副本，用于避免引用共享数据。
+
+        :return: 一个新的 Request 实例
+        """
+        return type(self)(
+            url=self.url,
+            callback=self.callback,
+            method=self.method,
+            headers=self.headers.copy(),
+            body=self.body,
+            form_data=None,  # form_data 不参与复制
+            json_body=None,   # json_body 参数也不参与复制
+            cb_kwargs=deepcopy(self.cb_kwargs),
+            err_back=self.err_back,
+            cookies=self.cookies.copy(),
+            meta=deepcopy(self._meta),
+            priority=-self.priority,
+            dont_filter=self.dont_filter,
+            timeout=self.timeout,
+            proxy=self.proxy,
+            allow_redirects=self.allow_redirects,
+            auth=self.auth,
+            verify=self.verify,
+            flags=self.flags.copy(),
+        )
+
+    def set_meta(self, key: str, value: Any) -> None:
+        """
+        设置 meta 中的某个键值对。
+
+        :param key: 要设置的键
+        :param value: 对应的值
+        """
+        self._meta[key] = value
+
+    def _set_url(self, url: str) -> None:
+        """
+        设置并验证 URL，确保其格式正确且包含 scheme。
+
+        :param url: 原始 URL 字符串
+        :raises TypeError: 如果传入的不是字符串
+        :raises ValueError: 如果 URL 没有 scheme
+        """
+        if not isinstance(url, str):
+            raise TypeError(f"Request url 必须为字符串类型，当前类型为 {type(url).__name__}")
+
+        s = safe_url_string(url, self.encoding)
+        escaped_url = escape_ajax(s)
+        self._url = escaped_url
+
+        if not self._url.startswith(('http://', 'https://', 'about:', '')):
+            raise ValueError(f"请求 URL 缺少 scheme（如 http://）: {self._url}")
+
+    @property
+    def url(self) -> str:
+        """
+        获取请求的 URL。
+
+        :return: 当前请求的 URL 字符串
+        """
+        return self._url
+
+    @property
+    def meta(self) -> Dict[str, Any]:
+        """
+        获取请求的元数据。
+
+        :return: 元数据字典
+        """
+        return self._meta
+
+    def __str__(self) -> str:
+        """
+        返回对象的字符串表示，用于调试和日志输出。
+
+        :return: 字符串 <Request url=... method=...>
+        """
+        return f'<Request url={self.url} method={self.method}>'
+
+    def __repr__(self) -> str:
+        """
+        返回对象的官方字符串表示。
+
+        :return: 字符串，与 __str__ 相同
+        """
+        return str(self)
+
+    def __lt__(self, other: _Request) -> bool:
+        """
+        比较两个请求的优先级，用于排序。
+
+        :param other: 另一个 Request 对象
+        :return: 如果当前请求优先级更高（数值更小）返回 True
+        """
+        return self.priority < other.priority