crawlo 1.0.1__py3-none-any.whl → 1.0.2__py3-none-any.whl

crawlo/__version__.py

@@ -1,2 +1,2 @@
 
-__version__ = "1.0.1"
+__version__ = "1.0.2"

crawlo/crawler.py

@@ -6,15 +6,15 @@ from typing import Type, Final, Set, Optional
 
 from crawlo.spider import Spider
 from crawlo.core.engine import Engine
-from crawlo.subscriber import Subscriber
-
 from crawlo.utils.log import get_logger
+from crawlo.subscriber import Subscriber
 from crawlo.extension import ExtensionManager
 from crawlo.exceptions import SpiderTypeError
-from crawlo.utils.project import merge_settings
 from crawlo.stats_collector import StatsCollector
 from crawlo.event import spider_opened, spider_closed
 from crawlo.settings.setting_manager import SettingManager
+from crawlo.utils.project import merge_settings, get_settings
+from crawlo.utils.concurrency_manager import calculate_optimal_concurrency
 
 logger = get_logger(__name__)
 
@@ -72,36 +72,151 @@ class Crawler:
 
 
 class CrawlerProcess:
+    """爬虫处理类，支持跨平台动态并发控制和精细化日志"""
 
-    def __init__(self, settings=None):
+    def __init__(self, settings=None, max_concurrency: Optional[int] = None, batch_size: int = 50):
         self.crawlers: Final[Set] = set()
         self._active_spiders: Final[Set] = set()
-        self.settings = settings
+        self.settings = settings or self._get_default_settings()
+        self.batch_size = batch_size
+
+        # 使用独立模块计算最大并发数
+        self.max_concurrency = calculate_optimal_concurrency(max_concurrency)
+        self.semaphore = asyncio.Semaphore(self.max_concurrency)
 
         signal.signal(signal.SIGINT, self._shutdown)
+        logger.info(f"初始化爬虫处理进程，最大并发数: {self.max_concurrency}")
 
-    async def crawl(self, spider: Type[Spider]):
-        crawler: Crawler = self._create_crawler(spider)
-        self.crawlers.add(crawler)
-        task = await self._crawl(crawler)
-        self._active_spiders.add(task)
+    async def crawl(self, spiders):
+        """支持单个或多个爬虫的批量处理，优化日志输出"""
+        if not spiders:
+            raise ValueError("至少需要提供一个爬虫类")
 
-    @staticmethod
-    async def _crawl(crawler):
-        return asyncio.create_task(crawler.crawl())
+        # 统一转换为列表
+        if isinstance(spiders, type) and issubclass(spiders, Spider):
+            spiders = [spiders]
+        elif isinstance(spiders, (list, tuple)):
+            spiders = list(spiders)
+        else:
+            raise TypeError("spiders 必须是爬虫类或爬虫类列表/元组")
+
+        # 按爬虫类名首字母排序（升序）
+        spiders.sort(key=lambda x: x.__name__.lower())
+
+        if len(spiders) == 1:
+            logger.info(f"启动爬虫: {spiders[0].__name__}")
+        else:
+            logger.info(f"启动{len(spiders)}个爬虫，按名称排序后分批处理中")
+
+        batches = [spiders[i:i + self.batch_size] for i in range(0, len(spiders), self.batch_size)]
+
+        for batch_idx, batch in enumerate(batches):
+            batch_tasks = set()
+
+            for spider_cls in batch:
+                crawler = self._create_crawler(spider_cls)
+                self.crawlers.add(crawler)
+
+                await self.semaphore.acquire()
+                task = asyncio.create_task(self._run_crawler_with_semaphore(crawler))
+                batch_tasks.add(task)
+                self._active_spiders.add(task)
+
+            if len(spiders) > 1:  # 仅对多爬虫显示批次信息
+                logger.info(f"启动第 {batch_idx + 1}/{len(batches)} 批爬虫，共 {len(batch)} 个")
+
+            await asyncio.gather(*batch_tasks)
+
+            if len(spiders) > 1:  # 仅对多爬虫显示批次完成信息
+                logger.info(f"第 {batch_idx + 1} 批爬虫处理完成")
+
+    async def _run_crawler_with_semaphore(self, crawler):
+        """使用信号量控制的爬虫运行函数"""
+        try:
+            await crawler.crawl()
+        finally:
+            self.semaphore.release()  # 确保资源释放
 
     async def start(self):
-        await asyncio.gather(*self._active_spiders)
+        """启动所有爬虫任务"""
+        if self._active_spiders:
+            logger.info(f"启动 {len(self._active_spiders)} 个爬虫任务，并发限制: {self.max_concurrency}")
+            await asyncio.gather(*self._active_spiders)
 
     def _create_crawler(self, spider_cls) -> Crawler:
+        """创建爬虫实例"""
         if isinstance(spider_cls, str):
             raise SpiderTypeError(f"{type(self)}.crawl args: String is not supported.")
         crawler: Crawler = Crawler(spider_cls, self.settings)
         return crawler
 
     def _shutdown(self, _signum, _frame):
+        """优雅关闭所有爬虫"""
+        logger.warning(f"收到关闭信号，正在优雅关闭 {len(self.crawlers)} 个爬虫...")
         for crawler in self.crawlers:
-            crawler.engine.running = False
-            crawler.engine.normal = False
-            crawler.stats.close_spider(crawler.spider, 'Ctrl C')
-        logger.warning(f'spiders received: `Ctrl C` signal, closed.')
+            if crawler.engine:
+                crawler.engine.running = False
+                crawler.engine.normal = False
+                crawler.stats.close_spider(crawler.spider, 'shutdown signal')
+
+        # 等待所有任务完成
+        asyncio.create_task(self._wait_for_tasks())
+
+    async def _wait_for_tasks(self):
+        """等待所有活跃任务完成"""
+        pending = [task for task in self._active_spiders if not task.done()]
+        if pending:
+            logger.info(f"等待 {len(pending)} 个活跃任务完成...")
+            await asyncio.gather(*pending)
+        logger.info("所有爬虫已优雅关闭")
+
+    @classmethod
+    def _get_default_settings(cls):
+        """框架自动获取默认配置"""
+        try:
+            return get_settings()
+        except ImportError:
+            return {}
+
+# class CrawlerProcess:
+#
+#     def __init__(self, settings=None):
+#         self.crawlers: Final[Set] = set()
+#         self._active_spiders: Final[Set] = set()
+#         self.settings = settings or self._get_default_settings()
+#
+#         signal.signal(signal.SIGINT, self._shutdown)
+#
+#     async def crawl(self, spider: Type[Spider]):
+#         crawler: Crawler = self._create_crawler(spider)
+#         self.crawlers.add(crawler)
+#         task = await self._crawl(crawler)
+#         self._active_spiders.add(task)
+#
+#     @classmethod
+#     def _get_default_settings(cls):
+#         """自动获取默认配置"""
+#         try:
+#             return get_settings()
+#         except ImportError:
+#             return {}
+#
+#     @staticmethod
+#     async def _crawl(crawler):
+#         return asyncio.create_task(crawler.crawl())
+#
+#     async def start(self):
+#         await asyncio.gather(*self._active_spiders)
+#
+#     def _create_crawler(self, spider_cls) -> Crawler:
+#         if isinstance(spider_cls, str):
+#             raise SpiderTypeError(f"{type(self)}.crawl args: String is not supported.")
+#         crawler: Crawler = Crawler(spider_cls, self.settings)
+#         return crawler
+#
+#     def _shutdown(self, _signum, _frame):
+#         for crawler in self.crawlers:
+#             crawler.engine.running = False
+#             crawler.engine.normal = False
+#             crawler.stats.close_spider(crawler.spider, 'Ctrl C')
+#         logger.warning(f'spiders received: `Ctrl C` signal, closed.')

crawlo/middleware/retry.py

@@ -80,7 +80,8 @@ class RetryMiddleware(object):
             retry_times += 1
             self.logger.info(f"{spider} {request} {reason} retrying {retry_times} time...")
             request.meta['retry_times'] = retry_times
-            request.dont_retry = True
+            # request.dont_retry = True
+            request.meta['dont_retry'] = True
             request.retry_priority = request.priority + self.retry_priority
             self.stats.inc_value("retry_count")
             return request

crawlo/network/request.py

@@ -1,155 +1,234 @@
 #!/usr/bin/python
-# -*- coding:UTF-8 -*-
-import hashlib
+# -*- coding: UTF-8 -*-
+import json
 from copy import deepcopy
+from urllib.parse import urlencode
 from w3lib.url import safe_url_string
-from typing import Dict, Optional, Callable, Union, Any
+from typing import Dict, Optional, Callable, Union, Any, TypeVar, List
 
 from crawlo.utils.url import escape_ajax
 
-
-class Request(object):
+_Request = TypeVar("_Request", bound="Request")
+
+
+class RequestPriority:
+    HIGH = -100
+    NORMAL = 0
+    LOW = 100
+
+
+class Request:
+    """
+    封装一个 HTTP 请求对象，用于爬虫框架中表示一个待抓取的请求任务。
+    支持设置回调函数、请求头、请求体、优先级、元数据等。
+    """
+
+    __slots__ = (
+        '_url',
+        '_meta',
+        'callback',
+        'cb_kwargs',
+        'err_back',
+        'headers',
+        'body',
+        'method',
+        'cookies',
+        'priority',
+        'encoding',
+        'dont_filter',
+        'timeout',
+        'proxy',
+        'allow_redirects',
+        'auth',
+        'verify',
+        'flags'
+    )
 
     def __init__(
-            self,
-            url: str,
-            *,
-            callback: Optional[Callable] = None,
-            headers: Optional[Dict[str, str]] = None,
-            body: Optional[Union[Dict, bytes, str]] = None,
-            method: Optional[str] = 'GET',
-            cookies: Optional[Dict[str, str]] = None,
-            priority: int = 0,
-            encoding: Optional[str] = 'utf-8',
-            meta: Optional[Dict[str, Any]] = None,
-            dont_filter: bool = False,
-            timeout: Optional[float] = None,
-            proxy: Optional[str] = None,
-            allow_redirects: bool = True,
-            auth: Optional[tuple] = None,
-            verify: bool = True
+        self,
+        url: str,
+        callback: Optional[Callable] = None,
+        method: Optional[str] = 'GET',
+        headers: Optional[Dict[str, str]] = None,
+        body: Optional[Union[Dict, bytes, str]] = None,
+        form_data: Optional[Dict] = None,
+        json_body: Optional[Dict] = None,   # ✅ 参数名从 json 改为 json_body
+        cb_kwargs: Optional[Dict[str, Any]] = None,
+        err_back: Optional[Callable] = None,
+        cookies: Optional[Dict[str, str]] = None,
+        meta: Optional[Dict[str, Any]] = None,
+        priority: int = RequestPriority.NORMAL,
+        dont_filter: bool = False,
+        timeout: Optional[float] = None,
+        proxy: Optional[str] = None,
+        allow_redirects: bool = True,
+        auth: Optional[tuple] = None,
+        verify: bool = True,
+        flags: Optional[List[str]] = None,
+        encoding: str = 'utf-8'
     ):
-        # 先初始化基本属性
+        """
+        初始化请求对象。
+
+        参数说明:
+        :param url: 请求的 URL 地址（必须）
+        :param callback: 响应处理回调函数（可选）
+        :param method: HTTP 请求方法，默认为 GET
+        :param headers: 请求头（可选）
+        :param body: 请求体（可为 dict、bytes 或 str）
+        :param form_data 表单数据，自动设置为 POST 并构造 x-www-form-urlencoded 请求体
+        :param json_body: 用于构造 JSON 请求体，自动设置 Content-Type 为 application/json
+        :param cb_kwargs: 传递给回调函数的额外参数（可选）
+        :param err_back: 请求失败时的错误回调函数（可选）
+        :param cookies: 请求 cookies（可选）
+        :param meta: 元数据字典，用于在请求间传递数据
+        :param priority: 请求优先级，数值越小优先级越高（默认为 0）
+        :param dont_filter: 是否跳过去重过滤（默认为 False）
+        :param timeout: 请求超时时间（秒）
+        :param proxy: 代理地址（如：http://127.0.0.1:8080）
+        :param allow_redirects: 是否允许重定向（默认为 True）
+        :param auth: 认证信息，格式为 (username, password)
+        :param verify: 是否验证 SSL 证书（默认为 True）
+        :param flags: 请求标记（调试、重试等用途）
+        """
         self.callback = callback
-        self.headers = headers if headers else {}
-        self.body = body
         self.method = str(method).upper()
-        self.cookies = cookies if cookies else {}
-        self.priority = -priority
-        self.encoding = encoding
-        self.dont_filter = dont_filter
-        self._meta = meta if meta is not None else {}
-        self.timeout = timeout
+        self.headers = headers or {}
+        self.body = body
+        self.cb_kwargs = cb_kwargs or {}
+        self.err_back = err_back
+        self.cookies = cookies or {}
+        self.priority = -priority  # 高优先级值更小，便于排序
+        self._meta = deepcopy(meta) if meta is not None else {}
+        self.timeout = self._meta.get('download_timeout', timeout)
         self.proxy = proxy
         self.allow_redirects = allow_redirects
         self.auth = auth
         self.verify = verify
+        self.flags = flags or []
 
-        # 最后处理URL，确保encoding等依赖属性已初始化
+        # 默认编码
+        self.encoding = encoding
+
+        # 优先使用 json_body 参数
+        if json_body is not None:
+            if 'Content-Type' not in self.headers:
+                self.headers['Content-Type'] = 'application/json'
+            self.body = json.dumps(json_body, ensure_ascii=False).encode(self.encoding)
+            if self.method == 'GET':
+                self.method = 'POST'
+
+        # 其次使用 form_data
+        elif form_data is not None:
+            if self.method == 'GET':
+                self.method = 'POST'
+            if 'Content-Type' not in self.headers:
+                self.headers['Content-Type'] = 'application/x-www-form-urlencoded'
+            self.body = urlencode(form_data)
+
+        # 最后处理 body 为 dict 的情况
+        elif isinstance(self.body, dict):
+            if 'Content-Type' not in self.headers:
+                self.headers['Content-Type'] = 'application/json'
+            self.body = json.dumps(self.body, ensure_ascii=False).encode(self.encoding)
+
+        self.dont_filter = dont_filter
         self._set_url(url)
 
-    def copy(self):
-        return deepcopy(self)
+    def copy(self: _Request) -> _Request:
+        """
+        创建当前 Request 的副本，用于避免引用共享数据。
+
+        :return: 一个新的 Request 实例
+        """
+        return type(self)(
+            url=self.url,
+            callback=self.callback,
+            method=self.method,
+            headers=self.headers.copy(),
+            body=self.body,
+            form_data=None,  # form_data 不参与复制
+            json_body=None,   # json_body 参数也不参与复制
+            cb_kwargs=deepcopy(self.cb_kwargs),
+            err_back=self.err_back,
+            cookies=self.cookies.copy(),
+            meta=deepcopy(self._meta),
+            priority=-self.priority,
+            dont_filter=self.dont_filter,
+            timeout=self.timeout,
+            proxy=self.proxy,
+            allow_redirects=self.allow_redirects,
+            auth=self.auth,
+            verify=self.verify,
+            flags=self.flags.copy(),
+        )
 
     def set_meta(self, key: str, value: Any) -> None:
+        """
+        设置 meta 中的某个键值对。
+
+        :param key: 要设置的键
+        :param value: 对应的值
+        """
         self._meta[key] = value
 
     def _set_url(self, url: str) -> None:
+        """
+        设置并验证 URL，确保其格式正确且包含 scheme。
+
+        :param url: 原始 URL 字符串
+        :raises TypeError: 如果传入的不是字符串
+        :raises ValueError: 如果 URL 没有 scheme
+        """
         if not isinstance(url, str):
-            raise TypeError(f"Request url must be str, got {type(url).__name__}")
+            raise TypeError(f"Request url 必须为字符串类型，当前类型为 {type(url).__name__}")
 
         s = safe_url_string(url, self.encoding)
-        self._url = escape_ajax(s)
+        escaped_url = escape_ajax(s)
+        self._url = escaped_url
 
-        if (
-                "://" not in self._url
-                and not self._url.startswith("about:")
-                and not self._url.startswith("data:")
-        ):
-            raise ValueError(f"Missing scheme in request url: {self._url}")
+        if not self._url.startswith(('http://', 'https://', 'about:', '')):
+            raise ValueError(f"请求 URL 缺少 scheme（如 http://）: {self._url}")
 
     @property
     def url(self) -> str:
+        """
+        获取请求的 URL。
+
+        :return: 当前请求的 URL 字符串
+        """
         return self._url
 
     @property
     def meta(self) -> Dict[str, Any]:
+        """
+        获取请求的元数据。
+
+        :return: 元数据字典
+        """
         return self._meta
 
     def __str__(self) -> str:
+        """
+        返回对象的字符串表示，用于调试和日志输出。
+
+        :return: 字符串 <Request url=... method=...>
+        """
         return f'<Request url={self.url} method={self.method}>'
 
     def __repr__(self) -> str:
-        return self.__str__()
-
-    def __lt__(self, other) -> bool:
-        return self.priority < other.priority
-
-
-# #!/usr/bin/python
-# # -*- coding:UTF-8 -*-
-# import hashlib
-# from copy import deepcopy
-# from w3lib.url import safe_url_string
-# from typing import Dict, Optional, Callable, Union
-#
-# from crawlo.utils.url import escape_ajax
-#
-#
-# class Request(object):
-#
-#     def __init__(
-#             self,
-#             url: str,
-#             *,
-#             callback: Optional[Callable] = None,
-#             headers: Optional[Dict[str, str]] = None,
-#             body: Optional[Dict] = None,
-#             method: Optional[str] = 'GET',
-#             cookies: Optional[Dict[str, str]] = None,
-#             priority: int = 0,
-#             encoding: Optional[str] = 'UTF-8',
-#             meta: Optional[Dict[str, str]] = None,
-#             dont_filter: bool = False
-#
-#     ):
-#         self.url = url
-#         self.callback = callback
-#         self.headers = headers if headers else {}
-#         self.body = body
-#         self.method = str(method).upper()
-#         self.cookies = cookies
-#         self.priority = -priority
-#         self.encoding = encoding
-#         self.dont_filter = dont_filter
-#         self._meta = meta if meta is not None else {}
-#
-#     def copy(self):
-#         return deepcopy(self)
-#
-#     def set_meta(self, key: str, value: str):
-#         self._meta[key] = value
-#
-#     def _set_url(self, url: str) -> None:
-#         if not isinstance(url, str):
-#             raise TypeError(f"Request url must be str, got {type(url).__name__}")
-#
-#         s = safe_url_string(url, self.encoding)
-#         self._url = escape_ajax(s)
-#
-#         if (
-#             "://" not in self._url
-#             and not self._url.startswith("about:")
-#             and not self._url.startswith("data:")
-#         ):
-#             raise ValueError(f"Missing scheme in request url: {self._url}")
-#
-#     @property
-#     def meta(self):
-#         return self._meta
-#
-#     def __str__(self):
-#         return f'<Request url={self.url}> method={self.method}>'
-#
-#     def __lt__(self, other):
-#         return self.priority < other.priority
+        """
+        返回对象的官方字符串表示。
+
+        :return: 字符串，与 __str__ 相同
+        """
+        return str(self)
+
+    def __lt__(self, other: _Request) -> bool:
+        """
+        比较两个请求的优先级，用于排序。
+
+        :param other: 另一个 Request 对象
+        :return: 如果当前请求优先级更高（数值更小）返回 True
+        """
+        return self.priority < other.priority