crawlo 1.0.0__py3-none-any.whl → 1.0.2__py3-none-any.whl

crawlo/__init__.py

@@ -3,3 +3,4 @@
 from crawlo.network.request import Request
 from crawlo.network.response import Response
 from crawlo.items.items import Item
+from .__version__ import __version__

crawlo/__version__.py

@@ -1,2 +1,2 @@
 
-__version__ = "1.0.0"
+__version__ = "1.0.2"

crawlo/core/engine.py

@@ -1,20 +1,20 @@
 #!/usr/bin/python
 # -*- coding:UTF-8 -*-
 import asyncio
+from inspect import iscoroutine
 from typing import Optional, Generator, Callable
-from inspect import iscoroutine, isgenerator, isasyncgen
 
 from crawlo import Request, Item
 from crawlo.spider import Spider
+from crawlo.utils.log import get_logger
+from crawlo.exceptions import OutputError
 from crawlo.core.scheduler import Scheduler
 from crawlo.core.processor import Processor
-from crawlo.utils.log import get_logger
 from crawlo.task_manager import TaskManager
 from crawlo.utils.project import load_class
 from crawlo.downloader import DownloaderBase
 from crawlo.utils.func_tools import transform
-from crawlo.exceptions import OutputError, TransformTypeError
-from crawlo.event import spider_opened, spider_error
+from crawlo.event import spider_opened, spider_error, request_scheduled
 
 
 class Engine(object):
@@ -49,7 +49,7 @@ class Engine(object):
     async def start_spider(self, spider):
         self.spider = spider
 
-        self.scheduler = Scheduler(self.crawler)
+        self.scheduler = Scheduler.create_instance(self.crawler)
         if hasattr(self.scheduler, 'open'):
             self.scheduler.open()
 
@@ -115,7 +115,7 @@ class Engine(object):
                 if iscoroutine(_outputs):
                     await _outputs
                 else:
-                    return transform(_outputs)
+                    return transform(_outputs, _response)
 
         _response = await self.downloader.fetch(request)
         if _response is None:
@@ -128,7 +128,8 @@ class Engine(object):
 
     async def _schedule_request(self, request):
         # TODO 去重
-        await self.scheduler.enqueue_request(request)
+        if await self.scheduler.enqueue_request(request):
+            asyncio.create_task(self.crawler.subscriber.notify(request_scheduled, request, self.crawler.spider))
 
     async def _get_next_request(self):
         return await self.scheduler.next_request()
@@ -152,6 +153,7 @@ class Engine(object):
 
     async def close_spider(self):
         await asyncio.gather(*self.task_manager.current_task)
+        await self.scheduler.close()
         await self.downloader.close()
         if self.normal:
             await self.crawler.close()

crawlo/core/processor.py

@@ -15,7 +15,7 @@ class Processor(object):
         self.pipelines: Optional[PipelineManager] = None
 
     def open(self):
-        self.pipelines = PipelineManager.create_instance(self.crawler)
+        self.pipelines = PipelineManager.from_crawler(self.crawler)
 
     async def process(self):
         while not self.idle():

crawlo/core/scheduler.py

@@ -1,35 +1,59 @@
 #!/usr/bin/python
 # -*- coding:UTF-8 -*-
-import asyncio
-from typing import Optional
+from typing import Optional, Callable
 
 from crawlo.utils.log import get_logger
-from crawlo.event import request_scheduled
+from crawlo.utils.request import set_request
 from crawlo.utils.pqueue import SpiderPriorityQueue
+from crawlo.utils.project import load_class, common_call
 
 
 class Scheduler:
-    def __init__(self, crawler):
+    def __init__(self, crawler, dupe_filter, stats, log_level, priority):
         self.crawler = crawler
         self.request_queue: Optional[SpiderPriorityQueue] = None
 
-        self.item_count = 0
-        self.response_count = 0
-        self.logger = get_logger(name=self.__class__.__name__, level=crawler.settings.get('LOG_LEVEL'))
+        # self.item_count = 0
+        # self.response_count = 0
+        self.logger = get_logger(name=self.__class__.__name__, level=log_level)
+        self.stats = stats
+        self.dupe_filter = dupe_filter
+        self.priority = priority
+
+    @classmethod
+    def create_instance(cls, crawler):
+        filter_cls = load_class(crawler.settings.get('FILTER_CLASS'))
+        o = cls(
+            crawler=crawler,
+            dupe_filter=filter_cls.create_instance(crawler),
+            stats=crawler.stats,
+            log_level=crawler.settings.get('LOG_LEVEL'),
+            priority=crawler.settings.get('DEPTH_PRIORITY')
+        )
+        return o
 
     def open(self):
         self.request_queue = SpiderPriorityQueue()
+        self.logger.info(f'requesting filter: {self.dupe_filter}')
 
     async def next_request(self):
         request = await self.request_queue.get()
         return request
 
     async def enqueue_request(self, request):
+        if not request.dont_filter and await common_call(self.dupe_filter.requested, request):
+            self.dupe_filter.log_stats(request)
+            return False
+        set_request(request, self.priority)
         await self.request_queue.put(request)
-        asyncio.create_task(self.crawler.subscriber.notify(request_scheduled, request, self.crawler.spider))
+        return True
 
     def idle(self) -> bool:
         return len(self) == 0
 
+    async def close(self):
+        if isinstance(closed := getattr(self.dupe_filter, 'closed', None), Callable):
+            await closed()
+
     def __len__(self):
         return self.request_queue.qsize()

crawlo/crawler.py

@@ -6,15 +6,15 @@ from typing import Type, Final, Set, Optional
 
 from crawlo.spider import Spider
 from crawlo.core.engine import Engine
-from crawlo.subscriber import Subscriber
-
 from crawlo.utils.log import get_logger
+from crawlo.subscriber import Subscriber
 from crawlo.extension import ExtensionManager
 from crawlo.exceptions import SpiderTypeError
-from crawlo.utils.project import merge_settings
 from crawlo.stats_collector import StatsCollector
 from crawlo.event import spider_opened, spider_closed
 from crawlo.settings.setting_manager import SettingManager
+from crawlo.utils.project import merge_settings, get_settings
+from crawlo.utils.concurrency_manager import calculate_optimal_concurrency
 
 logger = get_logger(__name__)
 
@@ -72,36 +72,151 @@ class Crawler:
 
 
 class CrawlerProcess:
+    """爬虫处理类，支持跨平台动态并发控制和精细化日志"""
 
-    def __init__(self, settings=None):
+    def __init__(self, settings=None, max_concurrency: Optional[int] = None, batch_size: int = 50):
         self.crawlers: Final[Set] = set()
         self._active_spiders: Final[Set] = set()
-        self.settings = settings
+        self.settings = settings or self._get_default_settings()
+        self.batch_size = batch_size
+
+        # 使用独立模块计算最大并发数
+        self.max_concurrency = calculate_optimal_concurrency(max_concurrency)
+        self.semaphore = asyncio.Semaphore(self.max_concurrency)
 
         signal.signal(signal.SIGINT, self._shutdown)
+        logger.info(f"初始化爬虫处理进程，最大并发数: {self.max_concurrency}")
 
-    async def crawl(self, spider: Type[Spider]):
-        crawler: Crawler = self._create_crawler(spider)
-        self.crawlers.add(crawler)
-        task = await self._crawl(crawler)
-        self._active_spiders.add(task)
+    async def crawl(self, spiders):
+        """支持单个或多个爬虫的批量处理，优化日志输出"""
+        if not spiders:
+            raise ValueError("至少需要提供一个爬虫类")
 
-    @staticmethod
-    async def _crawl(crawler):
-        return asyncio.create_task(crawler.crawl())
+        # 统一转换为列表
+        if isinstance(spiders, type) and issubclass(spiders, Spider):
+            spiders = [spiders]
+        elif isinstance(spiders, (list, tuple)):
+            spiders = list(spiders)
+        else:
+            raise TypeError("spiders 必须是爬虫类或爬虫类列表/元组")
+
+        # 按爬虫类名首字母排序（升序）
+        spiders.sort(key=lambda x: x.__name__.lower())
+
+        if len(spiders) == 1:
+            logger.info(f"启动爬虫: {spiders[0].__name__}")
+        else:
+            logger.info(f"启动{len(spiders)}个爬虫，按名称排序后分批处理中")
+
+        batches = [spiders[i:i + self.batch_size] for i in range(0, len(spiders), self.batch_size)]
+
+        for batch_idx, batch in enumerate(batches):
+            batch_tasks = set()
+
+            for spider_cls in batch:
+                crawler = self._create_crawler(spider_cls)
+                self.crawlers.add(crawler)
+
+                await self.semaphore.acquire()
+                task = asyncio.create_task(self._run_crawler_with_semaphore(crawler))
+                batch_tasks.add(task)
+                self._active_spiders.add(task)
+
+            if len(spiders) > 1:  # 仅对多爬虫显示批次信息
+                logger.info(f"启动第 {batch_idx + 1}/{len(batches)} 批爬虫，共 {len(batch)} 个")
+
+            await asyncio.gather(*batch_tasks)
+
+            if len(spiders) > 1:  # 仅对多爬虫显示批次完成信息
+                logger.info(f"第 {batch_idx + 1} 批爬虫处理完成")
+
+    async def _run_crawler_with_semaphore(self, crawler):
+        """使用信号量控制的爬虫运行函数"""
+        try:
+            await crawler.crawl()
+        finally:
+            self.semaphore.release()  # 确保资源释放
 
     async def start(self):
-        await asyncio.gather(*self._active_spiders)
+        """启动所有爬虫任务"""
+        if self._active_spiders:
+            logger.info(f"启动 {len(self._active_spiders)} 个爬虫任务，并发限制: {self.max_concurrency}")
+            await asyncio.gather(*self._active_spiders)
 
     def _create_crawler(self, spider_cls) -> Crawler:
+        """创建爬虫实例"""
         if isinstance(spider_cls, str):
             raise SpiderTypeError(f"{type(self)}.crawl args: String is not supported.")
         crawler: Crawler = Crawler(spider_cls, self.settings)
         return crawler
 
     def _shutdown(self, _signum, _frame):
+        """优雅关闭所有爬虫"""
+        logger.warning(f"收到关闭信号，正在优雅关闭 {len(self.crawlers)} 个爬虫...")
         for crawler in self.crawlers:
-            crawler.engine.running = False
-            crawler.engine.normal = False
-            crawler.stats.close_spider(crawler.spider, 'Ctrl C')
-        logger.warning(f'spiders received: `Ctrl C` signal, closed.')
+            if crawler.engine:
+                crawler.engine.running = False
+                crawler.engine.normal = False
+                crawler.stats.close_spider(crawler.spider, 'shutdown signal')
+
+        # 等待所有任务完成
+        asyncio.create_task(self._wait_for_tasks())
+
+    async def _wait_for_tasks(self):
+        """等待所有活跃任务完成"""
+        pending = [task for task in self._active_spiders if not task.done()]
+        if pending:
+            logger.info(f"等待 {len(pending)} 个活跃任务完成...")
+            await asyncio.gather(*pending)
+        logger.info("所有爬虫已优雅关闭")
+
+    @classmethod
+    def _get_default_settings(cls):
+        """框架自动获取默认配置"""
+        try:
+            return get_settings()
+        except ImportError:
+            return {}
+
+# class CrawlerProcess:
+#
+#     def __init__(self, settings=None):
+#         self.crawlers: Final[Set] = set()
+#         self._active_spiders: Final[Set] = set()
+#         self.settings = settings or self._get_default_settings()
+#
+#         signal.signal(signal.SIGINT, self._shutdown)
+#
+#     async def crawl(self, spider: Type[Spider]):
+#         crawler: Crawler = self._create_crawler(spider)
+#         self.crawlers.add(crawler)
+#         task = await self._crawl(crawler)
+#         self._active_spiders.add(task)
+#
+#     @classmethod
+#     def _get_default_settings(cls):
+#         """自动获取默认配置"""
+#         try:
+#             return get_settings()
+#         except ImportError:
+#             return {}
+#
+#     @staticmethod
+#     async def _crawl(crawler):
+#         return asyncio.create_task(crawler.crawl())
+#
+#     async def start(self):
+#         await asyncio.gather(*self._active_spiders)
+#
+#     def _create_crawler(self, spider_cls) -> Crawler:
+#         if isinstance(spider_cls, str):
+#             raise SpiderTypeError(f"{type(self)}.crawl args: String is not supported.")
+#         crawler: Crawler = Crawler(spider_cls, self.settings)
+#         return crawler
+#
+#     def _shutdown(self, _signum, _frame):
+#         for crawler in self.crawlers:
+#             crawler.engine.running = False
+#             crawler.engine.normal = False
+#             crawler.stats.close_spider(crawler.spider, 'Ctrl C')
+#         logger.warning(f'spiders received: `Ctrl C` signal, closed.')

crawlo/downloader/playwright_downloader.py

@@ -0,0 +1,161 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+from typing import Optional, Dict, Any
+from playwright.async_api import Browser, Page, Response as PlaywrightResponse
+from crawlo import Response, Request
+from crawlo.downloader import DownloaderBase
+
+
+class PlaywrightDownloader(DownloaderBase):
+    def __init__(self, crawler):
+        super().__init__(crawler)
+        # Playwright 核心对象
+        self.browser: Optional[Browser] = None  # 浏览器实例
+        self.context: Optional[Any] = None  # 浏览器上下文（隔离cookies等）
+
+        # 可配置参数（通过crawler.settings覆盖默认值）
+        self._browser_type: str = "chromium"  # 浏览器类型（chromium/firefox/webkit）
+        self._headless: bool = True  # 是否无头模式
+        self._timeout: int = 30000  # 操作超时（毫秒）
+        self._viewport: Dict[str, int] = {"width": 1280, "height": 720}  # 视口大小
+        self._extra_launch_args: Dict[str, Any] = {}  # 浏览器启动额外参数
+
+    async def _init_browser(self):
+        """初始化Playwright浏览器实例"""
+        from playwright.async_api import async_playwright
+
+        # 启动Playwright引擎
+        playwright = await async_playwright().start()
+
+        # 根据配置选择浏览器类型
+        browser_launcher = {
+            "chromium": playwright.chromium,
+            "firefox": playwright.firefox,
+            "webkit": playwright.webkit
+        }.get(self._browser_type, playwright.chromium)  # 默认chromium
+
+        # 启动浏览器（含启动参数）
+        self.browser = await browser_launcher.launch(
+            headless=self._headless,  # 无头模式开关
+            timeout=self._timeout,  # 启动超时
+            **self._extra_launch_args  # 透传额外参数（如代理配置）
+        )
+
+        # 创建浏览器上下文（隔离环境）
+        self.context = await self.browser.new_context(
+            viewport=self._viewport,  # 设置窗口大小
+            user_agent=self.crawler.settings.get("USER_AGENT")  # 自定义UA
+        )
+
+    def open(self):
+        """从crawler配置加载参数"""
+        super().open()  # 调用父类初始化
+
+        # 读取配置（支持在settings.py中覆盖）
+        self._browser_type = self.crawler.settings.get("PLAYWRIGHT_BROWSER", "chromium")
+        self._headless = self.crawler.settings.get_bool("HEADLESS", True)
+        self._timeout = self.crawler.settings.get_int("PLAYWRIGHT_TIMEOUT", 30000)
+        self._viewport = self.crawler.settings.get_dict("VIEWPORT", {"width": 1280, "height": 720})
+        self._extra_launch_args = self.crawler.settings.get_dict("PLAYWRIGHT_LAUNCH_ARGS", {})
+
+    async def download(self, request: Request) -> Response:
+        """
+        核心下载方法：
+        1. 创建新页面Tab
+        2. 加载目标URL
+        3. 获取渲染后的内容
+        """
+        if not self.browser:
+            await self._init_browser()  # 懒加载浏览器
+
+        page = await self.context.new_page()  # 每个请求独立Page（自动隔离）
+
+        try:
+            # 设置请求头（模拟浏览器）
+            if request.headers:
+                await page.set_extra_http_headers(request.headers)
+
+            # 导航到目标URL（支持等待策略配置）
+            response = await page.goto(
+                request.url,
+                timeout=self._timeout,
+                wait_until="domcontentloaded"  # 等待策略：domcontentloaded/networkidle/load
+            )
+
+            # 特殊处理POST请求（Playwright限制需用API方式）
+            if request.method.lower() == "post":
+                return await self._handle_post_request(request, page)
+
+            # 执行自定义JavaScript（用于提取动态数据）
+            if request.meta.get("execute_js"):
+                result = await page.evaluate(request.meta["execute_js"])
+                request.meta["js_result"] = result  # 存储JS执行结果
+
+            # 获取渲染后的完整HTML（含动态生成内容）
+            body = await page.content()
+
+            # 调试模式下截图（用于排查页面问题）
+            if self.crawler.settings.get_bool("DEBUG"):
+                screenshot = await page.screenshot(type="png")
+                request.meta["screenshot"] = screenshot  # 截图存入request.meta
+
+            # 构造统一响应对象
+            return self._structure_response(request, response, body)
+
+        except Exception as e:
+            self.logger.error(f"页面下载失败: {str(e)}")
+            raise
+        finally:
+            await page.close()  # 确保页面关闭，避免资源泄漏
+
+    async def _handle_post_request(self, request: Request, page: Page) -> Response:
+        """
+        处理POST请求的特殊方法：
+        通过页面内fetch API发送POST请求，并监听响应
+        """
+        async with page.expect_response(request.url) as response_info:
+            # 在页面上下文中执行fetch
+            await page.evaluate(
+                """async ({url, headers, body}) => {
+                    await fetch(url, {
+                        method: 'POST',
+                        headers: headers,
+                        body: body
+                    });
+                }""",
+                {
+                    "url": request.url,
+                    "headers": request.headers or {},
+                    "body": request.body or ""
+                }
+            )
+
+        response = await response_info.value  # 获取API响应
+        body = await response.text()  # 读取响应体
+        return self._structure_response(request, response, body)
+
+    @staticmethod
+    def _structure_response(
+            request: Request,
+            response: PlaywrightResponse,
+            body: str
+    ) -> Response:
+        """
+        标准化响应格式：
+        将Playwright的响应转换为crawlo的统一Response对象
+        """
+        return Response(
+            url=str(response.url),  # 最终URL（含重定向）
+            headers=response.headers,  # 响应头
+            status_code=response.status,  # HTTP状态码
+            body=body.encode('utf-8'),  # 响应体（转bytes）
+            request=request  # 关联的请求对象
+        )
+
+    async def close(self) -> None:
+        """资源清理：关闭浏览器实例和上下文"""
+        if self.context:
+            await self.context.close()
+        if self.browser:
+            await self.browser.close()
+        await super().close()  # 调用父类清理逻辑

crawlo/extension/log_stats.py

@@ -1,7 +1,7 @@
 #!/usr/bin/python
 # -*- coding:UTF-8 -*-
 from crawlo import event
-from crawlo.utils.date_tools import now, date_delta
+from crawlo.utils.date_tools import get_current_time, time_diff_seconds
 
 
 class LogStats(object):
@@ -22,11 +22,11 @@ class LogStats(object):
         return o
 
     async def spider_opened(self):
-        self._stats['start_time'] = now()
+        self._stats['start_time'] = get_current_time(fmt='%Y-%m-%d %H:%M:%S')
 
     async def spider_closed(self):
-        self._stats['end_time'] = now()
-        self._stats['cost_time(s)'] = date_delta(start=self._stats['start_time'], end=self._stats['end_time'])
+        self._stats['end_time'] = get_current_time(fmt='%Y-%m-%d %H:%M:%S')
+        self._stats['cost_time(s)'] = time_diff_seconds(start_time=self._stats['start_time'], end_time=self._stats['end_time'])
 
     async def item_successful(self, _item, _spider):
         self._stats.inc_value('item_successful_count')

crawlo/filters/__init__.py

@@ -0,0 +1,37 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+from abc import ABC, abstractmethod
+
+from crawlo import Request
+from crawlo.utils.request import request_fingerprint
+
+
+class BaseFilter(ABC):
+
+    def __init__(self, logger, stats, debug: bool):
+        self.logger = logger
+        self.stats = stats
+        self.debug = debug
+
+    @classmethod
+    def create_instance(cls, *args, **kwargs) -> 'BaseFilter':
+        return cls(*args, **kwargs)
+
+    def requested(self, request: Request):
+        fp = request_fingerprint(request)
+        if fp in self:
+            return True
+        self.add_fingerprint(fp)
+        return False
+
+    @abstractmethod
+    def add_fingerprint(self, fp) -> None:
+        pass
+
+    def log_stats(self, request: Request) -> None:
+        if self.debug:
+            self.logger.debug(f'Filtered duplicate request: {request}')
+        self.stats.inc_value(f'{self}/filtered_count')
+
+    def __str__(self) -> str:
+        return f'{self.__class__.__name__}'