crawlo 1.0.0__py3-none-any.whl → 1.0.1__py3-none-any.whl

crawlo/__init__.py

@@ -3,3 +3,4 @@
 from crawlo.network.request import Request
 from crawlo.network.response import Response
 from crawlo.items.items import Item
+from .__version__ import __version__

crawlo/__version__.py

@@ -1,2 +1,2 @@
 
-__version__ = "1.0.0"
+__version__ = "1.0.1"

crawlo/core/engine.py

@@ -1,20 +1,20 @@
 #!/usr/bin/python
 # -*- coding:UTF-8 -*-
 import asyncio
+from inspect import iscoroutine
 from typing import Optional, Generator, Callable
-from inspect import iscoroutine, isgenerator, isasyncgen
 
 from crawlo import Request, Item
 from crawlo.spider import Spider
+from crawlo.utils.log import get_logger
+from crawlo.exceptions import OutputError
 from crawlo.core.scheduler import Scheduler
 from crawlo.core.processor import Processor
-from crawlo.utils.log import get_logger
 from crawlo.task_manager import TaskManager
 from crawlo.utils.project import load_class
 from crawlo.downloader import DownloaderBase
 from crawlo.utils.func_tools import transform
-from crawlo.exceptions import OutputError, TransformTypeError
-from crawlo.event import spider_opened, spider_error
+from crawlo.event import spider_opened, spider_error, request_scheduled
 
 
 class Engine(object):
@@ -49,7 +49,7 @@ class Engine(object):
     async def start_spider(self, spider):
         self.spider = spider
 
-        self.scheduler = Scheduler(self.crawler)
+        self.scheduler = Scheduler.create_instance(self.crawler)
         if hasattr(self.scheduler, 'open'):
             self.scheduler.open()
 
@@ -115,7 +115,7 @@ class Engine(object):
                 if iscoroutine(_outputs):
                     await _outputs
                 else:
-                    return transform(_outputs)
+                    return transform(_outputs, _response)
 
         _response = await self.downloader.fetch(request)
         if _response is None:
@@ -128,7 +128,8 @@ class Engine(object):
 
     async def _schedule_request(self, request):
         # TODO 去重
-        await self.scheduler.enqueue_request(request)
+        if await self.scheduler.enqueue_request(request):
+            asyncio.create_task(self.crawler.subscriber.notify(request_scheduled, request, self.crawler.spider))
 
     async def _get_next_request(self):
         return await self.scheduler.next_request()
@@ -152,6 +153,7 @@ class Engine(object):
 
     async def close_spider(self):
         await asyncio.gather(*self.task_manager.current_task)
+        await self.scheduler.close()
         await self.downloader.close()
         if self.normal:
             await self.crawler.close()

crawlo/core/processor.py

@@ -15,7 +15,7 @@ class Processor(object):
         self.pipelines: Optional[PipelineManager] = None
 
     def open(self):
-        self.pipelines = PipelineManager.create_instance(self.crawler)
+        self.pipelines = PipelineManager.from_crawler(self.crawler)
 
     async def process(self):
         while not self.idle():

crawlo/core/scheduler.py

@@ -1,35 +1,59 @@
 #!/usr/bin/python
 # -*- coding:UTF-8 -*-
-import asyncio
-from typing import Optional
+from typing import Optional, Callable
 
 from crawlo.utils.log import get_logger
-from crawlo.event import request_scheduled
+from crawlo.utils.request import set_request
 from crawlo.utils.pqueue import SpiderPriorityQueue
+from crawlo.utils.project import load_class, common_call
 
 
 class Scheduler:
-    def __init__(self, crawler):
+    def __init__(self, crawler, dupe_filter, stats, log_level, priority):
         self.crawler = crawler
         self.request_queue: Optional[SpiderPriorityQueue] = None
 
-        self.item_count = 0
-        self.response_count = 0
-        self.logger = get_logger(name=self.__class__.__name__, level=crawler.settings.get('LOG_LEVEL'))
+        # self.item_count = 0
+        # self.response_count = 0
+        self.logger = get_logger(name=self.__class__.__name__, level=log_level)
+        self.stats = stats
+        self.dupe_filter = dupe_filter
+        self.priority = priority
+
+    @classmethod
+    def create_instance(cls, crawler):
+        filter_cls = load_class(crawler.settings.get('FILTER_CLASS'))
+        o = cls(
+            crawler=crawler,
+            dupe_filter=filter_cls.create_instance(crawler),
+            stats=crawler.stats,
+            log_level=crawler.settings.get('LOG_LEVEL'),
+            priority=crawler.settings.get('DEPTH_PRIORITY')
+        )
+        return o
 
     def open(self):
         self.request_queue = SpiderPriorityQueue()
+        self.logger.info(f'requesting filter: {self.dupe_filter}')
 
     async def next_request(self):
         request = await self.request_queue.get()
         return request
 
     async def enqueue_request(self, request):
+        if not request.dont_filter and await common_call(self.dupe_filter.requested, request):
+            self.dupe_filter.log_stats(request)
+            return False
+        set_request(request, self.priority)
         await self.request_queue.put(request)
-        asyncio.create_task(self.crawler.subscriber.notify(request_scheduled, request, self.crawler.spider))
+        return True
 
     def idle(self) -> bool:
         return len(self) == 0
 
+    async def close(self):
+        if isinstance(closed := getattr(self.dupe_filter, 'closed', None), Callable):
+            await closed()
+
     def __len__(self):
         return self.request_queue.qsize()

crawlo/downloader/playwright_downloader.py

@@ -0,0 +1,161 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+from typing import Optional, Dict, Any
+from playwright.async_api import Browser, Page, Response as PlaywrightResponse
+from crawlo import Response, Request
+from crawlo.downloader import DownloaderBase
+
+
+class PlaywrightDownloader(DownloaderBase):
+    def __init__(self, crawler):
+        super().__init__(crawler)
+        # Playwright 核心对象
+        self.browser: Optional[Browser] = None  # 浏览器实例
+        self.context: Optional[Any] = None  # 浏览器上下文（隔离cookies等）
+
+        # 可配置参数（通过crawler.settings覆盖默认值）
+        self._browser_type: str = "chromium"  # 浏览器类型（chromium/firefox/webkit）
+        self._headless: bool = True  # 是否无头模式
+        self._timeout: int = 30000  # 操作超时（毫秒）
+        self._viewport: Dict[str, int] = {"width": 1280, "height": 720}  # 视口大小
+        self._extra_launch_args: Dict[str, Any] = {}  # 浏览器启动额外参数
+
+    async def _init_browser(self):
+        """初始化Playwright浏览器实例"""
+        from playwright.async_api import async_playwright
+
+        # 启动Playwright引擎
+        playwright = await async_playwright().start()
+
+        # 根据配置选择浏览器类型
+        browser_launcher = {
+            "chromium": playwright.chromium,
+            "firefox": playwright.firefox,
+            "webkit": playwright.webkit
+        }.get(self._browser_type, playwright.chromium)  # 默认chromium
+
+        # 启动浏览器（含启动参数）
+        self.browser = await browser_launcher.launch(
+            headless=self._headless,  # 无头模式开关
+            timeout=self._timeout,  # 启动超时
+            **self._extra_launch_args  # 透传额外参数（如代理配置）
+        )
+
+        # 创建浏览器上下文（隔离环境）
+        self.context = await self.browser.new_context(
+            viewport=self._viewport,  # 设置窗口大小
+            user_agent=self.crawler.settings.get("USER_AGENT")  # 自定义UA
+        )
+
+    def open(self):
+        """从crawler配置加载参数"""
+        super().open()  # 调用父类初始化
+
+        # 读取配置（支持在settings.py中覆盖）
+        self._browser_type = self.crawler.settings.get("PLAYWRIGHT_BROWSER", "chromium")
+        self._headless = self.crawler.settings.get_bool("HEADLESS", True)
+        self._timeout = self.crawler.settings.get_int("PLAYWRIGHT_TIMEOUT", 30000)
+        self._viewport = self.crawler.settings.get_dict("VIEWPORT", {"width": 1280, "height": 720})
+        self._extra_launch_args = self.crawler.settings.get_dict("PLAYWRIGHT_LAUNCH_ARGS", {})
+
+    async def download(self, request: Request) -> Response:
+        """
+        核心下载方法：
+        1. 创建新页面Tab
+        2. 加载目标URL
+        3. 获取渲染后的内容
+        """
+        if not self.browser:
+            await self._init_browser()  # 懒加载浏览器
+
+        page = await self.context.new_page()  # 每个请求独立Page（自动隔离）
+
+        try:
+            # 设置请求头（模拟浏览器）
+            if request.headers:
+                await page.set_extra_http_headers(request.headers)
+
+            # 导航到目标URL（支持等待策略配置）
+            response = await page.goto(
+                request.url,
+                timeout=self._timeout,
+                wait_until="domcontentloaded"  # 等待策略：domcontentloaded/networkidle/load
+            )
+
+            # 特殊处理POST请求（Playwright限制需用API方式）
+            if request.method.lower() == "post":
+                return await self._handle_post_request(request, page)
+
+            # 执行自定义JavaScript（用于提取动态数据）
+            if request.meta.get("execute_js"):
+                result = await page.evaluate(request.meta["execute_js"])
+                request.meta["js_result"] = result  # 存储JS执行结果
+
+            # 获取渲染后的完整HTML（含动态生成内容）
+            body = await page.content()
+
+            # 调试模式下截图（用于排查页面问题）
+            if self.crawler.settings.get_bool("DEBUG"):
+                screenshot = await page.screenshot(type="png")
+                request.meta["screenshot"] = screenshot  # 截图存入request.meta
+
+            # 构造统一响应对象
+            return self._structure_response(request, response, body)
+
+        except Exception as e:
+            self.logger.error(f"页面下载失败: {str(e)}")
+            raise
+        finally:
+            await page.close()  # 确保页面关闭，避免资源泄漏
+
+    async def _handle_post_request(self, request: Request, page: Page) -> Response:
+        """
+        处理POST请求的特殊方法：
+        通过页面内fetch API发送POST请求，并监听响应
+        """
+        async with page.expect_response(request.url) as response_info:
+            # 在页面上下文中执行fetch
+            await page.evaluate(
+                """async ({url, headers, body}) => {
+                    await fetch(url, {
+                        method: 'POST',
+                        headers: headers,
+                        body: body
+                    });
+                }""",
+                {
+                    "url": request.url,
+                    "headers": request.headers or {},
+                    "body": request.body or ""
+                }
+            )
+
+        response = await response_info.value  # 获取API响应
+        body = await response.text()  # 读取响应体
+        return self._structure_response(request, response, body)
+
+    @staticmethod
+    def _structure_response(
+            request: Request,
+            response: PlaywrightResponse,
+            body: str
+    ) -> Response:
+        """
+        标准化响应格式：
+        将Playwright的响应转换为crawlo的统一Response对象
+        """
+        return Response(
+            url=str(response.url),  # 最终URL（含重定向）
+            headers=response.headers,  # 响应头
+            status_code=response.status,  # HTTP状态码
+            body=body.encode('utf-8'),  # 响应体（转bytes）
+            request=request  # 关联的请求对象
+        )
+
+    async def close(self) -> None:
+        """资源清理：关闭浏览器实例和上下文"""
+        if self.context:
+            await self.context.close()
+        if self.browser:
+            await self.browser.close()
+        await super().close()  # 调用父类清理逻辑

crawlo/extension/log_stats.py

@@ -1,7 +1,7 @@
 #!/usr/bin/python
 # -*- coding:UTF-8 -*-
 from crawlo import event
-from crawlo.utils.date_tools import now, date_delta
+from crawlo.utils.date_tools import get_current_time, time_diff_seconds
 
 
 class LogStats(object):
@@ -22,11 +22,11 @@ class LogStats(object):
         return o
 
     async def spider_opened(self):
-        self._stats['start_time'] = now()
+        self._stats['start_time'] = get_current_time(fmt='%Y-%m-%d %H:%M:%S')
 
     async def spider_closed(self):
-        self._stats['end_time'] = now()
-        self._stats['cost_time(s)'] = date_delta(start=self._stats['start_time'], end=self._stats['end_time'])
+        self._stats['end_time'] = get_current_time(fmt='%Y-%m-%d %H:%M:%S')
+        self._stats['cost_time(s)'] = time_diff_seconds(start_time=self._stats['start_time'], end_time=self._stats['end_time'])
 
     async def item_successful(self, _item, _spider):
         self._stats.inc_value('item_successful_count')

crawlo/filters/__init__.py

@@ -0,0 +1,37 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+from abc import ABC, abstractmethod
+
+from crawlo import Request
+from crawlo.utils.request import request_fingerprint
+
+
+class BaseFilter(ABC):
+
+    def __init__(self, logger, stats, debug: bool):
+        self.logger = logger
+        self.stats = stats
+        self.debug = debug
+
+    @classmethod
+    def create_instance(cls, *args, **kwargs) -> 'BaseFilter':
+        return cls(*args, **kwargs)
+
+    def requested(self, request: Request):
+        fp = request_fingerprint(request)
+        if fp in self:
+            return True
+        self.add_fingerprint(fp)
+        return False
+
+    @abstractmethod
+    def add_fingerprint(self, fp) -> None:
+        pass
+
+    def log_stats(self, request: Request) -> None:
+        if self.debug:
+            self.logger.debug(f'Filtered duplicate request: {request}')
+        self.stats.inc_value(f'{self}/filtered_count')
+
+    def __str__(self) -> str:
+        return f'{self.__class__.__name__}'

crawlo/filters/aioredis_filter.py

@@ -0,0 +1,130 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+from typing import Optional
+
+import aioredis
+
+from crawlo import Request
+from crawlo.filters import BaseFilter
+from crawlo.utils.log import get_logger
+from crawlo.utils.request import request_fingerprint
+
+
+class AioRedisFilter(BaseFilter):
+    """使用Redis集合实现的异步请求去重过滤器（适用于分布式爬虫）"""
+
+    def __init__(
+            self,
+            redis_key: str,
+            client: aioredis.Redis,
+            stats: dict,
+            debug: bool,
+            log_level: str,
+            cleanup_fp: bool = False
+    ):
+        """
+        初始化过滤器
+
+        参数说明:
+            redis_key: Redis中存储指纹的键名
+            client: aioredis客户端实例
+            stats: 统计信息字典
+            debug: 是否启用调试模式
+            log_level: 日志级别
+            save_fp: 爬虫关闭时是否保留指纹数据
+        """
+        # 初始化日志记录器（使用类名作为日志标识）
+        self.logger = get_logger(self.__class__.__name__, log_level)
+        super().__init__(self.logger, stats, debug)
+
+        self.redis_key = redis_key  # Redis存储键（如："project:request_fingerprints"）
+        self.redis = client  # Redis异步客户端
+        self.cleanup_fp = cleanup_fp  # 是否持久化指纹数据
+
+    @classmethod
+    def create_instance(cls, crawler) -> 'BaseFilter':
+        """从爬虫配置创建过滤器实例（工厂方法）"""
+        # 从配置获取Redis连接参数（带默认值）
+        redis_url = crawler.settings.get('REDIS_URL', 'redis://localhost:6379')
+        decode_responses = crawler.settings.get_bool('DECODE_RESPONSES', True)
+
+        try:
+            # 创建Redis连接池（限制最大连接数20）
+            redis_client = aioredis.from_url(
+                redis_url,
+                decode_responses=decode_responses,
+                max_connections=20
+            )
+        except Exception as e:
+            raise RuntimeError(f"Redis连接失败 {redis_url}: {str(e)}")
+
+        # 使用项目名+配置键组合作为Redis键
+        return cls(
+            redis_key=f"{crawler.settings.get('PROJECT_NAME')}:{crawler.settings.get('REDIS_KEY', 'request_fingerprints')}",
+            client=redis_client,
+            stats=crawler.stats,
+            cleanup_fp=crawler.settings.get_bool('CLEANUP_FP', False),
+            debug=crawler.settings.get_bool('FILTER_DEBUG', False),
+            log_level=crawler.settings.get('LOG_LEVEL', 'INFO')
+        )
+
+    async def requested(self, request: Request) -> bool:
+        """
+        检查请求是否重复
+
+        参数:
+            request: 要检查的请求对象
+
+        返回:
+            bool: True表示重复请求，False表示新请求
+        """
+        fp = request_fingerprint(request)  # 生成请求指纹
+        try:
+            # 检查指纹是否已存在集合中
+            is_duplicate = await self.redis.sismember(self.redis_key, fp)
+            if is_duplicate:
+                # self.logger.debug(f"发现重复请求: {fp}")
+                return True
+
+            # 新请求则添加指纹
+            await self.add_fingerprint(fp)
+            return False
+        except aioredis.RedisError as e:
+            self.logger.error(f"Redis操作失败: {str(e)}")
+            raise  # 向上抛出异常
+
+    async def add_fingerprint(self, fp: str) -> None:
+        """向Redis集合添加新指纹"""
+        try:
+            await self.redis.sadd(self.redis_key, fp)
+            self.logger.debug(f"新增指纹: {fp}")
+        except aioredis.RedisError as e:
+            self.logger.error(f"指纹添加失败: {str(e)}")
+            raise
+
+    async def closed(self, reason: Optional[str] = None) -> None:
+        """
+        爬虫关闭时的处理（兼容Scrapy的关闭逻辑）
+
+        参数:
+            reason: 爬虫关闭原因（Scrapy标准参数）
+        """
+        if self.cleanup_fp:  # 仅在配置明确要求时清理
+            try:
+                deleted = await self.redis.delete(self.redis_key)
+                self.logger.info(
+                    f"Cleaned {deleted} fingerprints from {self.redis_key} "
+                    f"(reason: {reason or 'manual'})"
+                )
+            except aioredis.RedisError as e:
+                self.logger.warning(f"Cleanup failed: {e}")
+            finally:
+                await self._close_redis()
+
+    async def _close_redis(self) -> None:
+        """安全关闭Redis连接"""
+        try:
+            await self.redis.close()
+            await self.redis.connection_pool.disconnect()
+        except Exception as e:
+            self.logger.warning(f"Redis close error: {e}")