crawlo 1.0.4__py3-none-any.whl → 1.0.5__py3-none-any.whl

crawlo/extension/__init__.py

@@ -1,31 +1,31 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-from typing import List
-from pprint import pformat
-
-from crawlo.utils.log import get_logger
-from crawlo.utils.project import load_class
-from crawlo.exceptions import ExtensionInitError
-
-
-class ExtensionManager(object):
-
-    def __init__(self, crawler):
-        self.crawler = crawler
-        self.extensions: List = []
-        extensions = self.crawler.settings.get_list('EXTENSIONS')
-        self.logger = get_logger(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
-        self._add_extensions(extensions)
-
-    @classmethod
-    def create_instance(cls, *args, **kwargs):
-        return cls(*args, **kwargs)
-
-    def _add_extensions(self, extensions):
-        for extension in extensions:
-            extension_cls = load_class(extension)
-            if not hasattr(extension_cls, 'create_instance'):
-                raise ExtensionInitError(f"extension init failed, Must have method 'create_instance()")
-            self.extensions.append(extension_cls.create_instance(self.crawler))
-        if extensions:
-            self.logger.info(f"enabled extensions: \n {pformat(extensions)}")
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+from typing import List
+from pprint import pformat
+
+from crawlo.utils.log import get_logger
+from crawlo.utils.project import load_class
+from crawlo.exceptions import ExtensionInitError
+
+
+class ExtensionManager(object):
+
+    def __init__(self, crawler):
+        self.crawler = crawler
+        self.extensions: List = []
+        extensions = self.crawler.settings.get_list('EXTENSIONS')
+        self.logger = get_logger(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
+        self._add_extensions(extensions)
+
+    @classmethod
+    def create_instance(cls, *args, **kwargs):
+        return cls(*args, **kwargs)
+
+    def _add_extensions(self, extensions):
+        for extension in extensions:
+            extension_cls = load_class(extension)
+            if not hasattr(extension_cls, 'create_instance'):
+                raise ExtensionInitError(f"extension init failed, Must have method 'create_instance()")
+            self.extensions.append(extension_cls.create_instance(self.crawler))
+        if extensions:
+            self.logger.info(f"enabled extensions: \n {pformat(extensions)}")

crawlo/extension/log_interval.py

@@ -1,49 +1,49 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-import asyncio
-
-from crawlo.utils.log import get_logger
-from crawlo.event import spider_opened, spider_closed
-
-
-class LogIntervalExtension(object):
-
-    def __init__(self, crawler):
-        self.task = None
-        self.stats = crawler.stats
-        self.item_count = 0
-        self.response_count = 0
-        self.seconds = crawler.settings.get('INTERVAL')
-        self.interval = int(self.seconds / 60) if self.seconds % 60 == 0 else self.seconds
-        self.interval = "" if self.interval == 1 else self.interval
-        self.unit = 'min' if self.seconds % 60 == 0 else 's'
-
-        self.logger = get_logger(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
-
-    @classmethod
-    def create_instance(cls, crawler):
-        o = cls(crawler)
-        crawler.subscriber.subscribe(o.spider_opened, event=spider_opened)
-        crawler.subscriber.subscribe(o.spider_closed, event=spider_closed)
-        return o
-
-    async def spider_opened(self):
-        self.task = asyncio.create_task(self.interval_log())
-        await self.task
-
-    async def spider_closed(self):
-        if self.task:
-            self.task.cancel()
-
-    async def interval_log(self):
-        while True:
-            last_item_count = self.stats.get_value('item_successful_count', default=0)
-            last_response_count = self.stats.get_value('response_received_count', default=0)
-            item_rate = last_item_count - self.item_count
-            response_rate = last_response_count - self.response_count
-            self.item_count, self.response_count = last_item_count, last_response_count
-            self.logger.info(
-                f'Crawled {last_response_count} pages (at {response_rate} pages/{self.interval}{self.unit}),'
-                f' Got {last_item_count} items (at {item_rate} items/{self.interval}{self.unit}).'
-            )
-            await asyncio.sleep(self.seconds)
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+import asyncio
+
+from crawlo.utils.log import get_logger
+from crawlo.event import spider_opened, spider_closed
+
+
+class LogIntervalExtension(object):
+
+    def __init__(self, crawler):
+        self.task = None
+        self.stats = crawler.stats
+        self.item_count = 0
+        self.response_count = 0
+        self.seconds = crawler.settings.get('INTERVAL')
+        self.interval = int(self.seconds / 60) if self.seconds % 60 == 0 else self.seconds
+        self.interval = "" if self.interval == 1 else self.interval
+        self.unit = 'min' if self.seconds % 60 == 0 else 's'
+
+        self.logger = get_logger(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
+
+    @classmethod
+    def create_instance(cls, crawler):
+        o = cls(crawler)
+        crawler.subscriber.subscribe(o.spider_opened, event=spider_opened)
+        crawler.subscriber.subscribe(o.spider_closed, event=spider_closed)
+        return o
+
+    async def spider_opened(self):
+        self.task = asyncio.create_task(self.interval_log())
+        await self.task
+
+    async def spider_closed(self):
+        if self.task:
+            self.task.cancel()
+
+    async def interval_log(self):
+        while True:
+            last_item_count = self.stats.get_value('item_successful_count', default=0)
+            last_response_count = self.stats.get_value('response_received_count', default=0)
+            item_rate = last_item_count - self.item_count
+            response_rate = last_response_count - self.response_count
+            self.item_count, self.response_count = last_item_count, last_response_count
+            self.logger.info(
+                f'Crawled {last_response_count} pages (at {response_rate} pages/{self.interval}{self.unit}),'
+                f' Got {last_item_count} items (at {item_rate} items/{self.interval}{self.unit}).'
+            )
+            await asyncio.sleep(self.seconds)

crawlo/extension/log_stats.py

@@ -1,44 +1,44 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-from crawlo import event
-from crawlo.utils.date_tools import get_current_time, time_diff_seconds
-
-
-class LogStats(object):
-
-    def __init__(self, stats):
-        self._stats = stats
-
-    @classmethod
-    def create_instance(cls, crawler):
-        o = cls(crawler.stats)
-        crawler.subscriber.subscribe(o.spider_opened, event=event.spider_opened)
-        crawler.subscriber.subscribe(o.spider_closed, event=event.spider_closed)
-        crawler.subscriber.subscribe(o.item_successful, event=event.item_successful)
-        crawler.subscriber.subscribe(o.item_discard, event=event.item_discard)
-        crawler.subscriber.subscribe(o.response_received, event=event.response_received)
-        crawler.subscriber.subscribe(o.request_scheduled, event=event.request_scheduled)
-
-        return o
-
-    async def spider_opened(self):
-        self._stats['start_time'] = get_current_time(fmt='%Y-%m-%d %H:%M:%S')
-
-    async def spider_closed(self):
-        self._stats['end_time'] = get_current_time(fmt='%Y-%m-%d %H:%M:%S')
-        self._stats['cost_time(s)'] = time_diff_seconds(start_time=self._stats['start_time'], end_time=self._stats['end_time'])
-
-    async def item_successful(self, _item, _spider):
-        self._stats.inc_value('item_successful_count')
-
-    async def item_discard(self, _item, exc, _spider):
-        self._stats.inc_value('item_discard_count')
-        reason = exc.msg
-        if reason:
-            self._stats.inc_value(f"item_discard/{reason}")
-
-    async def response_received(self, _response, _spider):
-        self._stats.inc_value('response_received_count')
-
-    async def request_scheduled(self, _request, _spider):
-        self._stats.inc_value('request_scheduler_count')
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+from crawlo import event
+from crawlo.utils.date_tools import now, time_diff
+
+
+class LogStats(object):
+
+    def __init__(self, stats):
+        self._stats = stats
+
+    @classmethod
+    def create_instance(cls, crawler):
+        o = cls(crawler.stats)
+        crawler.subscriber.subscribe(o.spider_opened, event=event.spider_opened)
+        crawler.subscriber.subscribe(o.spider_closed, event=event.spider_closed)
+        crawler.subscriber.subscribe(o.item_successful, event=event.item_successful)
+        crawler.subscriber.subscribe(o.item_discard, event=event.item_discard)
+        crawler.subscriber.subscribe(o.response_received, event=event.response_received)
+        crawler.subscriber.subscribe(o.request_scheduled, event=event.request_scheduled)
+
+        return o
+
+    async def spider_opened(self):
+        self._stats['start_time'] = now(fmt='%Y-%m-%d %H:%M:%S')
+
+    async def spider_closed(self):
+        self._stats['end_time'] = now(fmt='%Y-%m-%d %H:%M:%S')
+        self._stats['cost_time(s)'] = time_diff(start=self._stats['start_time'], end=self._stats['end_time'])
+
+    async def item_successful(self, _item, _spider):
+        self._stats.inc_value('item_successful_count')
+
+    async def item_discard(self, _item, exc, _spider):
+        self._stats.inc_value('item_discard_count')
+        reason = exc.msg
+        if reason:
+            self._stats.inc_value(f"item_discard/{reason}")
+
+    async def response_received(self, _response, _spider):
+        self._stats.inc_value('response_received_count')
+
+    async def request_scheduled(self, _request, _spider):
+        self._stats.inc_value('request_scheduler_count')

crawlo/extension/logging_extension.py

@@ -0,0 +1,35 @@
+from crawlo.exceptions import NotConfigured
+from crawlo.utils.log import get_logger
+from crawlo.utils.log import LoggerManager
+
+
+class CustomLoggerExtension:
+    """
+    日志系统初始化扩展
+    遵循与 ExtensionManager 一致的接口规范：使用 create_instance
+    """
+
+    def __init__(self, settings):
+        self.settings = settings
+        # 初始化全局日志配置
+        LoggerManager.configure(settings)
+
+    @classmethod
+    def create_instance(cls, crawler, *args, **kwargs):
+        """
+        工厂方法：兼容 ExtensionManager 的创建方式
+        被 ExtensionManager 调用
+        """
+        # 可以通过 settings 控制是否启用
+        if not crawler.settings.get('LOG_FILE') and not crawler.settings.get('LOG_ENABLE_CUSTOM'):
+            raise NotConfigured("CustomLoggerExtension: LOG_FILE not set and LOG_ENABLE_CUSTOM=False")
+
+        return cls(crawler.settings)
+
+    def spider_opened(self, spider):
+        logger = get_logger(__name__)
+        logger.info(
+            f"CustomLoggerExtension: Logging initialized. "
+            f"LOG_FILE={self.settings.get('LOG_FILE')}, "
+            f"LOG_LEVEL={self.settings.get('LOG_LEVEL')}"
+        )

crawlo/filters/__init__.py

@@ -1,37 +1,37 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-from abc import ABC, abstractmethod
-
-from crawlo import Request
-from crawlo.utils.request import request_fingerprint
-
-
-class BaseFilter(ABC):
-
-    def __init__(self, logger, stats, debug: bool):
-        self.logger = logger
-        self.stats = stats
-        self.debug = debug
-
-    @classmethod
-    def create_instance(cls, *args, **kwargs) -> 'BaseFilter':
-        return cls(*args, **kwargs)
-
-    def requested(self, request: Request):
-        fp = request_fingerprint(request)
-        if fp in self:
-            return True
-        self.add_fingerprint(fp)
-        return False
-
-    @abstractmethod
-    def add_fingerprint(self, fp) -> None:
-        pass
-
-    def log_stats(self, request: Request) -> None:
-        if self.debug:
-            self.logger.debug(f'Filtered duplicate request: {request}')
-        self.stats.inc_value(f'{self}/filtered_count')
-
-    def __str__(self) -> str:
-        return f'{self.__class__.__name__}'
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+from abc import ABC, abstractmethod
+
+from crawlo import Request
+from crawlo.utils.request import request_fingerprint
+
+
+class BaseFilter(ABC):
+
+    def __init__(self, logger, stats, debug: bool):
+        self.logger = logger
+        self.stats = stats
+        self.debug = debug
+
+    @classmethod
+    def create_instance(cls, *args, **kwargs) -> 'BaseFilter':
+        return cls(*args, **kwargs)
+
+    def requested(self, request: Request):
+        fp = request_fingerprint(request)
+        if fp in self:
+            return True
+        self.add_fingerprint(fp)
+        return False
+
+    @abstractmethod
+    def add_fingerprint(self, fp) -> None:
+        pass
+
+    def log_stats(self, request: Request) -> None:
+        if self.debug:
+            self.logger.debug(f'Filtered duplicate request: {request}')
+        self.stats.inc_value(f'{self}/filtered_count')
+
+    def __str__(self) -> str:
+        return f'{self.__class__.__name__}'

crawlo/filters/aioredis_filter.py

@@ -1,150 +1,150 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-from typing import Optional
-import aioredis
-from crawlo import Request
-from crawlo.filters import BaseFilter
-from crawlo.utils.log import get_logger
-from crawlo.utils.request import request_fingerprint
-
-
-class AioRedisFilter(BaseFilter):
-    """基于Redis集合实现的异步请求去重过滤器（支持分布式爬虫），提供TTL和清理控制"""
-
-    def __init__(
-            self,
-            redis_key: str,
-            client: aioredis.Redis,
-            stats: dict,
-            debug: bool,
-            log_level: str,
-            cleanup_fp: bool = False,
-            ttl: Optional[int] = None  # None表示持久化，>0表示过期时间(秒)
-    ):
-        """初始化过滤器"""
-        self.logger = get_logger(self.__class__.__name__, log_level)
-        super().__init__(self.logger, stats, debug)
-
-        self.redis_key = redis_key
-        self.redis = client
-        self.cleanup_fp = cleanup_fp
-        self.ttl = ttl
-
-    @classmethod
-    def create_instance(cls, crawler) -> 'BaseFilter':
-        """从爬虫配置创建过滤器实例"""
-        redis_url = crawler.settings.get('REDIS_URL', 'redis://localhost:6379')
-        decode_responses = crawler.settings.get_bool('DECODE_RESPONSES', False)
-        ttl_setting = crawler.settings.get_int('REDIS_TTL')
-
-        # 处理TTL设置
-        ttl = None
-        if ttl_setting is not None:
-            ttl = max(0, int(ttl_setting)) if ttl_setting > 0 else None
-
-        try:
-            redis_client = aioredis.from_url(
-                redis_url,
-                decode_responses=decode_responses,
-                max_connections=20,
-                encoding='utf-8'
-            )
-        except Exception as e:
-            raise RuntimeError(f"Redis连接失败: {redis_url} - {str(e)}")
-
-        return cls(
-            redis_key=f"{crawler.settings.get('PROJECT_NAME', 'default')}:{crawler.settings.get('REDIS_KEY', 'request_fingerprints')}",
-            client=redis_client,
-            stats=crawler.stats,
-            cleanup_fp=crawler.settings.get_bool('CLEANUP_FP', False),
-            ttl=ttl,
-            debug=crawler.settings.get_bool('FILTER_DEBUG', False),
-            log_level=crawler.settings.get('LOG_LEVEL', 'INFO')
-        )
-
-    async def requested(self, request: Request) -> bool:
-        """检查请求是否已存在"""
-        try:
-            fp = str(request_fingerprint(request))
-
-            # 1. 检查指纹是否存在
-            pipe = self.redis.pipeline()
-            pipe.sismember(self.redis_key, fp)  # 不单独 await
-            exists = (await pipe.execute())[0]  # 执行并获取结果
-
-            if exists:  # 如果已存在，返回 True
-                return True
-
-            # 2. 如果不存在，添加指纹并设置 TTL
-            pipe = self.redis.pipeline()
-            pipe.sadd(self.redis_key, fp)  # 不单独 await
-            if self.ttl and self.ttl > 0:
-                pipe.expire(self.redis_key, self.ttl)  # 不单独 await
-            await pipe.execute()  # 一次性执行所有命令
-
-            return False  # 表示是新请求
-
-        except Exception as e:
-            self.logger.error(f"请求检查失败: {getattr(request, 'url', '未知URL')}")
-            raise
-
-    async def add_fingerprint(self, fp: str) -> bool:
-        """添加新指纹到Redis集合"""
-        try:
-            fp = str(fp)
-            added = await self.redis.sadd(self.redis_key, fp)
-
-            if self.ttl and self.ttl > 0:
-                await self.redis.expire(self.redis_key, self.ttl)
-
-            return added == 1
-        except Exception as e:
-            self.logger.error("添加指纹失败")
-            raise
-
-    async def get_stats(self) -> dict:
-        """获取过滤器统计信息"""
-        try:
-            count = await self.redis.scard(self.redis_key)
-            stats = {
-                '指纹总数': count,
-                'Redis键名': self.redis_key,
-                'TTL配置': f"{self.ttl}秒" if self.ttl else "持久化"
-            }
-            stats.update(self.stats)
-            return stats
-        except Exception as e:
-            self.logger.error("获取统计信息失败")
-            return self.stats
-
-    async def clear_all(self) -> int:
-        """清空所有指纹数据"""
-        try:
-            deleted = await self.redis.delete(self.redis_key)
-            self.logger.info(f"已清除指纹数: {deleted}")
-            return deleted
-        except Exception as e:
-            self.logger.error("清空指纹失败")
-            raise
-
-    async def closed(self, reason: Optional[str] = None) -> None:
-        """爬虫关闭时的清理操作"""
-        try:
-            if self.cleanup_fp:
-                deleted = await self.redis.delete(self.redis_key)
-                self.logger.info(f"爬虫关闭清理: 已删除{deleted}个指纹")
-            else:
-                count = await self.redis.scard(self.redis_key)
-                ttl_info = f"{self.ttl}秒" if self.ttl else "持久化"
-                self.logger.info(f"保留指纹数: {count} (TTL: {ttl_info})")
-        finally:
-            await self._close_redis()
-
-    async def _close_redis(self) -> None:
-        """安全关闭Redis连接"""
-        try:
-            if hasattr(self.redis, 'close'):
-                await self.redis.close()
-                self.logger.debug("Redis连接已关闭")
-        except Exception as e:
-            self.logger.warning(f"Redis关闭时出错：{e}")
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+import aioredis
+from typing import Optional
+from crawlo import Request
+from crawlo.filters import BaseFilter
+from crawlo.utils.log import get_logger
+from crawlo.utils.request import request_fingerprint
+
+
+class AioRedisFilter(BaseFilter):
+    """基于Redis集合实现的异步请求去重过滤器（支持分布式爬虫），提供TTL和清理控制"""
+
+    def __init__(
+            self,
+            redis_key: str,
+            client: aioredis.Redis,
+            stats: dict,
+            debug: bool,
+            log_level: str,
+            cleanup_fp: bool = False,
+            ttl: Optional[int] = None
+    ):
+        """初始化过滤器"""
+        self.logger = get_logger(self.__class__.__name__, log_level)
+        super().__init__(self.logger, stats, debug)
+
+        self.redis_key = redis_key
+        self.redis = client
+        self.cleanup_fp = cleanup_fp
+        self.ttl = ttl
+
+    @classmethod
+    def create_instance(cls, crawler) -> 'BaseFilter':
+        """从爬虫配置创建过滤器实例"""
+        redis_url = crawler.settings.get('REDIS_URL', 'redis://localhost:6379')
+        decode_responses = crawler.settings.get_bool('DECODE_RESPONSES', False)
+        ttl_setting = crawler.settings.get_int('REDIS_TTL')
+
+        # 处理TTL设置
+        ttl = None
+        if ttl_setting is not None:
+            ttl = max(0, int(ttl_setting)) if ttl_setting > 0 else None
+
+        try:
+            redis_client = aioredis.from_url(
+                redis_url,
+                decode_responses=decode_responses,
+                max_connections=20,
+                encoding='utf-8'
+            )
+        except Exception as e:
+            raise RuntimeError(f"Redis连接失败: {redis_url} - {str(e)}")
+
+        return cls(
+            redis_key=f"{crawler.settings.get('PROJECT_NAME', 'default')}:{crawler.settings.get('REDIS_KEY', 'request_fingerprints')}",
+            client=redis_client,
+            stats=crawler.stats,
+            cleanup_fp=crawler.settings.get_bool('CLEANUP_FP', False),
+            ttl=ttl,
+            debug=crawler.settings.get_bool('FILTER_DEBUG', False),
+            log_level=crawler.settings.get('LOG_LEVEL', 'INFO')
+        )
+
+    async def requested(self, request: Request) -> bool:
+        """检查请求是否已存在"""
+        try:
+            fp = str(request_fingerprint(request))
+
+            # 1. 检查指纹是否存在
+            pipe = self.redis.pipeline()
+            pipe.sismember(self.redis_key, fp)  # 不单独 await
+            exists = (await pipe.execute())[0]  # 执行并获取结果
+
+            if exists:  # 如果已存在，返回 True
+                return True
+
+            # 2. 如果不存在，添加指纹并设置 TTL
+            pipe = self.redis.pipeline()
+            pipe.sadd(self.redis_key, fp)  # 不单独 await
+            if self.ttl and self.ttl > 0:
+                pipe.expire(self.redis_key, self.ttl)  # 不单独 await
+            await pipe.execute()  # 一次性执行所有命令
+
+            return False  # 表示是新请求
+
+        except Exception as e:
+            self.logger.error(f"请求检查失败: {getattr(request, 'url', '未知URL')}")
+            raise
+
+    async def add_fingerprint(self, fp: str) -> bool:
+        """添加新指纹到Redis集合"""
+        try:
+            fp = str(fp)
+            added = await self.redis.sadd(self.redis_key, fp)
+
+            if self.ttl and self.ttl > 0:
+                await self.redis.expire(self.redis_key, self.ttl)
+
+            return added == 1
+        except Exception as e:
+            self.logger.error("添加指纹失败")
+            raise
+
+    async def get_stats(self) -> dict:
+        """获取过滤器统计信息"""
+        try:
+            count = await self.redis.scard(self.redis_key)
+            stats = {
+                '指纹总数': count,
+                'Redis键名': self.redis_key,
+                'TTL配置': f"{self.ttl}秒" if self.ttl else "持久化"
+            }
+            stats.update(self.stats)
+            return stats
+        except Exception as e:
+            self.logger.error("获取统计信息失败")
+            return self.stats
+
+    async def clear_all(self) -> int:
+        """清空所有指纹数据"""
+        try:
+            deleted = await self.redis.delete(self.redis_key)
+            self.logger.info(f"已清除指纹数: {deleted}")
+            return deleted
+        except Exception as e:
+            self.logger.error("清空指纹失败")
+            raise
+
+    async def closed(self, reason: Optional[str] = None) -> None:
+        """爬虫关闭时的清理操作"""
+        try:
+            if self.cleanup_fp:
+                deleted = await self.redis.delete(self.redis_key)
+                self.logger.info(f"爬虫关闭清理: 已删除{deleted}个指纹")
+            else:
+                count = await self.redis.scard(self.redis_key)
+                ttl_info = f"{self.ttl}秒" if self.ttl else "持久化"
+                self.logger.info(f"保留指纹数: {count} (TTL: {ttl_info})")
+        finally:
+            await self._close_redis()
+
+    async def _close_redis(self) -> None:
+        """安全关闭Redis连接"""
+        try:
+            if hasattr(self.redis, 'close'):
+                await self.redis.close()
+                self.logger.debug("Redis连接已关闭")
+        except Exception as e:
+            self.logger.warning(f"Redis关闭时出错：{e}")