crawlo 1.0.3__py3-none-any.whl → 1.0.4__py3-none-any.whl

crawlo/__version__.py

@@ -1 +1 @@
-__version__ = "1.0.3"
+__version__ = "1.0.4"

crawlo/filters/aioredis_filter.py

@@ -9,7 +9,7 @@ from crawlo.utils.request import request_fingerprint
 
 
 class AioRedisFilter(BaseFilter):
-    """使用Redis集合实现的异步请求去重过滤器（适用于分布式爬虫）"""
+    """基于Redis集合实现的异步请求去重过滤器（支持分布式爬虫），提供TTL和清理控制"""
 
     def __init__(
             self,
@@ -18,7 +18,8 @@ class AioRedisFilter(BaseFilter):
             stats: dict,
             debug: bool,
             log_level: str,
-            cleanup_fp: bool = False
+            cleanup_fp: bool = False,
+            ttl: Optional[int] = None  # None表示持久化，>0表示过期时间(秒)
     ):
         """初始化过滤器"""
         self.logger = get_logger(self.__class__.__name__, log_level)
@@ -27,12 +28,19 @@ class AioRedisFilter(BaseFilter):
         self.redis_key = redis_key
         self.redis = client
         self.cleanup_fp = cleanup_fp
+        self.ttl = ttl
 
     @classmethod
     def create_instance(cls, crawler) -> 'BaseFilter':
         """从爬虫配置创建过滤器实例"""
         redis_url = crawler.settings.get('REDIS_URL', 'redis://localhost:6379')
-        decode_responses = crawler.settings.get_bool('DECODE_RESPONSES', False)  # 关键：改为False
+        decode_responses = crawler.settings.get_bool('DECODE_RESPONSES', False)
+        ttl_setting = crawler.settings.get_int('REDIS_TTL')
+
+        # 处理TTL设置
+        ttl = None
+        if ttl_setting is not None:
+            ttl = max(0, int(ttl_setting)) if ttl_setting > 0 else None
 
         try:
             redis_client = aioredis.from_url(
@@ -42,110 +50,93 @@ class AioRedisFilter(BaseFilter):
                 encoding='utf-8'
             )
         except Exception as e:
-            raise RuntimeError(f"Redis连接失败 {redis_url}: {str(e)}")
+            raise RuntimeError(f"Redis连接失败: {redis_url} - {str(e)}")
 
         return cls(
             redis_key=f"{crawler.settings.get('PROJECT_NAME', 'default')}:{crawler.settings.get('REDIS_KEY', 'request_fingerprints')}",
             client=redis_client,
             stats=crawler.stats,
             cleanup_fp=crawler.settings.get_bool('CLEANUP_FP', False),
+            ttl=ttl,
             debug=crawler.settings.get_bool('FILTER_DEBUG', False),
             log_level=crawler.settings.get('LOG_LEVEL', 'INFO')
         )
 
     async def requested(self, request: Request) -> bool:
-        """
-        检查请求是否重复
-        """
+        """检查请求是否已存在"""
         try:
-            fp = request_fingerprint(request)
-            self.logger.debug(f"Checking fingerprint: {fp}")
-
-            # 确保fp是字符串类型
-            if not isinstance(fp, str):
-                fp = str(fp)
-
-            # 检查Redis连接状态
-            if not self.redis:
-                raise RuntimeError("Redis client is not initialized")
+            fp = str(request_fingerprint(request))
 
-            # 检查指纹是否已存在
-            is_member = await self.redis.sismember(self.redis_key, fp)
-            self.logger.debug(f"Fingerprint {fp} exists: {is_member}")
+            # 1. 检查指纹是否存在
+            pipe = self.redis.pipeline()
+            pipe.sismember(self.redis_key, fp)  # 不单独 await
+            exists = (await pipe.execute())[0]  # 执行并获取结果
 
-            if is_member:
-                if self.debug:
-                    self.logger.debug(f"Filtered duplicate request: {fp}")
+            if exists:  # 如果已存在，返回 True
                 return True
 
-            # 添加新指纹
-            result = await self.redis.sadd(self.redis_key, fp)
+            # 2. 如果不存在，添加指纹并设置 TTL
+            pipe = self.redis.pipeline()
+            pipe.sadd(self.redis_key, fp)  # 不单独 await
+            if self.ttl and self.ttl > 0:
+                pipe.expire(self.redis_key, self.ttl)  # 不单独 await
+            await pipe.execute()  # 一次性执行所有命令
 
-            if self.debug:
-                if result == 1:
-                    self.logger.debug(f"Added new fingerprint: {fp}")
-                else:
-                    self.logger.warning(f"Failed to add fingerprint: {fp}")
-
-            return False
+            return False  # 表示是新请求
 
         except Exception as e:
-            self.logger.error(f"Filter check failed for {getattr(request, 'url', 'unknown')}: {str(e)}")
-            # 可以选择抛出异常或返回False（不过滤）
+            self.logger.error(f"请求检查失败: {getattr(request, 'url', '未知URL')}")
             raise
 
     async def add_fingerprint(self, fp: str) -> bool:
-        """向Redis集合添加新指纹"""
+        """添加新指纹到Redis集合"""
         try:
-            if not isinstance(fp, str):
-                fp = str(fp)
+            fp = str(fp)
+            added = await self.redis.sadd(self.redis_key, fp)
+
+            if self.ttl and self.ttl > 0:
+                await self.redis.expire(self.redis_key, self.ttl)
 
-            result = await self.redis.sadd(self.redis_key, fp)
-            if self.debug:
-                self.logger.debug(f"Added fingerprint {fp}, result: {result}")
-            return result == 1
+            return added == 1
         except Exception as e:
-            self.logger.error(f"Failed to add fingerprint {fp}: {str(e)}")
+            self.logger.error("添加指纹失败")
             raise
 
     async def get_stats(self) -> dict:
-        """获取当前过滤器统计信息"""
+        """获取过滤器统计信息"""
         try:
             count = await self.redis.scard(self.redis_key)
-            return {
-                'total_fingerprints': count,
-                'redis_key': self.redis_key,
-                **self.stats
+            stats = {
+                '指纹总数': count,
+                'Redis键名': self.redis_key,
+                'TTL配置': f"{self.ttl}秒" if self.ttl else "持久化"
             }
+            stats.update(self.stats)
+            return stats
         except Exception as e:
-            self.logger.error(f"Failed to get stats: {str(e)}")
+            self.logger.error("获取统计信息失败")
             return self.stats
 
     async def clear_all(self) -> int:
         """清空所有指纹数据"""
         try:
             deleted = await self.redis.delete(self.redis_key)
-            self.logger.info(f"Cleared {deleted} keys")
+            self.logger.info(f"已清除指纹数: {deleted}")
             return deleted
         except Exception as e:
-            self.logger.error(f"Failed to clear fingerprints: {str(e)}")
+            self.logger.error("清空指纹失败")
             raise
 
     async def closed(self, reason: Optional[str] = None) -> None:
-        """爬虫关闭时的处理"""
+        """爬虫关闭时的清理操作"""
         try:
             if self.cleanup_fp:
                 deleted = await self.redis.delete(self.redis_key)
-                self.logger.info(
-                    f"Cleaned {deleted} fingerprints from {self.redis_key} "
-                    f"(reason: {reason or 'manual'})"
-                )
+                self.logger.info(f"爬虫关闭清理: 已删除{deleted}个指纹")
             else:
-                # 显示统计信息
                 count = await self.redis.scard(self.redis_key)
-                self.logger.info(f"Total fingerprints preserved: {count}")
-        except Exception as e:
-            self.logger.warning(f"Close operation failed: {e}")
+                ttl_info = f"{self.ttl}秒" if self.ttl else "持久化"
+                self.logger.info(f"保留指纹数: {count} (TTL: {ttl_info})")
         finally:
             await self._close_redis()
 
@@ -154,5 +145,6 @@ class AioRedisFilter(BaseFilter):
         try:
             if hasattr(self.redis, 'close'):
                 await self.redis.close()
+                self.logger.debug("Redis连接已关闭")
         except Exception as e:
-            self.logger.warning(f"Redis close error: {e}")
+            self.logger.warning(f"Redis关闭时出错：{e}")

crawlo/settings/default_settings.py

@@ -84,7 +84,8 @@ FILTER_DEBUG = True
 FILTER_CLASS = 'crawlo.filters.memory_filter.MemoryFilter'
 
 # redis filter
-CLEANUP_FP = False
+REDIS_TTL = 0
+CLEANUP_FP = 0
 DECODE_RESPONSES = True
 REDIS_KEY = 'request_fingerprint'
 REDIS_HOST = os.getenv('REDIS_HOST', '127.0.0.1')

{crawlo-1.0.3.dist-info → crawlo-1.0.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: crawlo
-Version: 1.0.3
+Version: 1.0.4
 Summary: Crawlo 是一款基于异步IO的高性能Python爬虫框架，支持分布式抓取。
 Home-page: https://github.com/crawl-coder/Crawlo.git
 Author: crawl-coder

{crawlo-1.0.3.dist-info → crawlo-1.0.4.dist-info}/RECORD

@@ -1,5 +1,5 @@
 crawlo/__init__.py,sha256=XOWXajnhT2HVql5cycwGkQ0MS85bpQnFdM7tl0Fusik,327
-crawlo/__version__.py,sha256=2plzdEEb24FLjE2I2XyBBcJEPYWHccNL4SgtLC_6erg,22
+crawlo/__version__.py,sha256=acuR_XSJzp4OrQ5T8-Ac5gYe48mUwObuwjRmisFmZ7k,22
 crawlo/crawler.py,sha256=rqKjMLDU6qlm2D2gIhkezF5jFOCz0TgYyq-nS7MEFMU,9237
 crawlo/event.py,sha256=7-y6HNv_EIJSYQNzsj0mVK-Gg4ON3wdQeMdQjfFJPlw,313
 crawlo/exceptions.py,sha256=7dtEJBxb9yvmMJe6MQyDB0LuV9que1J_jQN4QYeyO4g,916
@@ -18,9 +18,8 @@ crawlo/extension/__init__.py,sha256=LPy9XyCu089k6L6oVENIi_imr75AEuY8QTtSJjRioiw,
 crawlo/extension/log_interval.py,sha256=S-hSoiz9GdmgHrac4vDQ52fleoBcH-kzdPUD8YRAons,1922
 crawlo/extension/log_stats.py,sha256=WeSnOoSKB8pI_xmcGdh906XnF1xwo6fgJnf_prElwwI,1742
 crawlo/filters/__init__.py,sha256=BCZl86BHiTfDGRe_b1TlNSr6pfNbMKTu0Uq0j4gX_1Q,977
-crawlo/filters/aioredis_filter.py,sha256=2KAfFohKjWU3lRKKaxNDCN4d2lvWbpmHHL5oM7jsmKs,5640
+crawlo/filters/aioredis_filter.py,sha256=MJT74BeVZTjdExKEzdrWKc7WPXFss1k-txc7E54H77E,5522
 crawlo/filters/memory_filter.py,sha256=bs2WUe7CdHiXgr344vzDqMfBv1b3RwXJMnwxpDb64Pw,6639
-crawlo/filters/redis_filter.py,sha256=W3Wam4Qdd1mZPyue3N9pYkaF72HUcXd38iHOsHHDfEg,4092
 crawlo/items/__init__.py,sha256=JUw4wZX50DidJuCMLkP41ik_wTKum2b8iDxm7EbRRds,2063
 crawlo/items/items.py,sha256=00TdAYChF5Rbbgm6a6d-GCxkx4gXP-rA-_Q7u33BuFI,3990
 crawlo/middleware/__init__.py,sha256=ldaGFNbiJnK9Fx12Vdf9fDNfzXxoETtShp5r-vodtw0,549
@@ -41,7 +40,7 @@ crawlo/pipelines/mysql_batch_pipline.py,sha256=g111iuPTRyKr0q4PHTJYIfsYAFf8CCuyY
 crawlo/pipelines/mysql_pipeline.py,sha256=ZlRWwZLewG9SBLBZ1wWNZ8yAj5xWWitb7BKRSrqEWtI,7857
 crawlo/pipelines/pipeline_manager.py,sha256=JIoX5D-oDfUT7VJrb5m355wi43SChb4nNb09z_0F4_g,2118
 crawlo/settings/__init__.py,sha256=xsukVKn_h2Hopm1Nj-bXkhbfyS62QTTvJi7fhZUwR9M,123
-crawlo/settings/default_settings.py,sha256=5_3TK65iiorS9KbpmQGy79qi0DUTwQTN9FLUSjk4eEo,2569
+crawlo/settings/default_settings.py,sha256=zNMVMo_9s1DGr1TiPzwZjSmxuD4qj_JT_oCCmkoMfjs,2579
 crawlo/settings/setting_manager.py,sha256=SxKB1aCWh4OySM_bH9cYng9I3PAmrSP-Q8XOZEWEwbI,2899
 crawlo/spider/__init__.py,sha256=pP_TChnozpHeuS87Bs-Sj31hb0R7glYN3K6BsRw4FOA,905
 crawlo/templates/item_template.tmpl,sha256=bo0cjaFOT1jMrtLjXs6z7Mhwev-s3037suD4BL2_ji4,351
@@ -69,12 +68,12 @@ tests/baidu_spider/middleware.py,sha256=I71ZMmWTiDBFq4t2zfTE7IIXCqwaaeQ1DvKGW70q
 tests/baidu_spider/pipeline.py,sha256=TUK_LnrU818UYmCn2_gKeNaTZjaj9qjrlndRLsR4wf0,1437
 tests/baidu_spider/request_fingerprints.txt,sha256=TJAuFJZZ_uvYExfruA9bEsIiArz86vxe95QoF2lbnfE,585
 tests/baidu_spider/run.py,sha256=YVe9qwn-2XBRRoZdUnwPRrWlBO5YAmKnyLRI3RpfogE,646
-tests/baidu_spider/settings.py,sha256=z8rtEwVZ0b2f_EZLZdaugAEMJBIp2nBV8xjSFtxe3vY,2737
+tests/baidu_spider/settings.py,sha256=EenFOFgupwnn7HIySKSHBgP9--qxxkiWgIi2NDltXRw,2811
 tests/baidu_spider/spiders/__init__.py,sha256=eJ_ih4GiGfwQzPILeouy1Hnc4BrPz0KNPYlLHYvrvoc,123
 tests/baidu_spider/spiders/bai_du.py,sha256=pw4WccbmBR07CuSqCgm_7x9SH63FDJS_sXSaN5Ew5Tw,1589
 tests/baidu_spider/spiders/sina.py,sha256=BKQGJiCS8aiZ2f27C99WcK90QQJwgUY-vS4fUaQSdIQ,2456
-crawlo-1.0.3.dist-info/METADATA,sha256=HjBB_eeDrEFN6TTm3lKNl_3xuMB1DLhMjI222q3cUV4,1743
-crawlo-1.0.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-crawlo-1.0.3.dist-info/entry_points.txt,sha256=GD9PBhKQN83EaxPYtz7NhcGeZeh3bdr2jWbTixOs-lw,59
-crawlo-1.0.3.dist-info/top_level.txt,sha256=bKtfejkszFTNHm7Z6aqtt0AUG8DdeNeL4AoZsg4XdZY,13
-crawlo-1.0.3.dist-info/RECORD,,
+crawlo-1.0.4.dist-info/METADATA,sha256=dzEuRJVuBVSeKTQeEvOXRhfRcyjhcZqJFlPWivAZ9UE,1743
+crawlo-1.0.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+crawlo-1.0.4.dist-info/entry_points.txt,sha256=GD9PBhKQN83EaxPYtz7NhcGeZeh3bdr2jWbTixOs-lw,59
+crawlo-1.0.4.dist-info/top_level.txt,sha256=bKtfejkszFTNHm7Z6aqtt0AUG8DdeNeL4AoZsg4XdZY,13
+crawlo-1.0.4.dist-info/RECORD,,

tests/baidu_spider/settings.py

@@ -72,7 +72,9 @@ DEFAULT_HEADERS = {
 Mongo_Params = ''
 MONGODB_DB = 'news'
 
-CLEANUP_FP = True
+REDIS_TTL = 0
+CLEANUP_FP = False
 
 FILTER_CLASS = 'crawlo.filters.aioredis_filter.AioRedisFilter'
+# FILTER_CLASS = 'crawlo.filters.redis_filter.RedisFilter'
 # FILTER_CLASS = 'crawlo.filters.memory_filter.MemoryFileFilter'

crawlo/filters/redis_filter.py

@@ -1,120 +0,0 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-import redis
-
-from crawlo import Request
-from crawlo.filters import BaseFilter
-from crawlo.utils.log import get_logger
-from crawlo.utils.request import request_fingerprint
-
-
-class RedisFilter(BaseFilter):
-    """使用Redis集合实现的同步请求去重过滤器"""
-
-    def __init__(
-            self,
-            redis_key: str,
-            client: redis.Redis,
-            stats: dict,
-            debug: bool,
-            log_level: str,
-            save_fp: bool
-    ):
-        """
-        初始化过滤器
-
-        :param redis_key: Redis存储键名
-        :param client: redis客户端实例
-        :param stats: 统计字典
-        :param debug: 是否启用调试模式
-        :param log_level: 日志级别
-        :param save_fp: 是否保留指纹数据
-        """
-        self.logger = get_logger(self.__class__.__name__, log_level)
-        super().__init__(self.logger, stats, debug)
-
-        self.redis_key = redis_key
-        self.redis = client
-        self.save_fp = save_fp
-
-    @classmethod
-    def create_instance(cls, crawler) -> 'BaseFilter':
-        """工厂方法创建实例"""
-        redis_url = crawler.settings.get('REDIS_URL', 'redis://localhost:6379')
-        decode_responses = crawler.settings.get_bool('DECODE_RESPONSES', True)
-
-        try:
-            # 添加连接池配置
-            redis_client = redis.from_url(
-                redis_url,
-                decode_responses=decode_responses,
-                socket_timeout=5,  # 超时设置
-                socket_connect_timeout=5,
-                max_connections=20  # 连接池大小
-            )
-            # 测试连接是否有效
-            redis_client.ping()
-        except redis.RedisError as e:
-            raise RuntimeError(f"Redis连接失败: {str(e)}")
-
-        return cls(
-            redis_key=f"{crawler.settings.get('PROJECT_NAME')}:{crawler.settings.get('REDIS_KEY', 'request_fingerprints')}",
-            client=redis_client,
-            stats=crawler.stats,
-            save_fp=crawler.settings.get_bool('SAVE_FP', False),
-            debug=crawler.settings.get_bool('FILTER_DEBUG', False),
-            log_level=crawler.settings.get('LOG_LEVEL', 'INFO')
-        )
-
-    def requested(self, request: Request) -> bool:
-        """
-        检查请求是否已存在
-
-        :param request: 请求对象
-        :return: 是否重复
-        """
-        fp = request_fingerprint(request)
-        try:
-            if self.redis.sismember(self.redis_key, fp):
-                self.logger.debug(f"重复请求: {fp}")
-                return True
-
-            self.add_fingerprint(fp)
-            return False
-        except redis.RedisError as e:
-            self.logger.error(f"Redis操作失败: {str(e)}")
-            raise
-
-    def add_fingerprint(self, fp: str) -> None:
-        """添加指纹到Redis集合"""
-        try:
-            self.redis.sadd(self.redis_key, fp)
-            self.logger.debug(f"新增指纹: {fp}")
-        except redis.RedisError as e:
-            self.logger.error(f"指纹添加失败: {str(e)}")
-            raise
-
-    def __contains__(self, item) -> bool:
-        """支持 in 操作符检查 (必须返回bool类型)"""
-        try:
-            # 显式将redis返回的0/1转换为bool
-            return bool(self.redis.sismember(self.redis_key, item))
-        except redis.RedisError as e:
-            self.logger.error(f"Redis查询失败: {str(e)}")
-            raise
-
-    def close(self) -> None:
-        """同步清理方法（注意不是异步的closed）"""
-        if not self.save_fp:
-            try:
-                count = self.redis.delete(self.redis_key)
-                self.logger.info(f"已清理Redis键 {self.redis_key}, 删除数量: {count}")
-            except redis.RedisError as e:
-                self.logger.error(f"清理失败: {str(e)}")
-            finally:
-                # 同步客户端需要手动关闭连接池
-                self.redis.close()
-
-    async def closed(self):
-        """兼容异步接口的同步实现"""
-        self.close()