crawlo 1.4.5__py3-none-any.whl → 1.4.7__py3-none-any.whl

tests/log_generation_timing_test.py

@@ -1,154 +1,154 @@
-#!/usr/bin/env python
-# -*- coding: UTF-8 -*-
-"""
-测试日志文件的生成时机
-"""
-
-import os
-import sys
-import time
-from pathlib import Path
-
-# 添加项目根目录到Python路径
-project_root = Path(__file__).parent.parent
-sys.path.insert(0, str(project_root))
-
-from crawlo.logging import configure_logging as configure, get_logger
-
-
-def test_log_file_generation_timing():
-    """测试日志文件的生成时机"""
-    print("=== 测试日志文件的生成时机 ===")
-    
-    # 设置日志文件路径
-    log_file = "logs/timing_test.log"
-    
-    # 确保日志目录存在
-    os.makedirs("logs", exist_ok=True)
-    
-    # 删除可能存在的旧日志文件
-    if os.path.exists(log_file):
-        os.remove(log_file)
-        print(f"已删除旧日志文件: {log_file}")
-    
-    print(f"日志文件路径: {log_file}")
-    print(f"日志文件是否存在: {os.path.exists(log_file)}")
-    
-    # 配置日志系统
-    print("\n1. 配置日志系统...")
-    configure(
-        level='INFO',
-        file_path=log_file,
-        console_enabled=True,
-        file_enabled=True
-    )
-    
-    print(f"配置后日志文件是否存在: {os.path.exists(log_file)}")
-    
-    # 获取logger
-    print("\n2. 获取logger...")
-    logger = get_logger('timing.test')
-    print(f"获取logger后日志文件是否存在: {os.path.exists(log_file)}")
-    
-    # 立即写入日志
-    print("\n3. 立即写入日志...")
-    logger.info("立即写入的第一条日志")
-    print(f"写入第一条日志后日志文件是否存在: {os.path.exists(log_file)}")
-    
-    # 检查文件内容
-    if os.path.exists(log_file):
-        with open(log_file, 'r', encoding='utf-8') as f:
-            content = f.read()
-            print(f"日志文件内容行数: {len(content.splitlines())}")
-            print(f"日志文件大小: {os.path.getsize(log_file)} 字节")
-    
-    # 等待一小段时间
-    print("\n4. 等待1秒...")
-    time.sleep(1)
-    print(f"等待后日志文件是否存在: {os.path.exists(log_file)}")
-    
-    # 再写入一些日志
-    print("\n5. 再写入一些日志...")
-    for i in range(5):
-        logger.info(f"第{i+1}条测试日志")
-        time.sleep(0.1)
-    
-    print(f"写入多条日志后日志文件是否存在: {os.path.exists(log_file)}")
-    
-    # 检查最终文件内容
-    if os.path.exists(log_file):
-        with open(log_file, 'r', encoding='utf-8') as f:
-            lines = f.readlines()
-            print(f"\n最终日志文件内容:")
-            print(f"  总行数: {len(lines)}")
-            print(f"  文件大小: {os.path.getsize(log_file)} 字节")
-            if lines:
-                print(f"  第一行: {lines[0].strip()}")
-                print(f"  最后一行: {lines[-1].strip()}")
-    
-    print("\n=== 测试完成 ===")
-
-
-def test_buffering_behavior():
-    """测试日志缓冲行为"""
-    print("\n=== 测试日志缓冲行为 ===")
-    
-    # 设置日志文件路径
-    log_file = "logs/buffering_test.log"
-    
-    # 删除可能存在的旧日志文件
-    if os.path.exists(log_file):
-        os.remove(log_file)
-    
-    # 配置日志系统
-    configure(
-        level='INFO',
-        file_path=log_file,
-        console_enabled=True,
-        file_enabled=True
-    )
-    
-    logger = get_logger('buffering.test')
-    
-    print("1. 写入日志后立即检查文件...")
-    logger.info("缓冲测试日志1")
-    print(f"   写入后文件是否存在: {os.path.exists(log_file)}")
-    
-    if os.path.exists(log_file):
-        with open(log_file, 'r', encoding='utf-8') as f:
-            content = f.read()
-            print(f"   文件内容: '{content.strip()}'")
-    
-    print("2. 强制刷新并检查...")
-    # 获取文件处理器并强制刷新
-    for handler in logger.handlers:
-        if hasattr(handler, 'flush'):
-            handler.flush()
-    
-    if os.path.exists(log_file):
-        with open(log_file, 'r', encoding='utf-8') as f:
-            content = f.read()
-            print(f"   刷新后文件内容: '{content.strip()}'")
-
-
-def main():
-    """主函数"""
-    print("开始测试日志文件的生成时机...")
-    
-    try:
-        test_log_file_generation_timing()
-        test_buffering_behavior()
-        
-        print("\n=== 所有测试完成 ===")
-        
-    except Exception as e:
-        print(f"\n测试过程中出现错误: {e}")
-        import traceback
-        traceback.print_exc()
-        return 1
-    
-    return 0
-
-
-if __name__ == '__main__':
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+"""
+测试日志文件的生成时机
+"""
+
+import os
+import sys
+import time
+from pathlib import Path
+
+# 添加项目根目录到Python路径
+project_root = Path(__file__).parent.parent
+sys.path.insert(0, str(project_root))
+
+from crawlo.logging import configure_logging as configure, get_logger
+
+
+def test_log_file_generation_timing():
+    """测试日志文件的生成时机"""
+    print("=== 测试日志文件的生成时机 ===")
+    
+    # 设置日志文件路径
+    log_file = "logs/timing_test.log"
+    
+    # 确保日志目录存在
+    os.makedirs("logs", exist_ok=True)
+    
+    # 删除可能存在的旧日志文件
+    if os.path.exists(log_file):
+        os.remove(log_file)
+        print(f"已删除旧日志文件: {log_file}")
+    
+    print(f"日志文件路径: {log_file}")
+    print(f"日志文件是否存在: {os.path.exists(log_file)}")
+    
+    # 配置日志系统
+    print("\n1. 配置日志系统...")
+    configure(
+        level='INFO',
+        file_path=log_file,
+        console_enabled=True,
+        file_enabled=True
+    )
+    
+    print(f"配置后日志文件是否存在: {os.path.exists(log_file)}")
+    
+    # 获取logger
+    print("\n2. 获取logger...")
+    logger = get_logger('timing.test')
+    print(f"获取logger后日志文件是否存在: {os.path.exists(log_file)}")
+    
+    # 立即写入日志
+    print("\n3. 立即写入日志...")
+    logger.info("立即写入的第一条日志")
+    print(f"写入第一条日志后日志文件是否存在: {os.path.exists(log_file)}")
+    
+    # 检查文件内容
+    if os.path.exists(log_file):
+        with open(log_file, 'r', encoding='utf-8') as f:
+            content = f.read()
+            print(f"日志文件内容行数: {len(content.splitlines())}")
+            print(f"日志文件大小: {os.path.getsize(log_file)} 字节")
+    
+    # 等待一小段时间
+    print("\n4. 等待1秒...")
+    time.sleep(1)
+    print(f"等待后日志文件是否存在: {os.path.exists(log_file)}")
+    
+    # 再写入一些日志
+    print("\n5. 再写入一些日志...")
+    for i in range(5):
+        logger.info(f"第{i+1}条测试日志")
+        time.sleep(0.1)
+    
+    print(f"写入多条日志后日志文件是否存在: {os.path.exists(log_file)}")
+    
+    # 检查最终文件内容
+    if os.path.exists(log_file):
+        with open(log_file, 'r', encoding='utf-8') as f:
+            lines = f.readlines()
+            print(f"\n最终日志文件内容:")
+            print(f"  总行数: {len(lines)}")
+            print(f"  文件大小: {os.path.getsize(log_file)} 字节")
+            if lines:
+                print(f"  第一行: {lines[0].strip()}")
+                print(f"  最后一行: {lines[-1].strip()}")
+    
+    print("\n=== 测试完成 ===")
+
+
+def test_buffering_behavior():
+    """测试日志缓冲行为"""
+    print("\n=== 测试日志缓冲行为 ===")
+    
+    # 设置日志文件路径
+    log_file = "logs/buffering_test.log"
+    
+    # 删除可能存在的旧日志文件
+    if os.path.exists(log_file):
+        os.remove(log_file)
+    
+    # 配置日志系统
+    configure(
+        level='INFO',
+        file_path=log_file,
+        console_enabled=True,
+        file_enabled=True
+    )
+    
+    logger = get_logger('buffering.test')
+    
+    print("1. 写入日志后立即检查文件...")
+    logger.info("缓冲测试日志1")
+    print(f"   写入后文件是否存在: {os.path.exists(log_file)}")
+    
+    if os.path.exists(log_file):
+        with open(log_file, 'r', encoding='utf-8') as f:
+            content = f.read()
+            print(f"   文件内容: '{content.strip()}'")
+    
+    print("2. 强制刷新并检查...")
+    # 获取文件处理器并强制刷新
+    for handler in logger.handlers:
+        if hasattr(handler, 'flush'):
+            handler.flush()
+    
+    if os.path.exists(log_file):
+        with open(log_file, 'r', encoding='utf-8') as f:
+            content = f.read()
+            print(f"   刷新后文件内容: '{content.strip()}'")
+
+
+def main():
+    """主函数"""
+    print("开始测试日志文件的生成时机...")
+    
+    try:
+        test_log_file_generation_timing()
+        test_buffering_behavior()
+        
+        print("\n=== 所有测试完成 ===")
+        
+    except Exception as e:
+        print(f"\n测试过程中出现错误: {e}")
+        import traceback
+        traceback.print_exc()
+        return 1
+    
+    return 0
+
+
+if __name__ == '__main__':
     sys.exit(main())

tests/monitor_redis_dedup.sh

@@ -0,0 +1,72 @@
+#!/bin/bash
+# Redis 去重监控脚本
+# 实时查看分布式去重的 Redis 操作
+
+echo "========================================"
+echo "Redis 去重实时监控"
+echo "========================================"
+echo ""
+echo "按 Ctrl+C 停止监控"
+echo ""
+
+# Redis 配置
+REDIS_HOST="127.0.0.1"
+REDIS_PORT="6379"
+REDIS_DB="15"
+
+# 项目名称
+PROJECT_NAME="dedup_test"
+
+# 清屏并显示标题
+clear
+
+while true; do
+    # 移动光标到顶部
+    tput cup 0 0
+    
+    echo "========================================"
+    echo "Redis 去重实时监控 (DB: $REDIS_DB)"
+    echo "时间: $(date '+%Y-%m-%d %H:%M:%S')"
+    echo "========================================"
+    echo ""
+    
+    # 获取过滤器指纹数量
+    FILTER_COUNT=$(redis-cli -h $REDIS_HOST -p $REDIS_PORT -n $REDIS_DB \
+        SCARD "crawlo:${PROJECT_NAME}:filter:fingerprint" 2>/dev/null || echo "0")
+    
+    # 获取数据项指纹数量
+    ITEM_COUNT=$(redis-cli -h $REDIS_HOST -p $REDIS_PORT -n $REDIS_DB \
+        SCARD "crawlo:${PROJECT_NAME}:item:fingerprint" 2>/dev/null || echo "0")
+    
+    # 获取队列长度
+    QUEUE_LEN=$(redis-cli -h $REDIS_HOST -p $REDIS_PORT -n $REDIS_DB \
+        ZCARD "crawlo:${PROJECT_NAME}:queue:requests" 2>/dev/null || echo "0")
+    
+    echo "📊 实时统计:"
+    echo "  - URL 指纹数 (过滤器): $FILTER_COUNT"
+    echo "  - 数据项指纹数: $ITEM_COUNT"
+    echo "  - 待处理队列长度: $QUEUE_LEN"
+    echo ""
+    
+    # 显示最近的 URL 指纹（前 5 个）
+    echo "🔍 最近的 URL 指纹:"
+    redis-cli -h $REDIS_HOST -p $REDIS_PORT -n $REDIS_DB \
+        SRANDMEMBER "crawlo:${PROJECT_NAME}:filter:fingerprint" 5 2>/dev/null | \
+        head -5 | sed 's/^/  - /'
+    echo ""
+    
+    # 显示所有相关的 key
+    echo "🔑 Redis Keys:"
+    redis-cli -h $REDIS_HOST -p $REDIS_PORT -n $REDIS_DB \
+        KEYS "crawlo:${PROJECT_NAME}:*" 2>/dev/null | \
+        sed 's/^/  - /'
+    echo ""
+    
+    echo "========================================"
+    echo "提示: 在另一个终端运行测试脚本"
+    echo "  python tests/distributed_dedup_test.py"
+    echo "========================================"
+    
+    # 每秒更新一次
+    sleep 1
+done

tests/ofweek_scrapy/ofweek_scrapy/items.py

@@ -1,12 +1,12 @@
-# Define here the models for your scraped items
-#
-# See documentation in:
-# https://docs.scrapy.org/en/latest/topics/items.html
-
-import scrapy
-
-
-class OfweekScrapyItem(scrapy.Item):
-    # define the fields for your item here like:
-    # name = scrapy.Field()
-    pass
+# Define here the models for your scraped items
+#
+# See documentation in:
+# https://docs.scrapy.org/en/latest/topics/items.html
+
+import scrapy
+
+
+class OfweekScrapyItem(scrapy.Item):
+    # define the fields for your item here like:
+    # name = scrapy.Field()
+    pass

tests/ofweek_scrapy/ofweek_scrapy/middlewares.py

@@ -1,100 +1,100 @@
-# Define here the models for your spider middleware
-#
-# See documentation in:
-# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
-
-from scrapy import signals
-
-# useful for handling different item types with a single interface
-from itemadapter import ItemAdapter
-
-
-class OfweekScrapySpiderMiddleware:
-    # Not all methods need to be defined. If a method is not defined,
-    # scrapy acts as if the spider middleware does not modify the
-    # passed objects.
-
-    @classmethod
-    def from_crawler(cls, crawler):
-        # This method is used by Scrapy to create your spiders.
-        s = cls()
-        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
-        return s
-
-    def process_spider_input(self, response, spider):
-        # Called for each response that goes through the spider
-        # middleware and into the spider.
-
-        # Should return None or raise an exception.
-        return None
-
-    def process_spider_output(self, response, result, spider):
-        # Called with the results returned from the Spider, after
-        # it has processed the response.
-
-        # Must return an iterable of Request, or item objects.
-        for i in result:
-            yield i
-
-    def process_spider_exception(self, response, exception, spider):
-        # Called when a spider or process_spider_input() method
-        # (from other spider middleware) raises an exception.
-
-        # Should return either None or an iterable of Request or item objects.
-        pass
-
-    async def process_start(self, start):
-        # Called with an async iterator over the spider start() method or the
-        # maching method of an earlier spider middleware.
-        async for item_or_request in start:
-            yield item_or_request
-
-    def spider_opened(self, spider):
-        spider.logger.info("Spider opened: %s" % spider.name)
-
-
-class OfweekScrapyDownloaderMiddleware:
-    # Not all methods need to be defined. If a method is not defined,
-    # scrapy acts as if the downloader middleware does not modify the
-    # passed objects.
-
-    @classmethod
-    def from_crawler(cls, crawler):
-        # This method is used by Scrapy to create your spiders.
-        s = cls()
-        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
-        return s
-
-    def process_request(self, request, spider):
-        # Called for each request that goes through the downloader
-        # middleware.
-
-        # Must either:
-        # - return None: continue processing this request
-        # - or return a Response object
-        # - or return a Request object
-        # - or raise IgnoreRequest: process_exception() methods of
-        #   installed downloader middleware will be called
-        return None
-
-    def process_response(self, request, response, spider):
-        # Called with the response returned from the downloader.
-
-        # Must either;
-        # - return a Response object
-        # - return a Request object
-        # - or raise IgnoreRequest
-        return response
-
-    def process_exception(self, request, exception, spider):
-        # Called when a download handler or a process_request()
-        # (from other downloader middleware) raises an exception.
-
-        # Must either:
-        # - return None: continue processing this exception
-        # - return a Response object: stops process_exception() chain
-        # - return a Request object: stops process_exception() chain
-        pass
-
-    def spider_opened(self, spider):
-        spider.logger.info("Spider opened: %s" % spider.name)
+# Define here the models for your spider middleware
+#
+# See documentation in:
+# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
+
+from scrapy import signals
+
+# useful for handling different item types with a single interface
+from itemadapter import ItemAdapter
+
+
+class OfweekScrapySpiderMiddleware:
+    # Not all methods need to be defined. If a method is not defined,
+    # scrapy acts as if the spider middleware does not modify the
+    # passed objects.
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        # This method is used by Scrapy to create your spiders.
+        s = cls()
+        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
+        return s
+
+    def process_spider_input(self, response, spider):
+        # Called for each response that goes through the spider
+        # middleware and into the spider.
+
+        # Should return None or raise an exception.
+        return None
+
+    def process_spider_output(self, response, result, spider):
+        # Called with the results returned from the Spider, after
+        # it has processed the response.
+
+        # Must return an iterable of Request, or item objects.
+        for i in result:
+            yield i
+
+    def process_spider_exception(self, response, exception, spider):
+        # Called when a spider or process_spider_input() method
+        # (from other spider middleware) raises an exception.
+
+        # Should return either None or an iterable of Request or item objects.
+        pass
+
+    async def process_start(self, start):
+        # Called with an async iterator over the spider start() method or the
+        # maching method of an earlier spider middleware.
+        async for item_or_request in start:
+            yield item_or_request
+
+    def spider_opened(self, spider):
+        spider.logger.info("Spider opened: %s" % spider.name)
+
+
+class OfweekScrapyDownloaderMiddleware:
+    # Not all methods need to be defined. If a method is not defined,
+    # scrapy acts as if the downloader middleware does not modify the
+    # passed objects.
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        # This method is used by Scrapy to create your spiders.
+        s = cls()
+        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
+        return s
+
+    def process_request(self, request, spider):
+        # Called for each request that goes through the downloader
+        # middleware.
+
+        # Must either:
+        # - return None: continue processing this request
+        # - or return a Response object
+        # - or return a Request object
+        # - or raise IgnoreRequest: process_exception() methods of
+        #   installed downloader middleware will be called
+        return None
+
+    def process_response(self, request, response, spider):
+        # Called with the response returned from the downloader.
+
+        # Must either;
+        # - return a Response object
+        # - return a Request object
+        # - or raise IgnoreRequest
+        return response
+
+    def process_exception(self, request, exception, spider):
+        # Called when a download handler or a process_request()
+        # (from other downloader middleware) raises an exception.
+
+        # Must either:
+        # - return None: continue processing this exception
+        # - return a Response object: stops process_exception() chain
+        # - return a Request object: stops process_exception() chain
+        pass
+
+    def spider_opened(self, spider):
+        spider.logger.info("Spider opened: %s" % spider.name)

tests/ofweek_scrapy/ofweek_scrapy/pipelines.py

@@ -1,13 +1,13 @@
-# Define your item pipelines here
-#
-# Don't forget to add your pipeline to the ITEM_PIPELINES setting
-# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
-
-
-# useful for handling different item types with a single interface
-from itemadapter import ItemAdapter
-
-
-class OfweekScrapyPipeline:
-    def process_item(self, item, spider):
-        return item
+# Define your item pipelines here
+#
+# Don't forget to add your pipeline to the ITEM_PIPELINES setting
+# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
+
+
+# useful for handling different item types with a single interface
+from itemadapter import ItemAdapter
+
+
+class OfweekScrapyPipeline:
+    def process_item(self, item, spider):
+        return item