PyPI - crawlo - Versions diffs - 1.1.3__py3-none-any.whl → 1.1.5__py3-none-any.whl - Mend

crawlo 1.1.3py3-none-any.whl → 1.1.5py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.

This version of crawlo might be problematic. Click here for more details.

Files changed (115) hide show

crawlo/__init__.py +28 -1
crawlo/__version__.py +1 -1
crawlo/cleaners/__init__.py +61 -0
crawlo/cleaners/data_formatter.py +226 -0
crawlo/cleaners/encoding_converter.py +126 -0
crawlo/cleaners/text_cleaner.py +233 -0
crawlo/commands/startproject.py +117 -13
crawlo/config.py +30 -0
crawlo/config_validator.py +253 -0
crawlo/core/engine.py +185 -11
crawlo/core/scheduler.py +49 -78
crawlo/crawler.py +6 -6
crawlo/downloader/__init__.py +24 -0
crawlo/downloader/aiohttp_downloader.py +8 -0
crawlo/downloader/cffi_downloader.py +5 -0
crawlo/downloader/hybrid_downloader.py +214 -0
crawlo/downloader/playwright_downloader.py +403 -0
crawlo/downloader/selenium_downloader.py +473 -0
crawlo/extension/__init__.py +17 -10
crawlo/extension/health_check.py +142 -0
crawlo/extension/log_interval.py +27 -18
crawlo/extension/log_stats.py +62 -24
crawlo/extension/logging_extension.py +18 -9
crawlo/extension/memory_monitor.py +105 -0
crawlo/extension/performance_profiler.py +134 -0
crawlo/extension/request_recorder.py +108 -0
crawlo/filters/aioredis_filter.py +50 -12
crawlo/middleware/proxy.py +26 -2
crawlo/mode_manager.py +24 -19
crawlo/network/request.py +30 -3
crawlo/network/response.py +114 -25
crawlo/pipelines/mongo_pipeline.py +81 -66
crawlo/pipelines/mysql_pipeline.py +165 -43
crawlo/pipelines/redis_dedup_pipeline.py +7 -3
crawlo/queue/queue_manager.py +15 -2
crawlo/queue/redis_priority_queue.py +144 -76
crawlo/settings/default_settings.py +93 -121
crawlo/subscriber.py +62 -37
crawlo/templates/project/items.py.tmpl +1 -1
crawlo/templates/project/middlewares.py.tmpl +73 -49
crawlo/templates/project/pipelines.py.tmpl +51 -295
crawlo/templates/project/settings.py.tmpl +93 -17
crawlo/templates/project/settings_distributed.py.tmpl +120 -0
crawlo/templates/project/settings_gentle.py.tmpl +95 -0
crawlo/templates/project/settings_high_performance.py.tmpl +152 -0
crawlo/templates/project/settings_simple.py.tmpl +69 -0
crawlo/templates/spider/spider.py.tmpl +2 -38
crawlo/tools/__init__.py +183 -0
crawlo/tools/anti_crawler.py +269 -0
crawlo/tools/authenticated_proxy.py +241 -0
crawlo/tools/data_validator.py +181 -0
crawlo/tools/date_tools.py +36 -0
crawlo/tools/distributed_coordinator.py +387 -0
crawlo/tools/retry_mechanism.py +221 -0
crawlo/tools/scenario_adapter.py +263 -0
crawlo/utils/__init__.py +29 -1
crawlo/utils/batch_processor.py +261 -0
crawlo/utils/date_tools.py +58 -1
crawlo/utils/enhanced_error_handler.py +360 -0
crawlo/utils/env_config.py +106 -0
crawlo/utils/error_handler.py +126 -0
crawlo/utils/performance_monitor.py +285 -0
crawlo/utils/redis_connection_pool.py +335 -0
crawlo/utils/redis_key_validator.py +200 -0
crawlo-1.1.5.dist-info/METADATA +401 -0
crawlo-1.1.5.dist-info/RECORD +185 -0
tests/advanced_tools_example.py +276 -0
tests/authenticated_proxy_example.py +237 -0
tests/cleaners_example.py +161 -0
tests/config_validation_demo.py +103 -0
tests/date_tools_example.py +181 -0
tests/dynamic_loading_example.py +524 -0
tests/dynamic_loading_test.py +105 -0
tests/env_config_example.py +134 -0
tests/error_handling_example.py +172 -0
tests/redis_key_validation_demo.py +131 -0
tests/response_improvements_example.py +145 -0
tests/test_advanced_tools.py +149 -0
tests/test_all_redis_key_configs.py +146 -0
tests/test_authenticated_proxy.py +142 -0
tests/test_cleaners.py +55 -0
tests/test_comprehensive.py +147 -0
tests/test_config_validator.py +194 -0
tests/test_date_tools.py +124 -0
tests/test_dynamic_downloaders_proxy.py +125 -0
tests/test_dynamic_proxy.py +93 -0
tests/test_dynamic_proxy_config.py +147 -0
tests/test_dynamic_proxy_real.py +110 -0
tests/test_edge_cases.py +304 -0
tests/test_enhanced_error_handler.py +271 -0
tests/test_env_config.py +122 -0
tests/test_error_handler_compatibility.py +113 -0
tests/test_framework_env_usage.py +104 -0
tests/test_integration.py +357 -0
tests/test_item_dedup_redis_key.py +123 -0
tests/test_parsel.py +30 -0
tests/test_performance.py +328 -0
tests/test_queue_manager_redis_key.py +177 -0
tests/test_redis_connection_pool.py +295 -0
tests/test_redis_key_naming.py +182 -0
tests/test_redis_key_validator.py +124 -0
tests/test_response_improvements.py +153 -0
tests/test_simple_response.py +62 -0
tests/test_telecom_spider_redis_key.py +206 -0
tests/test_template_content.py +88 -0
tests/test_template_redis_key.py +135 -0
tests/test_tools.py +154 -0
tests/tools_example.py +258 -0
crawlo/core/enhanced_engine.py +0 -190
crawlo-1.1.3.dist-info/METADATA +0 -635
crawlo-1.1.3.dist-info/RECORD +0 -113
{crawlo-1.1.3.dist-info → crawlo-1.1.5.dist-info}/WHEEL +0 -0
{crawlo-1.1.3.dist-info → crawlo-1.1.5.dist-info}/entry_points.txt +0 -0
{crawlo-1.1.3.dist-info → crawlo-1.1.5.dist-info}/top_level.txt +0 -0
{examples → tests}/controlled_spider_example.py +0 -0

tests/test_response_improvements.py ADDED Viewed

@@ -0,0 +1,153 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+Response 改进功能测试
+"""
+import unittest
+from crawlo.network.response import Response
+class TestResponseImprovements(unittest.TestCase):
+    """Response 改进功能测试类"""
+    def setUp(self):
+        """测试前准备"""
+        # 创建一个模拟的HTML响应
+        html_content = """
+        <html>
+        <head>
+            <title>测试页面</title>
+        </head>
+        <body>
+            <div class="content">
+                <h1>主标题</h1>
+                <p class="intro">这是介绍段落</p>
+                <ul class="list">
+                    <li>项目1</li>
+                    <li>项目2</li>
+                    <li>项目3</li>
+                </ul>
+                <a href="https://example.com" class="link">链接文本</a>
+                <img src="image.jpg" alt="图片描述" class="image">
+            </div>
+        </body>
+        </html>
+        """
+        self.response = Response(
+            url="https://example.com/test",
+            body=html_content.encode('utf-8'),
+            headers={"content-type": "text/html; charset=utf-8"}
+        )
+    def test_extract_text_with_css(self):
+        """测试使用CSS选择器提取文本"""
+        # 测试提取单个元素文本
+        title = self.response.extract_text('title')
+        self.assertEqual(title, "测试页面")
+        # 测试提取class元素文本
+        h1_text = self.response.extract_text('.content h1')
+        self.assertEqual(h1_text, "主标题")
+        # 测试提取带有默认值的情况
+        non_exist = self.response.extract_text('.non-exist', default='默认值')
+        self.assertEqual(non_exist, '默认值')
+    def test_extract_text_with_xpath(self):
+        """测试使用XPath选择器提取文本"""
+        # 测试提取单个元素文本
+        title = self.response.extract_text('//title')
+        self.assertEqual(title, "测试页面")
+        # 测试提取class元素文本
+        h1_text = self.response.extract_text('//div[@class="content"]/h1')
+        self.assertEqual(h1_text, "主标题")
+    def test_extract_texts_with_css(self):
+        """测试使用CSS选择器提取多个文本"""
+        # 测试提取多个li元素的文本
+        list_items = self.response.extract_texts('.list li')
+        expected = ["项目1", "项目2", "项目3"]
+        self.assertEqual(list_items, expected)
+        # 测试提取不存在元素的默认值
+        non_exist = self.response.extract_texts('.non-exist', default=['默认值'])
+        self.assertEqual(non_exist, ['默认值'])
+    def test_extract_texts_with_xpath(self):
+        """测试使用XPath选择器提取多个文本"""
+        # 测试提取多个li元素的文本
+        list_items = self.response.extract_texts('//ul[@class="list"]/li')
+        expected = ["项目1", "项目2", "项目3"]
+        self.assertEqual(list_items, expected)
+    def test_extract_attr(self):
+        """测试提取元素属性"""
+        # 测试提取链接的href属性
+        link_href = self.response.extract_attr('.link', 'href')
+        self.assertEqual(link_href, "https://example.com")
+        # 测试提取图片的alt属性
+        img_alt = self.response.extract_attr('.image', 'alt')
+        self.assertEqual(img_alt, "图片描述")
+        # 测试提取不存在属性的默认值
+        non_exist = self.response.extract_attr('.link', 'non-exist', default='默认值')
+        self.assertEqual(non_exist, '默认值')
+    def test_extract_attrs(self):
+        """测试提取多个元素的属性"""
+        # 测试提取所有li元素的属性（这里我们测试class属性）
+        list_classes = self.response.extract_attrs('.list li', 'class')
+        # 注意：在当前HTML中li元素没有class属性，所以应该返回空列表
+        self.assertEqual(list_classes, [])
+        # 测试提取所有图片元素的alt属性
+        img_alts = self.response.extract_attrs('.image', 'alt')
+        self.assertEqual(img_alts, ['图片描述'])
+        # 测试提取不存在元素时的默认值
+        non_exist = self.response.extract_attrs('.non-exist-elements', 'alt', default=['默认值'])
+        self.assertEqual(non_exist, ['默认值'])
+    def test_extract_text_from_elements(self):
+        """测试从复杂元素中提取文本"""
+        # 创建包含嵌套标签的HTML
+        complex_html = """
+        <div class="complex">
+            <p>段落文本 <strong>粗体文本</strong> 普通文本</p>
+            <p>第二段落 <em>斜体文本</em></p>
+        </div>
+        """
+        complex_response = Response(
+            url="https://example.com/complex",
+            body=complex_html.encode('utf-8')
+        )
+        # 测试提取复杂元素的文本
+        complex_text = complex_response.extract_text('.complex p', join_str=' ')
+        self.assertIn("段落文本", complex_text)
+        self.assertIn("粗体文本", complex_text)
+        self.assertIn("普通文本", complex_text)
+    def test_edge_cases(self):
+        """测试边界情况"""
+        # 测试空响应
+        empty_response = Response(url="https://example.com/empty", body=b"")
+        empty_text = empty_response.extract_text('title', default='默认标题')
+        self.assertEqual(empty_text, '默认标题')
+        # 测试只包含空白字符的元素
+        whitespace_html = "<div class='whitespace'>   </div>"
+        whitespace_response = Response(
+            url="https://example.com/whitespace",
+            body=whitespace_html.encode('utf-8')
+        )
+        whitespace_text = whitespace_response.extract_text('.whitespace')
+        self.assertEqual(whitespace_text, '')
+if __name__ == '__main__':
+    unittest.main()

tests/test_simple_response.py ADDED Viewed

@@ -0,0 +1,62 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+Response 简单功能测试
+"""
+import sys
+import os
+# 添加项目根目录到路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+from crawlo.network.response import Response
+def test_basic_functionality():
+    """测试基本功能"""
+    print("测试基本功能...")
+    # 创建一个简单的HTML响应
+    html_content = """
+    <html>
+    <head>
+        <title>测试页面</title>
+    </head>
+    <body>
+        <div class="content">
+            <h1>主标题</h1>
+            <p class="intro">这是介绍段落</p>
+        </div>
+    </body>
+    </html>
+    """
+    response = Response(
+        url="https://example.com/test",
+        body=html_content.encode('utf-8'),
+        headers={"content-type": "text/html; charset=utf-8"}
+    )
+    # 测试基本属性
+    print(f"URL: {response.url}")
+    print(f"状态码: {response.status_code}")
+    # 测试文本提取（使用新方法）
+    title = response.extract_text('title')
+    print(f"标题: {title}")
+    h1_text = response.extract_text('.content h1')
+    print(f"H1文本: {h1_text}")
+    intro_text = response.extract_text('.intro')
+    print(f"介绍文本: {intro_text}")
+    # 测试XPath（使用新方法）
+    title_xpath = response.extract_text('//title')
+    print(f"XPath标题: {title_xpath}")
+    print("基本功能测试完成")
+if __name__ == '__main__':
+    test_basic_functionality()

tests/test_telecom_spider_redis_key.py ADDED Viewed

@@ -0,0 +1,206 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+电信设备许可证爬虫Redis Key测试脚本
+用于验证分布式爬虫是否符合新的Redis key命名规范
+"""
+import sys
+import os
+import asyncio
+import tempfile
+import shutil
+from pathlib import Path
+# 添加项目根目录到路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+# 导入相关模块
+from crawlo.queue.queue_manager import QueueManager, QueueConfig, QueueType
+from crawlo.filters.aioredis_filter import AioRedisFilter
+from crawlo.pipelines.redis_dedup_pipeline import RedisDedupPipeline
+class MockSettings:
+    """模拟设置类"""
+    def __init__(self, project_name="telecom_licenses_distributed"):
+        self.project_name = project_name
+        self.REDIS_HOST = '127.0.0.1'
+        self.REDIS_PORT = 6379
+        self.REDIS_PASSWORD = ''
+        self.REDIS_DB = 2
+        self.REDIS_URL = f'redis://127.0.0.1:6379/{self.REDIS_DB}'
+        self.REDIS_TTL = 0
+        self.CLEANUP_FP = 0
+        self.FILTER_DEBUG = True
+        self.LOG_LEVEL = "INFO"
+        self.DECODE_RESPONSES = True
+        self.SCHEDULER_QUEUE_NAME = f'crawlo:{project_name}:queue:requests'
+    def get(self, key, default=None):
+        if key == 'PROJECT_NAME':
+            return self.project_name
+        elif key == 'REDIS_HOST':
+            return self.REDIS_HOST
+        elif key == 'REDIS_PASSWORD':
+            return self.REDIS_PASSWORD
+        elif key == 'REDIS_URL':
+            return self.REDIS_URL
+        elif key == 'FILTER_DEBUG':
+            return self.FILTER_DEBUG
+        elif key == 'LOG_LEVEL':
+            return self.LOG_LEVEL
+        elif key == 'DECODE_RESPONSES':
+            return self.DECODE_RESPONSES
+        elif key == 'SCHEDULER_QUEUE_NAME':
+            return self.SCHEDULER_QUEUE_NAME
+        return default
+    def get_bool(self, key, default=False):
+        if key == 'FILTER_DEBUG':
+            return self.FILTER_DEBUG
+        elif key == 'DECODE_RESPONSES':
+            return self.DECODE_RESPONSES
+        elif key == 'CLEANUP_FP':
+            return self.CLEANUP_FP
+        return default
+    def get_int(self, key, default=0):  # 修复方法名
+        if key == 'REDIS_TTL':
+            return self.REDIS_TTL
+        elif key == 'REDIS_PORT':
+            return self.REDIS_PORT
+        elif key == 'REDIS_DB':
+            return self.REDIS_DB
+        elif key == 'SCHEDULER_MAX_QUEUE_SIZE':
+            return 1000
+        elif key == 'QUEUE_MAX_RETRIES':
+            return 3
+        elif key == 'QUEUE_TIMEOUT':
+            return 300
+        return default
+class MockCrawler:
+    """模拟爬虫类"""
+    def __init__(self, project_name="telecom_licenses_distributed"):
+        self.settings = MockSettings(project_name)
+        self.stats = {}
+async def test_telecom_spider_redis_key():
+    """测试电信设备许可证爬虫Redis key命名规范"""
+    print("🔍 测试电信设备许可证爬虫Redis key命名规范...")
+    project_name = "telecom_licenses_distributed"
+    expected_prefix = f"crawlo:{project_name}"
+    try:
+        # 1. 测试QueueManager和RedisPriorityQueue
+        print("   1. 测试队列管理器...")
+        queue_config = QueueConfig(
+            queue_type=QueueType.REDIS,
+            redis_url="redis://127.0.0.1:6379/2",
+            queue_name=f"crawlo:{project_name}:queue:requests",  # 使用统一命名规范
+            max_queue_size=1000,
+            max_retries=3,
+            timeout=300
+        )
+        queue_manager = QueueManager(queue_config)
+        queue = await queue_manager._create_queue(QueueType.REDIS)
+        # 验证队列名称是否符合规范
+        expected_queue_name = f"{expected_prefix}:queue:requests"
+        expected_processing_queue = f"{expected_prefix}:queue:processing"
+        expected_failed_queue = f"{expected_prefix}:queue:failed"
+        assert queue.queue_name == expected_queue_name, f"队列名称不匹配: {queue.queue_name} != {expected_queue_name}"
+        assert queue.processing_queue == expected_processing_queue, f"处理中队列名称不匹配: {queue.processing_queue} != {expected_processing_queue}"
+        assert queue.failed_queue == expected_failed_queue, f"失败队列名称不匹配: {queue.failed_queue} != {expected_failed_queue}"
+        print(f"      ✅ 请求队列: {queue.queue_name}")
+        print(f"      ✅ 处理中队列: {queue.processing_queue}")
+        print(f"      ✅ 失败队列: {queue.failed_queue}")
+        # 2. 测试AioRedisFilter
+        print("   2. 测试请求去重过滤器...")
+        mock_crawler = MockCrawler(project_name)
+        filter_instance = AioRedisFilter.create_instance(mock_crawler)
+        expected_filter_key = f"{expected_prefix}:filter:fingerprint"
+        assert filter_instance.redis_key == expected_filter_key, f"过滤器key不匹配: {filter_instance.redis_key} != {expected_filter_key}"
+        print(f"      ✅ 请求去重key: {filter_instance.redis_key}")
+        # 3. 测试RedisDedupPipeline
+        print("   3. 测试数据项去重管道...")
+        dedup_pipeline = RedisDedupPipeline.from_crawler(mock_crawler)
+        expected_item_key = f"{expected_prefix}:item:fingerprint"
+        assert dedup_pipeline.redis_key == expected_item_key, f"数据项去重key不匹配: {dedup_pipeline.redis_key} != {expected_item_key}"
+        print(f"      ✅ 数据项去重key: {dedup_pipeline.redis_key}")
+        # 4. 验证所有key都使用统一前缀
+        print("   4. 验证统一前缀...")
+        all_keys = [
+            queue.queue_name,
+            queue.processing_queue,
+            queue.failed_queue,
+            filter_instance.redis_key,
+            dedup_pipeline.redis_key
+        ]
+        for key in all_keys:
+            assert key.startswith(expected_prefix), f"Key未使用统一前缀: {key}"
+            print(f"      ✅ {key}")
+        print("✅ 电信设备许可证爬虫Redis key命名规范测试通过！")
+        return True
+    except Exception as e:
+        print(f"❌ 测试失败: {e}")
+        import traceback
+        traceback.print_exc()
+        return False
+    finally:
+        # 清理资源
+        try:
+            if 'queue' in locals():
+                await queue.close()
+            if 'filter_instance' in locals() and hasattr(filter_instance, 'redis'):
+                await filter_instance.redis.close()
+            if 'dedup_pipeline' in locals() and hasattr(dedup_pipeline, 'redis_client'):
+                dedup_pipeline.redis_client.close()
+        except:
+            pass
+async def main():
+    """主测试函数"""
+    print("🚀 开始电信设备许可证爬虫Redis key命名规范测试...")
+    print("=" * 60)
+    try:
+        success = await test_telecom_spider_redis_key()
+        print("=" * 60)
+        if success:
+            print("🎉 所有测试通过！电信设备许可证爬虫符合新的Redis key命名规范")
+        else:
+            print("❌ 测试失败，请检查实现")
+            return 1
+    except Exception as e:
+        print("=" * 60)
+        print(f"❌ 测试过程中发生异常: {e}")
+        import traceback
+        traceback.print_exc()
+        return 1
+    return 0
+if __name__ == "__main__":
+    exit_code = asyncio.run(main())
+    sys.exit(exit_code)

tests/test_template_content.py ADDED Viewed

@@ -0,0 +1,88 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+模板文件内容测试脚本
+用于验证模板文件是否符合新的Redis key命名规范
+"""
+import sys
+import os
+# 添加项目根目录到路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+def test_template_content():
+    """测试模板文件内容"""
+    print("🔍 测试模板文件内容...")
+    try:
+        # 检查settings.py.tmpl模板文件
+        template_file = "crawlo/templates/project/settings.py.tmpl"
+        if not os.path.exists(template_file):
+            print(f"❌ 模板文件不存在: {template_file}")
+            return False
+        with open(template_file, 'r', encoding='utf-8') as f:
+            content = f.read()
+        # 检查是否移除了旧的REDIS_KEY配置
+        old_config = "REDIS_KEY = f'{{project_name}}:fingerprint'"
+        if old_config in content:
+            print("❌ 仍然存在旧的REDIS_KEY配置")
+            return False
+        print("      ✅ 已移除旧的REDIS_KEY配置")
+        # 检查是否添加了新的注释
+        filter_comment = "# crawlo:{project_name}:filter:fingerprint (请求去重)"
+        if filter_comment not in content:
+            print("❌ 缺少请求去重的Redis key命名规范注释")
+            return False
+        print("      ✅ 包含请求去重的Redis key命名规范注释")
+        item_comment = "# crawlo:{project_name}:item:fingerprint (数据项去重)"
+        if item_comment not in content:
+            print("❌ 缺少数据项去重的Redis key命名规范注释")
+            return False
+        print("      ✅ 包含数据项去重的Redis key命名规范注释")
+        # 检查是否保留了队列名称配置
+        queue_config = "SCHEDULER_QUEUE_NAME = f'crawlo:{{project_name}}:queue:requests'"
+        if queue_config not in content:
+            print("❌ 缺少队列名称配置")
+            return False
+        print("      ✅ 包含队列名称配置")
+        print("✅ 模板文件内容测试通过！")
+        return True
+    except Exception as e:
+        print(f"❌ 测试过程中发生错误: {e}")
+        return False
+def main():
+    """主测试函数"""
+    print("🚀 开始模板文件内容测试...")
+    print("=" * 50)
+    try:
+        success = test_template_content()
+        print("=" * 50)
+        if success:
+            print("🎉 所有测试通过！模板文件符合新的Redis key命名规范")
+        else:
+            print("❌ 测试失败，请检查模板文件")
+            return 1
+    except Exception as e:
+        print("=" * 50)
+        print(f"❌ 测试过程中发生异常: {e}")
+        return 1
+    return 0
+if __name__ == "__main__":
+    exit_code = main()
+    sys.exit(exit_code)

tests/test_template_redis_key.py ADDED Viewed

@@ -0,0 +1,135 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+模板项目Redis Key测试脚本
+用于验证通过模板生成的项目是否符合新的Redis key命名规范
+"""
+import sys
+import os
+import tempfile
+import shutil
+import subprocess
+from pathlib import Path
+# 添加项目根目录到路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+def test_template_project_redis_key():
+    """测试模板项目Redis key命名规范"""
+    print("🔍 测试模板项目Redis key命名规范...")
+    # 创建临时目录
+    with tempfile.TemporaryDirectory() as temp_dir:
+        try:
+            # 在原始工作目录中创建项目，然后移动到临时目录
+            original_cwd = os.getcwd()
+            # 创建测试项目（在原始工作目录中）
+            print("   1. 创建测试项目...")
+            cmd_path = os.path.join(original_cwd, "crawlo", "commands", "startproject.py")
+            result = subprocess.run([
+                sys.executable, cmd_path, "test_project"
+            ], cwd=original_cwd, capture_output=True, text=True)
+            if result.returncode != 0:
+                print(f"❌ 创建项目失败: {result.stderr}")
+                return False
+            print("      ✅ 项目创建成功")
+            # 检查生成的文件
+            project_dir = Path(original_cwd) / "test_project"
+            if not project_dir.exists():
+                print("❌ 项目目录未创建")
+                return False
+            # 移动项目到临时目录
+            target_dir = Path(temp_dir) / "test_project"
+            shutil.move(str(project_dir), str(target_dir))
+            project_dir = target_dir
+            settings_file = project_dir / "test_project" / "settings.py"
+            if not settings_file.exists():
+                print("❌ settings.py文件未创建")
+                return False
+            # 读取settings.py内容
+            with open(settings_file, 'r', encoding='utf-8') as f:
+                settings_content = f.read()
+            # 检查是否移除了旧的REDIS_KEY配置
+            if "REDIS_KEY = f'{{project_name}}:fingerprint'" in settings_content:
+                print("❌ 仍然存在旧的REDIS_KEY配置")
+                return False
+            # 检查是否添加了新的注释
+            if "# crawlo:{project_name}:filter:fingerprint (请求去重)" not in settings_content:
+                print("❌ 缺少新的Redis key命名规范注释")
+                return False
+            if "# crawlo:{project_name}:item:fingerprint (数据项去重)" not in settings_content:
+                print("❌ 缺少数据项去重的Redis key命名规范注释")
+                return False
+            print("      ✅ settings.py符合新的Redis key命名规范")
+            # 检查crawlo.cfg
+            cfg_file = project_dir / "crawlo.cfg"
+            if not cfg_file.exists():
+                print("❌ crawlo.cfg文件未创建")
+                return False
+            with open(cfg_file, 'r', encoding='utf-8') as f:
+                cfg_content = f.read()
+            if "default = test_project.settings" not in cfg_content:
+                print("❌ crawlo.cfg配置不正确")
+                return False
+            print("      ✅ crawlo.cfg配置正确")
+            print("✅ 模板项目Redis key命名规范测试通过！")
+            return True
+        except Exception as e:
+            print(f"❌ 测试过程中发生错误: {e}")
+            import traceback
+            traceback.print_exc()
+            return False
+        finally:
+            # 清理创建的项目目录
+            project_dir = Path(original_cwd) / "test_project"
+            if project_dir.exists():
+                shutil.rmtree(str(project_dir), ignore_errors=True)
+            # 恢复原始工作目录
+            os.chdir(original_cwd)
+def main():
+    """主测试函数"""
+    print("🚀 开始模板项目Redis key命名规范测试...")
+    print("=" * 50)
+    try:
+        success = test_template_project_redis_key()
+        print("=" * 50)
+        if success:
+            print("🎉 所有测试通过！模板项目符合新的Redis key命名规范")
+        else:
+            print("❌ 测试失败，请检查模板文件")
+            return 1
+    except Exception as e:
+        print("=" * 50)
+        print(f"❌ 测试过程中发生异常: {e}")
+        return 1
+    return 0
+if __name__ == "__main__":
+    exit_code = main()
+    sys.exit(exit_code)

crawlo 1.1.3__py3-none-any.whl → 1.1.5__py3-none-any.whl

Potentially problematic release.

crawlo 1.1.3py3-none-any.whl → 1.1.5py3-none-any.whl