PyPI - crawlo - Versions diffs - 1.1.4__py3-none-any.whl → 1.1.6__py3-none-any.whl - Mend

crawlo 1.1.4py3-none-any.whl → 1.1.6py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.

This version of crawlo might be problematic. Click here for more details.

Files changed (190) hide show

crawlo/__init__.py +61 -34
crawlo/__version__.py +1 -1
crawlo/cleaners/__init__.py +61 -0
crawlo/cleaners/data_formatter.py +226 -0
crawlo/cleaners/encoding_converter.py +126 -0
crawlo/cleaners/text_cleaner.py +233 -0
crawlo/cli.py +40 -40
crawlo/commands/__init__.py +13 -13
crawlo/commands/check.py +594 -594
crawlo/commands/genspider.py +151 -151
crawlo/commands/list.py +155 -155
crawlo/commands/run.py +292 -285
crawlo/commands/startproject.py +419 -196
crawlo/commands/stats.py +188 -188
crawlo/commands/utils.py +186 -186
crawlo/config.py +312 -279
crawlo/config_validator.py +253 -0
crawlo/core/__init__.py +2 -2
crawlo/core/engine.py +346 -172
crawlo/core/processor.py +40 -40
crawlo/core/scheduler.py +137 -166
crawlo/crawler.py +1027 -1027
crawlo/downloader/__init__.py +266 -242
crawlo/downloader/aiohttp_downloader.py +220 -212
crawlo/downloader/cffi_downloader.py +256 -251
crawlo/downloader/httpx_downloader.py +259 -259
crawlo/downloader/hybrid_downloader.py +214 -0
crawlo/downloader/playwright_downloader.py +403 -0
crawlo/downloader/selenium_downloader.py +473 -0
crawlo/event.py +11 -11
crawlo/exceptions.py +81 -81
crawlo/extension/__init__.py +37 -37
crawlo/extension/health_check.py +141 -141
crawlo/extension/log_interval.py +57 -57
crawlo/extension/log_stats.py +81 -81
crawlo/extension/logging_extension.py +43 -43
crawlo/extension/memory_monitor.py +104 -88
crawlo/extension/performance_profiler.py +133 -117
crawlo/extension/request_recorder.py +107 -107
crawlo/filters/__init__.py +154 -154
crawlo/filters/aioredis_filter.py +281 -242
crawlo/filters/memory_filter.py +269 -269
crawlo/items/__init__.py +23 -23
crawlo/items/base.py +21 -21
crawlo/items/fields.py +53 -53
crawlo/items/items.py +104 -104
crawlo/middleware/__init__.py +21 -21
crawlo/middleware/default_header.py +32 -32
crawlo/middleware/download_delay.py +28 -28
crawlo/middleware/middleware_manager.py +135 -135
crawlo/middleware/proxy.py +272 -248
crawlo/middleware/request_ignore.py +30 -30
crawlo/middleware/response_code.py +18 -18
crawlo/middleware/response_filter.py +26 -26
crawlo/middleware/retry.py +124 -124
crawlo/mode_manager.py +212 -201
crawlo/network/__init__.py +21 -21
crawlo/network/request.py +338 -311
crawlo/network/response.py +360 -271
crawlo/pipelines/__init__.py +21 -21
crawlo/pipelines/bloom_dedup_pipeline.py +156 -156
crawlo/pipelines/console_pipeline.py +39 -39
crawlo/pipelines/csv_pipeline.py +316 -316
crawlo/pipelines/database_dedup_pipeline.py +224 -224
crawlo/pipelines/json_pipeline.py +218 -218
crawlo/pipelines/memory_dedup_pipeline.py +115 -115
crawlo/pipelines/mongo_pipeline.py +131 -131
crawlo/pipelines/mysql_pipeline.py +316 -316
crawlo/pipelines/pipeline_manager.py +61 -56
crawlo/pipelines/redis_dedup_pipeline.py +167 -162
crawlo/project.py +188 -153
crawlo/queue/pqueue.py +37 -37
crawlo/queue/queue_manager.py +334 -307
crawlo/queue/redis_priority_queue.py +299 -209
crawlo/settings/__init__.py +7 -7
crawlo/settings/default_settings.py +219 -278
crawlo/settings/setting_manager.py +123 -100
crawlo/spider/__init__.py +639 -639
crawlo/stats_collector.py +59 -59
crawlo/subscriber.py +130 -130
crawlo/task_manager.py +30 -30
crawlo/templates/crawlo.cfg.tmpl +10 -10
crawlo/templates/project/__init__.py.tmpl +3 -3
crawlo/templates/project/items.py.tmpl +17 -17
crawlo/templates/project/middlewares.py.tmpl +110 -110
crawlo/templates/project/pipelines.py.tmpl +97 -97
crawlo/templates/project/run.py.tmpl +251 -251
crawlo/templates/project/settings.py.tmpl +326 -279
crawlo/templates/project/settings_distributed.py.tmpl +120 -0
crawlo/templates/project/settings_gentle.py.tmpl +95 -0
crawlo/templates/project/settings_high_performance.py.tmpl +152 -0
crawlo/templates/project/settings_simple.py.tmpl +69 -0
crawlo/templates/project/spiders/__init__.py.tmpl +5 -5
crawlo/templates/spider/spider.py.tmpl +141 -141
crawlo/tools/__init__.py +183 -0
crawlo/tools/anti_crawler.py +269 -0
crawlo/tools/authenticated_proxy.py +241 -0
crawlo/tools/data_validator.py +181 -0
crawlo/tools/date_tools.py +36 -0
crawlo/tools/distributed_coordinator.py +387 -0
crawlo/tools/retry_mechanism.py +221 -0
crawlo/tools/scenario_adapter.py +263 -0
crawlo/utils/__init__.py +35 -7
crawlo/utils/batch_processor.py +261 -0
crawlo/utils/controlled_spider_mixin.py +439 -439
crawlo/utils/date_tools.py +290 -233
crawlo/utils/db_helper.py +343 -343
crawlo/utils/enhanced_error_handler.py +360 -0
crawlo/utils/env_config.py +106 -0
crawlo/utils/error_handler.py +126 -0
crawlo/utils/func_tools.py +82 -82
crawlo/utils/large_scale_config.py +286 -286
crawlo/utils/large_scale_helper.py +343 -343
crawlo/utils/log.py +128 -128
crawlo/utils/performance_monitor.py +285 -0
crawlo/utils/queue_helper.py +175 -175
crawlo/utils/redis_connection_pool.py +335 -0
crawlo/utils/redis_key_validator.py +200 -0
crawlo/utils/request.py +267 -267
crawlo/utils/request_serializer.py +219 -219
crawlo/utils/spider_loader.py +62 -62
crawlo/utils/system.py +11 -11
crawlo/utils/tools.py +4 -4
crawlo/utils/url.py +39 -39
{crawlo-1.1.4.dist-info → crawlo-1.1.6.dist-info}/METADATA +401 -403
crawlo-1.1.6.dist-info/RECORD +189 -0
examples/__init__.py +7 -7
tests/DOUBLE_CRAWLO_PREFIX_FIX_REPORT.md +82 -0
tests/__init__.py +7 -7
tests/advanced_tools_example.py +276 -0
tests/authenticated_proxy_example.py +237 -0
tests/cleaners_example.py +161 -0
tests/config_validation_demo.py +103 -0
{examples → tests}/controlled_spider_example.py +205 -205
tests/date_tools_example.py +181 -0
tests/dynamic_loading_example.py +524 -0
tests/dynamic_loading_test.py +105 -0
tests/env_config_example.py +134 -0
tests/error_handling_example.py +172 -0
tests/redis_key_validation_demo.py +131 -0
tests/response_improvements_example.py +145 -0
tests/test_advanced_tools.py +149 -0
tests/test_all_redis_key_configs.py +146 -0
tests/test_authenticated_proxy.py +142 -0
tests/test_cleaners.py +55 -0
tests/test_comprehensive.py +147 -0
tests/test_config_validator.py +194 -0
tests/test_date_tools.py +124 -0
tests/test_double_crawlo_fix.py +208 -0
tests/test_double_crawlo_fix_simple.py +125 -0
tests/test_dynamic_downloaders_proxy.py +125 -0
tests/test_dynamic_proxy.py +93 -0
tests/test_dynamic_proxy_config.py +147 -0
tests/test_dynamic_proxy_real.py +110 -0
tests/test_edge_cases.py +304 -0
tests/test_enhanced_error_handler.py +271 -0
tests/test_env_config.py +122 -0
tests/test_error_handler_compatibility.py +113 -0
tests/test_final_validation.py +153 -153
tests/test_framework_env_usage.py +104 -0
tests/test_integration.py +357 -0
tests/test_item_dedup_redis_key.py +123 -0
tests/test_parsel.py +30 -0
tests/test_performance.py +328 -0
tests/test_proxy_health_check.py +32 -32
tests/test_proxy_middleware_integration.py +136 -136
tests/test_proxy_providers.py +56 -56
tests/test_proxy_stats.py +19 -19
tests/test_proxy_strategies.py +59 -59
tests/test_queue_manager_double_crawlo.py +231 -0
tests/test_queue_manager_redis_key.py +177 -0
tests/test_redis_config.py +28 -28
tests/test_redis_connection_pool.py +295 -0
tests/test_redis_key_naming.py +182 -0
tests/test_redis_key_validator.py +124 -0
tests/test_redis_queue.py +224 -224
tests/test_request_serialization.py +70 -70
tests/test_response_improvements.py +153 -0
tests/test_scheduler.py +241 -241
tests/test_simple_response.py +62 -0
tests/test_telecom_spider_redis_key.py +206 -0
tests/test_template_content.py +88 -0
tests/test_template_redis_key.py +135 -0
tests/test_tools.py +154 -0
tests/tools_example.py +258 -0
crawlo/core/enhanced_engine.py +0 -190
crawlo-1.1.4.dist-info/RECORD +0 -117
{crawlo-1.1.4.dist-info → crawlo-1.1.6.dist-info}/WHEEL +0 -0
{crawlo-1.1.4.dist-info → crawlo-1.1.6.dist-info}/entry_points.txt +0 -0
{crawlo-1.1.4.dist-info → crawlo-1.1.6.dist-info}/top_level.txt +0 -0

tests/test_simple_response.py ADDED Viewed

@@ -0,0 +1,62 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+Response 简单功能测试
+"""
+import sys
+import os
+# 添加项目根目录到路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+from crawlo.network.response import Response
+def test_basic_functionality():
+    """测试基本功能"""
+    print("测试基本功能...")
+    # 创建一个简单的HTML响应
+    html_content = """
+    <html>
+    <head>
+        <title>测试页面</title>
+    </head>
+    <body>
+        <div class="content">
+            <h1>主标题</h1>
+            <p class="intro">这是介绍段落</p>
+        </div>
+    </body>
+    </html>
+    """
+    response = Response(
+        url="https://example.com/test",
+        body=html_content.encode('utf-8'),
+        headers={"content-type": "text/html; charset=utf-8"}
+    )
+    # 测试基本属性
+    print(f"URL: {response.url}")
+    print(f"状态码: {response.status_code}")
+    # 测试文本提取（使用新方法）
+    title = response.extract_text('title')
+    print(f"标题: {title}")
+    h1_text = response.extract_text('.content h1')
+    print(f"H1文本: {h1_text}")
+    intro_text = response.extract_text('.intro')
+    print(f"介绍文本: {intro_text}")
+    # 测试XPath（使用新方法）
+    title_xpath = response.extract_text('//title')
+    print(f"XPath标题: {title_xpath}")
+    print("基本功能测试完成")
+if __name__ == '__main__':
+    test_basic_functionality()

tests/test_telecom_spider_redis_key.py ADDED Viewed

@@ -0,0 +1,206 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+电信设备许可证爬虫Redis Key测试脚本
+用于验证分布式爬虫是否符合新的Redis key命名规范
+"""
+import sys
+import os
+import asyncio
+import tempfile
+import shutil
+from pathlib import Path
+# 添加项目根目录到路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+# 导入相关模块
+from crawlo.queue.queue_manager import QueueManager, QueueConfig, QueueType
+from crawlo.filters.aioredis_filter import AioRedisFilter
+from crawlo.pipelines.redis_dedup_pipeline import RedisDedupPipeline
+class MockSettings:
+    """模拟设置类"""
+    def __init__(self, project_name="telecom_licenses_distributed"):
+        self.project_name = project_name
+        self.REDIS_HOST = '127.0.0.1'
+        self.REDIS_PORT = 6379
+        self.REDIS_PASSWORD = ''
+        self.REDIS_DB = 2
+        self.REDIS_URL = f'redis://127.0.0.1:6379/{self.REDIS_DB}'
+        self.REDIS_TTL = 0
+        self.CLEANUP_FP = 0
+        self.FILTER_DEBUG = True
+        self.LOG_LEVEL = "INFO"
+        self.DECODE_RESPONSES = True
+        self.SCHEDULER_QUEUE_NAME = f'crawlo:{project_name}:queue:requests'
+    def get(self, key, default=None):
+        if key == 'PROJECT_NAME':
+            return self.project_name
+        elif key == 'REDIS_HOST':
+            return self.REDIS_HOST
+        elif key == 'REDIS_PASSWORD':
+            return self.REDIS_PASSWORD
+        elif key == 'REDIS_URL':
+            return self.REDIS_URL
+        elif key == 'FILTER_DEBUG':
+            return self.FILTER_DEBUG
+        elif key == 'LOG_LEVEL':
+            return self.LOG_LEVEL
+        elif key == 'DECODE_RESPONSES':
+            return self.DECODE_RESPONSES
+        elif key == 'SCHEDULER_QUEUE_NAME':
+            return self.SCHEDULER_QUEUE_NAME
+        return default
+    def get_bool(self, key, default=False):
+        if key == 'FILTER_DEBUG':
+            return self.FILTER_DEBUG
+        elif key == 'DECODE_RESPONSES':
+            return self.DECODE_RESPONSES
+        elif key == 'CLEANUP_FP':
+            return self.CLEANUP_FP
+        return default
+    def get_int(self, key, default=0):  # 修复方法名
+        if key == 'REDIS_TTL':
+            return self.REDIS_TTL
+        elif key == 'REDIS_PORT':
+            return self.REDIS_PORT
+        elif key == 'REDIS_DB':
+            return self.REDIS_DB
+        elif key == 'SCHEDULER_MAX_QUEUE_SIZE':
+            return 1000
+        elif key == 'QUEUE_MAX_RETRIES':
+            return 3
+        elif key == 'QUEUE_TIMEOUT':
+            return 300
+        return default
+class MockCrawler:
+    """模拟爬虫类"""
+    def __init__(self, project_name="telecom_licenses_distributed"):
+        self.settings = MockSettings(project_name)
+        self.stats = {}
+async def test_telecom_spider_redis_key():
+    """测试电信设备许可证爬虫Redis key命名规范"""
+    print("🔍 测试电信设备许可证爬虫Redis key命名规范...")
+    project_name = "telecom_licenses_distributed"
+    expected_prefix = f"crawlo:{project_name}"
+    try:
+        # 1. 测试QueueManager和RedisPriorityQueue
+        print("   1. 测试队列管理器...")
+        queue_config = QueueConfig(
+            queue_type=QueueType.REDIS,
+            redis_url="redis://127.0.0.1:6379/2",
+            queue_name=f"crawlo:{project_name}:queue:requests",  # 使用统一命名规范
+            max_queue_size=1000,
+            max_retries=3,
+            timeout=300
+        )
+        queue_manager = QueueManager(queue_config)
+        queue = await queue_manager._create_queue(QueueType.REDIS)
+        # 验证队列名称是否符合规范
+        expected_queue_name = f"{expected_prefix}:queue:requests"
+        expected_processing_queue = f"{expected_prefix}:queue:processing"
+        expected_failed_queue = f"{expected_prefix}:queue:failed"
+        assert queue.queue_name == expected_queue_name, f"队列名称不匹配: {queue.queue_name} != {expected_queue_name}"
+        assert queue.processing_queue == expected_processing_queue, f"处理中队列名称不匹配: {queue.processing_queue} != {expected_processing_queue}"
+        assert queue.failed_queue == expected_failed_queue, f"失败队列名称不匹配: {queue.failed_queue} != {expected_failed_queue}"
+        print(f"      ✅ 请求队列: {queue.queue_name}")
+        print(f"      ✅ 处理中队列: {queue.processing_queue}")
+        print(f"      ✅ 失败队列: {queue.failed_queue}")
+        # 2. 测试AioRedisFilter
+        print("   2. 测试请求去重过滤器...")
+        mock_crawler = MockCrawler(project_name)
+        filter_instance = AioRedisFilter.create_instance(mock_crawler)
+        expected_filter_key = f"{expected_prefix}:filter:fingerprint"
+        assert filter_instance.redis_key == expected_filter_key, f"过滤器key不匹配: {filter_instance.redis_key} != {expected_filter_key}"
+        print(f"      ✅ 请求去重key: {filter_instance.redis_key}")
+        # 3. 测试RedisDedupPipeline
+        print("   3. 测试数据项去重管道...")
+        dedup_pipeline = RedisDedupPipeline.from_crawler(mock_crawler)
+        expected_item_key = f"{expected_prefix}:item:fingerprint"
+        assert dedup_pipeline.redis_key == expected_item_key, f"数据项去重key不匹配: {dedup_pipeline.redis_key} != {expected_item_key}"
+        print(f"      ✅ 数据项去重key: {dedup_pipeline.redis_key}")
+        # 4. 验证所有key都使用统一前缀
+        print("   4. 验证统一前缀...")
+        all_keys = [
+            queue.queue_name,
+            queue.processing_queue,
+            queue.failed_queue,
+            filter_instance.redis_key,
+            dedup_pipeline.redis_key
+        ]
+        for key in all_keys:
+            assert key.startswith(expected_prefix), f"Key未使用统一前缀: {key}"
+            print(f"      ✅ {key}")
+        print("✅ 电信设备许可证爬虫Redis key命名规范测试通过！")
+        return True
+    except Exception as e:
+        print(f"❌ 测试失败: {e}")
+        import traceback
+        traceback.print_exc()
+        return False
+    finally:
+        # 清理资源
+        try:
+            if 'queue' in locals():
+                await queue.close()
+            if 'filter_instance' in locals() and hasattr(filter_instance, 'redis'):
+                await filter_instance.redis.close()
+            if 'dedup_pipeline' in locals() and hasattr(dedup_pipeline, 'redis_client'):
+                dedup_pipeline.redis_client.close()
+        except:
+            pass
+async def main():
+    """主测试函数"""
+    print("🚀 开始电信设备许可证爬虫Redis key命名规范测试...")
+    print("=" * 60)
+    try:
+        success = await test_telecom_spider_redis_key()
+        print("=" * 60)
+        if success:
+            print("🎉 所有测试通过！电信设备许可证爬虫符合新的Redis key命名规范")
+        else:
+            print("❌ 测试失败，请检查实现")
+            return 1
+    except Exception as e:
+        print("=" * 60)
+        print(f"❌ 测试过程中发生异常: {e}")
+        import traceback
+        traceback.print_exc()
+        return 1
+    return 0
+if __name__ == "__main__":
+    exit_code = asyncio.run(main())
+    sys.exit(exit_code)

tests/test_template_content.py ADDED Viewed

@@ -0,0 +1,88 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+模板文件内容测试脚本
+用于验证模板文件是否符合新的Redis key命名规范
+"""
+import sys
+import os
+# 添加项目根目录到路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+def test_template_content():
+    """测试模板文件内容"""
+    print("🔍 测试模板文件内容...")
+    try:
+        # 检查settings.py.tmpl模板文件
+        template_file = "crawlo/templates/project/settings.py.tmpl"
+        if not os.path.exists(template_file):
+            print(f"❌ 模板文件不存在: {template_file}")
+            return False
+        with open(template_file, 'r', encoding='utf-8') as f:
+            content = f.read()
+        # 检查是否移除了旧的REDIS_KEY配置
+        old_config = "REDIS_KEY = f'{{project_name}}:fingerprint'"
+        if old_config in content:
+            print("❌ 仍然存在旧的REDIS_KEY配置")
+            return False
+        print("      ✅ 已移除旧的REDIS_KEY配置")
+        # 检查是否添加了新的注释
+        filter_comment = "# crawlo:{project_name}:filter:fingerprint (请求去重)"
+        if filter_comment not in content:
+            print("❌ 缺少请求去重的Redis key命名规范注释")
+            return False
+        print("      ✅ 包含请求去重的Redis key命名规范注释")
+        item_comment = "# crawlo:{project_name}:item:fingerprint (数据项去重)"
+        if item_comment not in content:
+            print("❌ 缺少数据项去重的Redis key命名规范注释")
+            return False
+        print("      ✅ 包含数据项去重的Redis key命名规范注释")
+        # 检查是否保留了队列名称配置
+        queue_config = "SCHEDULER_QUEUE_NAME = f'crawlo:{{project_name}}:queue:requests'"
+        if queue_config not in content:
+            print("❌ 缺少队列名称配置")
+            return False
+        print("      ✅ 包含队列名称配置")
+        print("✅ 模板文件内容测试通过！")
+        return True
+    except Exception as e:
+        print(f"❌ 测试过程中发生错误: {e}")
+        return False
+def main():
+    """主测试函数"""
+    print("🚀 开始模板文件内容测试...")
+    print("=" * 50)
+    try:
+        success = test_template_content()
+        print("=" * 50)
+        if success:
+            print("🎉 所有测试通过！模板文件符合新的Redis key命名规范")
+        else:
+            print("❌ 测试失败，请检查模板文件")
+            return 1
+    except Exception as e:
+        print("=" * 50)
+        print(f"❌ 测试过程中发生异常: {e}")
+        return 1
+    return 0
+if __name__ == "__main__":
+    exit_code = main()
+    sys.exit(exit_code)

tests/test_template_redis_key.py ADDED Viewed

@@ -0,0 +1,135 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+模板项目Redis Key测试脚本
+用于验证通过模板生成的项目是否符合新的Redis key命名规范
+"""
+import sys
+import os
+import tempfile
+import shutil
+import subprocess
+from pathlib import Path
+# 添加项目根目录到路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+def test_template_project_redis_key():
+    """测试模板项目Redis key命名规范"""
+    print("🔍 测试模板项目Redis key命名规范...")
+    # 创建临时目录
+    with tempfile.TemporaryDirectory() as temp_dir:
+        try:
+            # 在原始工作目录中创建项目，然后移动到临时目录
+            original_cwd = os.getcwd()
+            # 创建测试项目（在原始工作目录中）
+            print("   1. 创建测试项目...")
+            cmd_path = os.path.join(original_cwd, "crawlo", "commands", "startproject.py")
+            result = subprocess.run([
+                sys.executable, cmd_path, "test_project"
+            ], cwd=original_cwd, capture_output=True, text=True)
+            if result.returncode != 0:
+                print(f"❌ 创建项目失败: {result.stderr}")
+                return False
+            print("      ✅ 项目创建成功")
+            # 检查生成的文件
+            project_dir = Path(original_cwd) / "test_project"
+            if not project_dir.exists():
+                print("❌ 项目目录未创建")
+                return False
+            # 移动项目到临时目录
+            target_dir = Path(temp_dir) / "test_project"
+            shutil.move(str(project_dir), str(target_dir))
+            project_dir = target_dir
+            settings_file = project_dir / "test_project" / "settings.py"
+            if not settings_file.exists():
+                print("❌ settings.py文件未创建")
+                return False
+            # 读取settings.py内容
+            with open(settings_file, 'r', encoding='utf-8') as f:
+                settings_content = f.read()
+            # 检查是否移除了旧的REDIS_KEY配置
+            if "REDIS_KEY = f'{{project_name}}:fingerprint'" in settings_content:
+                print("❌ 仍然存在旧的REDIS_KEY配置")
+                return False
+            # 检查是否添加了新的注释
+            if "# crawlo:{project_name}:filter:fingerprint (请求去重)" not in settings_content:
+                print("❌ 缺少新的Redis key命名规范注释")
+                return False
+            if "# crawlo:{project_name}:item:fingerprint (数据项去重)" not in settings_content:
+                print("❌ 缺少数据项去重的Redis key命名规范注释")
+                return False
+            print("      ✅ settings.py符合新的Redis key命名规范")
+            # 检查crawlo.cfg
+            cfg_file = project_dir / "crawlo.cfg"
+            if not cfg_file.exists():
+                print("❌ crawlo.cfg文件未创建")
+                return False
+            with open(cfg_file, 'r', encoding='utf-8') as f:
+                cfg_content = f.read()
+            if "default = test_project.settings" not in cfg_content:
+                print("❌ crawlo.cfg配置不正确")
+                return False
+            print("      ✅ crawlo.cfg配置正确")
+            print("✅ 模板项目Redis key命名规范测试通过！")
+            return True
+        except Exception as e:
+            print(f"❌ 测试过程中发生错误: {e}")
+            import traceback
+            traceback.print_exc()
+            return False
+        finally:
+            # 清理创建的项目目录
+            project_dir = Path(original_cwd) / "test_project"
+            if project_dir.exists():
+                shutil.rmtree(str(project_dir), ignore_errors=True)
+            # 恢复原始工作目录
+            os.chdir(original_cwd)
+def main():
+    """主测试函数"""
+    print("🚀 开始模板项目Redis key命名规范测试...")
+    print("=" * 50)
+    try:
+        success = test_template_project_redis_key()
+        print("=" * 50)
+        if success:
+            print("🎉 所有测试通过！模板项目符合新的Redis key命名规范")
+        else:
+            print("❌ 测试失败，请检查模板文件")
+            return 1
+    except Exception as e:
+        print("=" * 50)
+        print(f"❌ 测试过程中发生异常: {e}")
+        return 1
+    return 0
+if __name__ == "__main__":
+    exit_code = main()
+    sys.exit(exit_code)

tests/test_tools.py ADDED Viewed

@@ -0,0 +1,154 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+工具包测试
+"""
+import unittest
+from crawlo.tools import (
+    # 日期工具
+    parse_time,
+    format_time,
+    time_diff,
+    # 数据清洗工具
+    clean_text,
+    format_currency,
+    extract_emails,
+    # 数据验证工具
+    validate_email,
+    validate_url,
+    validate_phone,
+    validate_chinese_id_card,
+    validate_date,
+    validate_number_range,
+    # 请求处理工具
+    build_url,
+    add_query_params,
+    merge_headers,
+    # 反爬虫应对工具
+    get_random_user_agent,
+    rotate_proxy,
+    # 分布式协调工具
+    generate_task_id,
+    get_cluster_info
+)
+class TestTools(unittest.TestCase):
+    """工具包测试类"""
+    def test_date_tools(self):
+        """测试日期工具"""
+        # 测试时间解析
+        time_str = "2025-09-10 14:30:00"
+        parsed_time = parse_time(time_str)
+        self.assertIsNotNone(parsed_time)
+        # 测试时间格式化
+        formatted_time = format_time(parsed_time, "%Y-%m-%d")
+        self.assertEqual(formatted_time, "2025-09-10")
+        # 测试时间差计算
+        time_str2 = "2025-09-11 14:30:00"
+        parsed_time2 = parse_time(time_str2)
+        diff = time_diff(parsed_time2, parsed_time)
+        self.assertEqual(diff, 86400)  # 24小时 = 86400秒
+    def test_data_cleaning_tools(self):
+        """测试数据清洗工具"""
+        # 测试文本清洗
+        dirty_text = "<p>这是一个&nbsp;<b>测试</b>&amp;文本</p>"
+        clean_result = clean_text(dirty_text)
+        self.assertEqual(clean_result, "这是一个 测试&文本")
+        # 测试货币格式化
+        price = 1234.567
+        formatted_price = format_currency(price, "¥", 2)
+        self.assertEqual(formatted_price, "¥1,234.57")
+        # 测试邮箱提取
+        text_with_email = "联系邮箱: test@example.com, support@crawler.com"
+        emails = extract_emails(text_with_email)
+        self.assertIn("test@example.com", emails)
+        self.assertIn("support@crawler.com", emails)
+    def test_data_validation_tools(self):
+        """测试数据验证工具"""
+        # 测试邮箱验证
+        self.assertTrue(validate_email("test@example.com"))
+        self.assertFalse(validate_email("invalid-email"))
+        # 测试URL验证
+        self.assertTrue(validate_url("https://example.com"))
+        self.assertFalse(validate_url("invalid-url"))
+        # 测试电话验证
+        self.assertTrue(validate_phone("13812345678"))
+        self.assertFalse(validate_phone("12345"))
+        # 测试身份证验证
+        self.assertTrue(validate_chinese_id_card("110101199001011234"))
+        self.assertFalse(validate_chinese_id_card("invalid-id"))
+        # 测试日期验证
+        self.assertTrue(validate_date("2025-09-10"))
+        self.assertFalse(validate_date("invalid-date"))
+        # 测试数值范围验证
+        self.assertTrue(validate_number_range(50, 1, 100))
+        self.assertFalse(validate_number_range(150, 1, 100))
+    def test_request_handling_tools(self):
+        """测试请求处理工具"""
+        # 测试URL构建
+        base_url = "https://api.example.com"
+        path = "/v1/users"
+        query_params = {"page": 1, "limit": 10}
+        full_url = build_url(base_url, path, query_params)
+        self.assertIn("https://api.example.com/v1/users", full_url)
+        self.assertIn("page=1", full_url)
+        self.assertIn("limit=10", full_url)
+        # 测试添加查询参数
+        existing_url = "https://api.example.com/v1/users?page=1"
+        new_params = {"sort": "name"}
+        updated_url = add_query_params(existing_url, new_params)
+        self.assertIn("sort=name", updated_url)
+        # 测试合并请求头
+        base_headers = {"Content-Type": "application/json"}
+        additional_headers = {"Authorization": "Bearer token123"}
+        merged_headers = merge_headers(base_headers, additional_headers)
+        self.assertEqual(merged_headers["Content-Type"], "application/json")
+        self.assertEqual(merged_headers["Authorization"], "Bearer token123")
+    def test_anti_crawler_tools(self):
+        """测试反爬虫应对工具"""
+        # 测试随机User-Agent
+        user_agent = get_random_user_agent()
+        self.assertIsInstance(user_agent, str)
+        self.assertGreater(len(user_agent), 0)
+        # 测试代理轮换
+        proxy = rotate_proxy()
+        self.assertIsInstance(proxy, dict)
+    def test_distributed_coordinator_tools(self):
+        """测试分布式协调工具"""
+        # 测试任务ID生成
+        task_id = generate_task_id("https://example.com", "test_spider")
+        self.assertIsInstance(task_id, str)
+        self.assertEqual(len(task_id), 32)  # MD5 hash长度
+        # 测试集群信息获取
+        cluster_info = get_cluster_info()
+        self.assertIsInstance(cluster_info, dict)
+        self.assertIn("worker_count", cluster_info)
+if __name__ == '__main__':
+    unittest.main()

crawlo 1.1.4__py3-none-any.whl → 1.1.6__py3-none-any.whl

Potentially problematic release.

crawlo 1.1.4py3-none-any.whl → 1.1.6py3-none-any.whl