PyPI - crawlo - Versions diffs - 1.1.8__py3-none-any.whl → 1.2.0__py3-none-any.whl - Mend

crawlo 1.1.8py3-none-any.whl → 1.2.0py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.

This version of crawlo might be problematic. Click here for more details.

Files changed (191) hide show

crawlo/__init__.py +61 -61
crawlo/__version__.py +1 -1
crawlo/cleaners/__init__.py +60 -60
crawlo/cleaners/data_formatter.py +225 -225
crawlo/cleaners/encoding_converter.py +125 -125
crawlo/cleaners/text_cleaner.py +232 -232
crawlo/cli.py +65 -65
crawlo/commands/__init__.py +14 -14
crawlo/commands/check.py +594 -594
crawlo/commands/genspider.py +151 -151
crawlo/commands/help.py +132 -132
crawlo/commands/list.py +155 -155
crawlo/commands/run.py +292 -292
crawlo/commands/startproject.py +418 -418
crawlo/commands/stats.py +188 -188
crawlo/commands/utils.py +186 -186
crawlo/config.py +312 -312
crawlo/config_validator.py +252 -252
crawlo/core/__init__.py +2 -2
crawlo/core/engine.py +354 -345
crawlo/core/processor.py +40 -40
crawlo/core/scheduler.py +143 -136
crawlo/crawler.py +1027 -1027
crawlo/downloader/__init__.py +266 -266
crawlo/downloader/aiohttp_downloader.py +220 -220
crawlo/downloader/cffi_downloader.py +256 -256
crawlo/downloader/httpx_downloader.py +259 -259
crawlo/downloader/hybrid_downloader.py +213 -213
crawlo/downloader/playwright_downloader.py +402 -402
crawlo/downloader/selenium_downloader.py +472 -472
crawlo/event.py +11 -11
crawlo/exceptions.py +81 -81
crawlo/extension/__init__.py +37 -37
crawlo/extension/health_check.py +141 -141
crawlo/extension/log_interval.py +57 -57
crawlo/extension/log_stats.py +81 -81
crawlo/extension/logging_extension.py +43 -43
crawlo/extension/memory_monitor.py +104 -104
crawlo/extension/performance_profiler.py +133 -133
crawlo/extension/request_recorder.py +107 -107
crawlo/filters/__init__.py +154 -154
crawlo/filters/aioredis_filter.py +280 -280
crawlo/filters/memory_filter.py +269 -269
crawlo/items/__init__.py +23 -23
crawlo/items/base.py +21 -21
crawlo/items/fields.py +53 -53
crawlo/items/items.py +104 -104
crawlo/middleware/__init__.py +21 -21
crawlo/middleware/default_header.py +32 -32
crawlo/middleware/download_delay.py +28 -28
crawlo/middleware/middleware_manager.py +135 -135
crawlo/middleware/proxy.py +272 -272
crawlo/middleware/request_ignore.py +30 -30
crawlo/middleware/response_code.py +18 -18
crawlo/middleware/response_filter.py +26 -26
crawlo/middleware/retry.py +124 -124
crawlo/mode_manager.py +211 -211
crawlo/network/__init__.py +21 -21
crawlo/network/request.py +338 -338
crawlo/network/response.py +359 -359
crawlo/pipelines/__init__.py +21 -21
crawlo/pipelines/bloom_dedup_pipeline.py +156 -156
crawlo/pipelines/console_pipeline.py +39 -39
crawlo/pipelines/csv_pipeline.py +316 -316
crawlo/pipelines/database_dedup_pipeline.py +224 -224
crawlo/pipelines/json_pipeline.py +218 -218
crawlo/pipelines/memory_dedup_pipeline.py +115 -115
crawlo/pipelines/mongo_pipeline.py +131 -131
crawlo/pipelines/mysql_pipeline.py +316 -316
crawlo/pipelines/pipeline_manager.py +61 -61
crawlo/pipelines/redis_dedup_pipeline.py +167 -167
crawlo/project.py +187 -187
crawlo/queue/pqueue.py +37 -37
crawlo/queue/queue_manager.py +337 -334
crawlo/queue/redis_priority_queue.py +298 -298
crawlo/settings/__init__.py +7 -7
crawlo/settings/default_settings.py +219 -219
crawlo/settings/setting_manager.py +122 -122
crawlo/spider/__init__.py +639 -639
crawlo/stats_collector.py +59 -59
crawlo/subscriber.py +130 -130
crawlo/task_manager.py +30 -30
crawlo/templates/crawlo.cfg.tmpl +10 -10
crawlo/templates/project/__init__.py.tmpl +3 -3
crawlo/templates/project/items.py.tmpl +17 -17
crawlo/templates/project/middlewares.py.tmpl +109 -109
crawlo/templates/project/pipelines.py.tmpl +96 -96
crawlo/templates/project/run.py.tmpl +45 -45
crawlo/templates/project/settings.py.tmpl +326 -326
crawlo/templates/project/settings_distributed.py.tmpl +119 -119
crawlo/templates/project/settings_gentle.py.tmpl +94 -94
crawlo/templates/project/settings_high_performance.py.tmpl +151 -151
crawlo/templates/project/settings_simple.py.tmpl +68 -68
crawlo/templates/project/spiders/__init__.py.tmpl +5 -5
crawlo/templates/spider/spider.py.tmpl +141 -141
crawlo/tools/__init__.py +182 -182
crawlo/tools/anti_crawler.py +268 -268
crawlo/tools/authenticated_proxy.py +240 -240
crawlo/tools/data_validator.py +180 -180
crawlo/tools/date_tools.py +35 -35
crawlo/tools/distributed_coordinator.py +386 -386
crawlo/tools/retry_mechanism.py +220 -220
crawlo/tools/scenario_adapter.py +262 -262
crawlo/utils/__init__.py +35 -35
crawlo/utils/batch_processor.py +260 -260
crawlo/utils/controlled_spider_mixin.py +439 -439
crawlo/utils/date_tools.py +290 -290
crawlo/utils/db_helper.py +343 -343
crawlo/utils/enhanced_error_handler.py +359 -359
crawlo/utils/env_config.py +105 -105
crawlo/utils/error_handler.py +125 -125
crawlo/utils/func_tools.py +82 -82
crawlo/utils/large_scale_config.py +286 -286
crawlo/utils/large_scale_helper.py +343 -343
crawlo/utils/log.py +128 -128
crawlo/utils/performance_monitor.py +284 -284
crawlo/utils/queue_helper.py +175 -175
crawlo/utils/redis_connection_pool.py +334 -334
crawlo/utils/redis_key_validator.py +199 -199
crawlo/utils/request.py +267 -267
crawlo/utils/request_serializer.py +219 -219
crawlo/utils/spider_loader.py +62 -62
crawlo/utils/system.py +11 -11
crawlo/utils/tools.py +4 -4
crawlo/utils/url.py +39 -39
crawlo-1.2.0.dist-info/METADATA +697 -0
crawlo-1.2.0.dist-info/RECORD +190 -0
examples/__init__.py +7 -7
tests/DOUBLE_CRAWLO_PREFIX_FIX_REPORT.md +81 -81
tests/__init__.py +7 -7
tests/advanced_tools_example.py +275 -275
tests/authenticated_proxy_example.py +236 -236
tests/cleaners_example.py +160 -160
tests/config_validation_demo.py +102 -102
tests/controlled_spider_example.py +205 -205
tests/date_tools_example.py +180 -180
tests/dynamic_loading_example.py +523 -523
tests/dynamic_loading_test.py +104 -104
tests/env_config_example.py +133 -133
tests/error_handling_example.py +171 -171
tests/redis_key_validation_demo.py +130 -130
tests/response_improvements_example.py +144 -144
tests/test_advanced_tools.py +148 -148
tests/test_all_redis_key_configs.py +145 -145
tests/test_authenticated_proxy.py +141 -141
tests/test_cleaners.py +54 -54
tests/test_comprehensive.py +146 -146
tests/test_config_validator.py +193 -193
tests/test_date_tools.py +123 -123
tests/test_double_crawlo_fix.py +207 -207
tests/test_double_crawlo_fix_simple.py +124 -124
tests/test_dynamic_downloaders_proxy.py +124 -124
tests/test_dynamic_proxy.py +92 -92
tests/test_dynamic_proxy_config.py +146 -146
tests/test_dynamic_proxy_real.py +109 -109
tests/test_edge_cases.py +303 -303
tests/test_enhanced_error_handler.py +270 -270
tests/test_env_config.py +121 -121
tests/test_error_handler_compatibility.py +112 -112
tests/test_final_validation.py +153 -153
tests/test_framework_env_usage.py +103 -103
tests/test_integration.py +356 -356
tests/test_item_dedup_redis_key.py +122 -122
tests/test_parsel.py +29 -29
tests/test_performance.py +327 -327
tests/test_proxy_health_check.py +32 -32
tests/test_proxy_middleware_integration.py +136 -136
tests/test_proxy_providers.py +56 -56
tests/test_proxy_stats.py +19 -19
tests/test_proxy_strategies.py +59 -59
tests/test_queue_manager_double_crawlo.py +174 -231
tests/test_queue_manager_redis_key.py +176 -176
tests/test_redis_config.py +28 -28
tests/test_redis_connection_pool.py +294 -294
tests/test_redis_key_naming.py +181 -181
tests/test_redis_key_validator.py +123 -123
tests/test_redis_queue.py +224 -224
tests/test_request_serialization.py +70 -70
tests/test_response_improvements.py +152 -152
tests/test_scheduler.py +241 -241
tests/test_simple_response.py +61 -61
tests/test_telecom_spider_redis_key.py +205 -205
tests/test_template_content.py +87 -87
tests/test_template_redis_key.py +134 -134
tests/test_tools.py +153 -153
tests/tools_example.py +257 -257
crawlo-1.1.8.dist-info/METADATA +0 -626
crawlo-1.1.8.dist-info/RECORD +0 -190
{crawlo-1.1.8.dist-info → crawlo-1.2.0.dist-info}/WHEEL +0 -0
{crawlo-1.1.8.dist-info → crawlo-1.2.0.dist-info}/entry_points.txt +0 -0
{crawlo-1.1.8.dist-info → crawlo-1.2.0.dist-info}/top_level.txt +0 -0

tests/response_improvements_example.py CHANGED Viewed

@@ -1,145 +1,145 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-"""
-Response 改进功能使用示例
-"""
-from crawlo.network.response import Response
-def demo_response_improvements():
-    """演示Response改进功能"""
-    print("=== Response 改进功能演示 ===\n")
-    # 创建一个示例HTML响应
-    html_content = """
-    <html>
-    <head>
-        <title>Crawlo框架示例页面</title>
-    </head>
-    <body>
-        <div class="container">
-            <h1>产品列表</h1>
-            <div class="product-list">
-                <div class="product-item" data-id="1">
-                    <h2>产品A</h2>
-                    <p class="price">¥99.99</p>
-                    <p class="description">这是产品A的描述信息</p>
-                    <a href="/product/1" class="details-link">查看详情</a>
-                </div>
-                <div class="product-item" data-id="2">
-                    <h2>产品B</h2>
-                    <p class="price">¥149.99</p>
-                    <p class="description">这是产品B的描述信息</p>
-                    <a href="/product/2" class="details-link">查看详情</a>
-                </div>
-                <div class="product-item" data-id="3">
-                    <h2>产品C</h2>
-                    <p class="price">¥199.99</p>
-                    <p class="description">这是产品C的描述信息</p>
-                    <a href="/product/3" class="details-link">查看详情</a>
-                </div>
-            </div>
-            <div class="pagination">
-                <a href="/page/1" class="page-link">1</a>
-                <a href="/page/2" class="page-link active">2</a>
-                <a href="/page/3" class="page-link">3</a>
-            </div>
-        </div>
-    </body>
-    </html>
-    """
-    # 创建Response对象
-    response = Response(
-        url="https://example.com/products",
-        body=html_content.encode('utf-8'),
-        headers={"content-type": "text/html; charset=utf-8"}
-    )
-    # 1. 演示 extract_text 方法（支持CSS和XPath）
-    print("1. 提取文本内容:")
-    title = response.extract_text('title')
-    print(f"   页面标题: {title}")
-    # 使用CSS选择器提取第一个产品名称
-    first_product_name = response.extract_text('.product-item:first-child h2')
-    print(f"   第一个产品名称 (CSS): {first_product_name}")
-    # 使用XPath选择器
-    first_product_name_xpath = response.extract_text('//div[@class="product-item"][1]/h2')
-    print(f"   第一个产品名称 (XPath): {first_product_name_xpath}")
-    # 使用默认值处理不存在的元素
-    non_exist = response.extract_text('.non-exist', default='未找到')
-    print(f"   不存在的元素: {non_exist}")
-    print()
-    # 2. 演示 extract_texts 方法（提取多个元素的文本）
-    print("2. 提取多个元素的文本:")
-    # 提取所有产品名称
-    product_names = response.extract_texts('.product-item h2')
-    print(f"   所有产品名称: {product_names}")
-    # 提取所有价格
-    prices = response.extract_texts('.price')
-    print(f"   所有价格: {prices}")
-    # 使用XPath提取所有产品名称
-    product_names_xpath = response.extract_texts('//div[@class="product-item"]/h2')
-    print(f"   所有产品名称 (XPath): {product_names_xpath}")
-    print()
-    # 3. 演示 extract_attr 方法（提取元素属性）
-    print("3. 提取元素属性:")
-    # 提取第一个产品项的data-id属性
-    first_product_id = response.extract_attr('.product-item', 'data-id')
-    print(f"   第一个产品ID: {first_product_id}")
-    # 提取详情链接的href属性
-    first_detail_link = response.extract_attr('.details-link', 'href')
-    print(f"   第一个详情链接: {first_detail_link}")
-    # 提取不存在属性的默认值
-    non_exist_attr = response.extract_attr('.product-item', 'non-exist', default='默认值')
-    print(f"   不存在的属性: {non_exist_attr}")
-    print()
-    # 4. 演示 extract_attrs 方法（提取多个元素的属性）
-    print("4. 提取多个元素的属性:")
-    # 提取所有产品项的data-id属性
-    product_ids = response.extract_attrs('.product-item', 'data-id')
-    print(f"   所有产品ID: {product_ids}")
-    # 提取所有详情链接的href属性
-    detail_links = response.extract_attrs('.details-link', 'href')
-    print(f"   所有详情链接: {detail_links}")
-    # 提取分页链接的href属性
-    page_links = response.extract_attrs('.page-link', 'href')
-    print(f"   分页链接: {page_links}")
-    print()
-    # 5. 演示复杂文本提取
-    print("5. 复杂文本提取:")
-    # 提取所有产品描述
-    descriptions = response.extract_texts('.description')
-    print(f"   所有产品描述: {descriptions}")
-    print()
-    # 6. 边界情况处理
-    print("6. 边界情况处理:")
-    # 空响应测试
-    empty_response = Response(url="https://example.com/empty", body=b"")
-    empty_text = empty_response.extract_text('title', default='默认标题')
-    print(f"   空响应默认值: {empty_text}")
-    print("\n=== 演示完成 ===")
-if __name__ == '__main__':
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+Response 改进功能使用示例
+"""
+from crawlo.network.response import Response
+def demo_response_improvements():
+    """演示Response改进功能"""
+    print("=== Response 改进功能演示 ===\n")
+    # 创建一个示例HTML响应
+    html_content = """
+    <html>
+    <head>
+        <title>Crawlo框架示例页面</title>
+    </head>
+    <body>
+        <div class="container">
+            <h1>产品列表</h1>
+            <div class="product-list">
+                <div class="product-item" data-id="1">
+                    <h2>产品A</h2>
+                    <p class="price">¥99.99</p>
+                    <p class="description">这是产品A的描述信息</p>
+                    <a href="/product/1" class="details-link">查看详情</a>
+                </div>
+                <div class="product-item" data-id="2">
+                    <h2>产品B</h2>
+                    <p class="price">¥149.99</p>
+                    <p class="description">这是产品B的描述信息</p>
+                    <a href="/product/2" class="details-link">查看详情</a>
+                </div>
+                <div class="product-item" data-id="3">
+                    <h2>产品C</h2>
+                    <p class="price">¥199.99</p>
+                    <p class="description">这是产品C的描述信息</p>
+                    <a href="/product/3" class="details-link">查看详情</a>
+                </div>
+            </div>
+            <div class="pagination">
+                <a href="/page/1" class="page-link">1</a>
+                <a href="/page/2" class="page-link active">2</a>
+                <a href="/page/3" class="page-link">3</a>
+            </div>
+        </div>
+    </body>
+    </html>
+    """
+    # 创建Response对象
+    response = Response(
+        url="https://example.com/products",
+        body=html_content.encode('utf-8'),
+        headers={"content-type": "text/html; charset=utf-8"}
+    )
+    # 1. 演示 extract_text 方法（支持CSS和XPath）
+    print("1. 提取文本内容:")
+    title = response.extract_text('title')
+    print(f"   页面标题: {title}")
+    # 使用CSS选择器提取第一个产品名称
+    first_product_name = response.extract_text('.product-item:first-child h2')
+    print(f"   第一个产品名称 (CSS): {first_product_name}")
+    # 使用XPath选择器
+    first_product_name_xpath = response.extract_text('//div[@class="product-item"][1]/h2')
+    print(f"   第一个产品名称 (XPath): {first_product_name_xpath}")
+    # 使用默认值处理不存在的元素
+    non_exist = response.extract_text('.non-exist', default='未找到')
+    print(f"   不存在的元素: {non_exist}")
+    print()
+    # 2. 演示 extract_texts 方法（提取多个元素的文本）
+    print("2. 提取多个元素的文本:")
+    # 提取所有产品名称
+    product_names = response.extract_texts('.product-item h2')
+    print(f"   所有产品名称: {product_names}")
+    # 提取所有价格
+    prices = response.extract_texts('.price')
+    print(f"   所有价格: {prices}")
+    # 使用XPath提取所有产品名称
+    product_names_xpath = response.extract_texts('//div[@class="product-item"]/h2')
+    print(f"   所有产品名称 (XPath): {product_names_xpath}")
+    print()
+    # 3. 演示 extract_attr 方法（提取元素属性）
+    print("3. 提取元素属性:")
+    # 提取第一个产品项的data-id属性
+    first_product_id = response.extract_attr('.product-item', 'data-id')
+    print(f"   第一个产品ID: {first_product_id}")
+    # 提取详情链接的href属性
+    first_detail_link = response.extract_attr('.details-link', 'href')
+    print(f"   第一个详情链接: {first_detail_link}")
+    # 提取不存在属性的默认值
+    non_exist_attr = response.extract_attr('.product-item', 'non-exist', default='默认值')
+    print(f"   不存在的属性: {non_exist_attr}")
+    print()
+    # 4. 演示 extract_attrs 方法（提取多个元素的属性）
+    print("4. 提取多个元素的属性:")
+    # 提取所有产品项的data-id属性
+    product_ids = response.extract_attrs('.product-item', 'data-id')
+    print(f"   所有产品ID: {product_ids}")
+    # 提取所有详情链接的href属性
+    detail_links = response.extract_attrs('.details-link', 'href')
+    print(f"   所有详情链接: {detail_links}")
+    # 提取分页链接的href属性
+    page_links = response.extract_attrs('.page-link', 'href')
+    print(f"   分页链接: {page_links}")
+    print()
+    # 5. 演示复杂文本提取
+    print("5. 复杂文本提取:")
+    # 提取所有产品描述
+    descriptions = response.extract_texts('.description')
+    print(f"   所有产品描述: {descriptions}")
+    print()
+    # 6. 边界情况处理
+    print("6. 边界情况处理:")
+    # 空响应测试
+    empty_response = Response(url="https://example.com/empty", body=b"")
+    empty_text = empty_response.extract_text('title', default='默认标题')
+    print(f"   空响应默认值: {empty_text}")
+    print("\n=== 演示完成 ===")
+if __name__ == '__main__':
     demo_response_improvements()

tests/test_advanced_tools.py CHANGED Viewed

@@ -1,149 +1,149 @@
-#!/usr/bin/python
-# -*- coding: UTF-8 -*-
-"""
-高级工具测试
-"""
-import unittest
-from crawlo.tools import (
-    # 数据处理工具
-    clean_text,
-    format_currency,
-    validate_email,
-    validate_url,
-    check_data_integrity,
-    # 重试机制
-    RetryMechanism,
-    should_retry,
-    exponential_backoff,
-    # 反爬虫应对工具
-    AntiCrawler,
-    rotate_proxy,
-    handle_captcha,
-    detect_rate_limiting,
-    # 分布式协调工具
-    generate_pagination_tasks,
-    distribute_tasks,
-    DistributedCoordinator,
-    TaskDistributor,
-    DeduplicationTool
-)
-class TestAdvancedTools(unittest.TestCase):
-    """高级工具测试类"""
-    def test_data_processing_tools(self):
-        """测试数据处理工具"""
-        # 测试数据清洗
-        dirty_text = "<p>这是一个&nbsp;<b>测试</b>&amp;文本</p>"
-        clean_result = clean_text(dirty_text)
-        self.assertEqual(clean_result, "这是一个 测试&文本")
-        # 测试数据格式化
-        price = 1234.567
-        formatted_price = format_currency(price, "¥", 2)
-        self.assertEqual(formatted_price, "¥1,234.57")
-        # 测试字段验证
-        self.assertTrue(validate_email("test@example.com"))
-        self.assertFalse(validate_email("invalid-email"))
-        self.assertTrue(validate_url("https://example.com"))
-        self.assertFalse(validate_url("invalid-url"))
-        # 测试数据完整性检查
-        data = {
-            "name": "张三",
-            "email": "test@example.com",
-            "phone": "13812345678"
-        }
-        required_fields = ["name", "email", "phone"]
-        integrity_result = check_data_integrity(data, required_fields)
-        self.assertTrue(integrity_result["is_valid"])
-    def test_retry_mechanism(self):
-        """测试重试机制"""
-        # 测试指数退避
-        delay = exponential_backoff(0)
-        self.assertGreater(delay, 0)
-        # 测试是否应该重试
-        self.assertTrue(should_retry(status_code=500))
-        self.assertTrue(should_retry(exception=ConnectionError()))
-        self.assertFalse(should_retry(status_code=200))
-    def test_anti_crawler_tools(self):
-        """测试反爬虫应对工具"""
-        # 测试反爬虫工具
-        anti_crawler = AntiCrawler()
-        # 测试随机User-Agent
-        user_agent = anti_crawler.get_random_user_agent()
-        self.assertIsInstance(user_agent, str)
-        self.assertGreater(len(user_agent), 0)
-        # 测试代理轮换
-        proxy = anti_crawler.rotate_proxy()
-        self.assertIsInstance(proxy, dict)
-        # 测试验证码检测
-        self.assertTrue(anti_crawler.handle_captcha("请输入验证码进行验证"))
-        self.assertFalse(anti_crawler.handle_captcha("正常页面内容"))
-        # 测试频率限制检测
-        self.assertTrue(anti_crawler.detect_rate_limiting(429, {}))
-        self.assertFalse(anti_crawler.detect_rate_limiting(200, {}))
-    def test_distributed_coordinator_tools(self):
-        """测试分布式协调工具"""
-        # 测试任务分发器
-        distributor = TaskDistributor()
-        # 测试分页任务生成
-        base_url = "https://example.com/products"
-        pagination_tasks = distributor.generate_pagination_tasks(base_url, 1, 5)
-        self.assertEqual(len(pagination_tasks), 5)
-        # 测试任务分发
-        tasks = list(range(1, 21))  # 20个任务
-        distributed = distributor.distribute_tasks(tasks, 4)  # 分发给4个工作节点
-        self.assertEqual(len(distributed), 4)
-        self.assertEqual(sum(len(worker_tasks) for worker_tasks in distributed), 20)
-        # 测试去重工具
-        dedup_tool = DeduplicationTool()
-        # 测试数据指纹生成
-        fingerprint1 = dedup_tool.generate_fingerprint({"name": "test", "value": 123})
-        fingerprint2 = dedup_tool.generate_fingerprint({"name": "test", "value": 123})
-        self.assertEqual(fingerprint1, fingerprint2)
-        # 测试去重功能
-        self.assertFalse(dedup_tool.is_duplicate({"name": "test", "value": 123}))
-        self.assertTrue(dedup_tool.add_to_dedup({"name": "test", "value": 123}))
-        self.assertTrue(dedup_tool.is_duplicate({"name": "test", "value": 123}))
-        self.assertFalse(dedup_tool.add_to_dedup({"name": "test", "value": 123}))
-        # 测试分布式协调器
-        coordinator = DistributedCoordinator()
-        # 测试任务ID生成
-        task_id = coordinator.generate_task_id("https://example.com", "test_spider")
-        self.assertIsInstance(task_id, str)
-        self.assertEqual(len(task_id), 32)  # MD5 hash长度
-        # 测试分页任务生成
-        pagination_tasks = coordinator.generate_pagination_tasks("https://example.com/products", 1, 5)
-        self.assertEqual(len(pagination_tasks), 5)
-        # 测试任务分发
-        tasks = list(range(1, 21))  # 20个任务
-        distributed = coordinator.distribute_tasks(tasks, 4)  # 分发给4个工作节点
-        self.assertEqual(len(distributed), 4)
-if __name__ == '__main__':
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+高级工具测试
+"""
+import unittest
+from crawlo.tools import (
+    # 数据处理工具
+    clean_text,
+    format_currency,
+    validate_email,
+    validate_url,
+    check_data_integrity,
+    # 重试机制
+    RetryMechanism,
+    should_retry,
+    exponential_backoff,
+    # 反爬虫应对工具
+    AntiCrawler,
+    rotate_proxy,
+    handle_captcha,
+    detect_rate_limiting,
+    # 分布式协调工具
+    generate_pagination_tasks,
+    distribute_tasks,
+    DistributedCoordinator,
+    TaskDistributor,
+    DeduplicationTool
+)
+class TestAdvancedTools(unittest.TestCase):
+    """高级工具测试类"""
+    def test_data_processing_tools(self):
+        """测试数据处理工具"""
+        # 测试数据清洗
+        dirty_text = "<p>这是一个&nbsp;<b>测试</b>&amp;文本</p>"
+        clean_result = clean_text(dirty_text)
+        self.assertEqual(clean_result, "这是一个 测试&文本")
+        # 测试数据格式化
+        price = 1234.567
+        formatted_price = format_currency(price, "¥", 2)
+        self.assertEqual(formatted_price, "¥1,234.57")
+        # 测试字段验证
+        self.assertTrue(validate_email("test@example.com"))
+        self.assertFalse(validate_email("invalid-email"))
+        self.assertTrue(validate_url("https://example.com"))
+        self.assertFalse(validate_url("invalid-url"))
+        # 测试数据完整性检查
+        data = {
+            "name": "张三",
+            "email": "test@example.com",
+            "phone": "13812345678"
+        }
+        required_fields = ["name", "email", "phone"]
+        integrity_result = check_data_integrity(data, required_fields)
+        self.assertTrue(integrity_result["is_valid"])
+    def test_retry_mechanism(self):
+        """测试重试机制"""
+        # 测试指数退避
+        delay = exponential_backoff(0)
+        self.assertGreater(delay, 0)
+        # 测试是否应该重试
+        self.assertTrue(should_retry(status_code=500))
+        self.assertTrue(should_retry(exception=ConnectionError()))
+        self.assertFalse(should_retry(status_code=200))
+    def test_anti_crawler_tools(self):
+        """测试反爬虫应对工具"""
+        # 测试反爬虫工具
+        anti_crawler = AntiCrawler()
+        # 测试随机User-Agent
+        user_agent = anti_crawler.get_random_user_agent()
+        self.assertIsInstance(user_agent, str)
+        self.assertGreater(len(user_agent), 0)
+        # 测试代理轮换
+        proxy = anti_crawler.rotate_proxy()
+        self.assertIsInstance(proxy, dict)
+        # 测试验证码检测
+        self.assertTrue(anti_crawler.handle_captcha("请输入验证码进行验证"))
+        self.assertFalse(anti_crawler.handle_captcha("正常页面内容"))
+        # 测试频率限制检测
+        self.assertTrue(anti_crawler.detect_rate_limiting(429, {}))
+        self.assertFalse(anti_crawler.detect_rate_limiting(200, {}))
+    def test_distributed_coordinator_tools(self):
+        """测试分布式协调工具"""
+        # 测试任务分发器
+        distributor = TaskDistributor()
+        # 测试分页任务生成
+        base_url = "https://example.com/products"
+        pagination_tasks = distributor.generate_pagination_tasks(base_url, 1, 5)
+        self.assertEqual(len(pagination_tasks), 5)
+        # 测试任务分发
+        tasks = list(range(1, 21))  # 20个任务
+        distributed = distributor.distribute_tasks(tasks, 4)  # 分发给4个工作节点
+        self.assertEqual(len(distributed), 4)
+        self.assertEqual(sum(len(worker_tasks) for worker_tasks in distributed), 20)
+        # 测试去重工具
+        dedup_tool = DeduplicationTool()
+        # 测试数据指纹生成
+        fingerprint1 = dedup_tool.generate_fingerprint({"name": "test", "value": 123})
+        fingerprint2 = dedup_tool.generate_fingerprint({"name": "test", "value": 123})
+        self.assertEqual(fingerprint1, fingerprint2)
+        # 测试去重功能
+        self.assertFalse(dedup_tool.is_duplicate({"name": "test", "value": 123}))
+        self.assertTrue(dedup_tool.add_to_dedup({"name": "test", "value": 123}))
+        self.assertTrue(dedup_tool.is_duplicate({"name": "test", "value": 123}))
+        self.assertFalse(dedup_tool.add_to_dedup({"name": "test", "value": 123}))
+        # 测试分布式协调器
+        coordinator = DistributedCoordinator()
+        # 测试任务ID生成
+        task_id = coordinator.generate_task_id("https://example.com", "test_spider")
+        self.assertIsInstance(task_id, str)
+        self.assertEqual(len(task_id), 32)  # MD5 hash长度
+        # 测试分页任务生成
+        pagination_tasks = coordinator.generate_pagination_tasks("https://example.com/products", 1, 5)
+        self.assertEqual(len(pagination_tasks), 5)
+        # 测试任务分发
+        tasks = list(range(1, 21))  # 20个任务
+        distributed = coordinator.distribute_tasks(tasks, 4)  # 分发给4个工作节点
+        self.assertEqual(len(distributed), 4)
+if __name__ == '__main__':
     unittest.main()

crawlo 1.1.8__py3-none-any.whl → 1.2.0__py3-none-any.whl

Potentially problematic release.

crawlo 1.1.8py3-none-any.whl → 1.2.0py3-none-any.whl