crawlo 1.1.3__py3-none-any.whl → 1.1.5__py3-none-any.whl

tests/test_tools.py

@@ -0,0 +1,154 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+工具包测试
+"""
+import unittest
+from crawlo.tools import (
+    # 日期工具
+    parse_time,
+    format_time,
+    time_diff,
+    
+    # 数据清洗工具
+    clean_text,
+    format_currency,
+    extract_emails,
+    
+    # 数据验证工具
+    validate_email,
+    validate_url,
+    validate_phone,
+    validate_chinese_id_card,
+    validate_date,
+    validate_number_range,
+    
+    # 请求处理工具
+    build_url,
+    add_query_params,
+    merge_headers,
+    
+    # 反爬虫应对工具
+    get_random_user_agent,
+    rotate_proxy,
+    
+    # 分布式协调工具
+    generate_task_id,
+    get_cluster_info
+)
+
+
+class TestTools(unittest.TestCase):
+    """工具包测试类"""
+
+    def test_date_tools(self):
+        """测试日期工具"""
+        # 测试时间解析
+        time_str = "2025-09-10 14:30:00"
+        parsed_time = parse_time(time_str)
+        self.assertIsNotNone(parsed_time)
+        
+        # 测试时间格式化
+        formatted_time = format_time(parsed_time, "%Y-%m-%d")
+        self.assertEqual(formatted_time, "2025-09-10")
+        
+        # 测试时间差计算
+        time_str2 = "2025-09-11 14:30:00"
+        parsed_time2 = parse_time(time_str2)
+        diff = time_diff(parsed_time2, parsed_time)
+        self.assertEqual(diff, 86400)  # 24小时 = 86400秒
+
+    def test_data_cleaning_tools(self):
+        """测试数据清洗工具"""
+        # 测试文本清洗
+        dirty_text = "<p>这是一个&nbsp;<b>测试</b>&amp;文本</p>"
+        clean_result = clean_text(dirty_text)
+        self.assertEqual(clean_result, "这是一个 测试&文本")
+        
+        # 测试货币格式化
+        price = 1234.567
+        formatted_price = format_currency(price, "¥", 2)
+        self.assertEqual(formatted_price, "¥1,234.57")
+        
+        # 测试邮箱提取
+        text_with_email = "联系邮箱: test@example.com, support@crawler.com"
+        emails = extract_emails(text_with_email)
+        self.assertIn("test@example.com", emails)
+        self.assertIn("support@crawler.com", emails)
+
+    def test_data_validation_tools(self):
+        """测试数据验证工具"""
+        # 测试邮箱验证
+        self.assertTrue(validate_email("test@example.com"))
+        self.assertFalse(validate_email("invalid-email"))
+        
+        # 测试URL验证
+        self.assertTrue(validate_url("https://example.com"))
+        self.assertFalse(validate_url("invalid-url"))
+        
+        # 测试电话验证
+        self.assertTrue(validate_phone("13812345678"))
+        self.assertFalse(validate_phone("12345"))
+        
+        # 测试身份证验证
+        self.assertTrue(validate_chinese_id_card("110101199001011234"))
+        self.assertFalse(validate_chinese_id_card("invalid-id"))
+        
+        # 测试日期验证
+        self.assertTrue(validate_date("2025-09-10"))
+        self.assertFalse(validate_date("invalid-date"))
+        
+        # 测试数值范围验证
+        self.assertTrue(validate_number_range(50, 1, 100))
+        self.assertFalse(validate_number_range(150, 1, 100))
+
+    def test_request_handling_tools(self):
+        """测试请求处理工具"""
+        # 测试URL构建
+        base_url = "https://api.example.com"
+        path = "/v1/users"
+        query_params = {"page": 1, "limit": 10}
+        full_url = build_url(base_url, path, query_params)
+        self.assertIn("https://api.example.com/v1/users", full_url)
+        self.assertIn("page=1", full_url)
+        self.assertIn("limit=10", full_url)
+        
+        # 测试添加查询参数
+        existing_url = "https://api.example.com/v1/users?page=1"
+        new_params = {"sort": "name"}
+        updated_url = add_query_params(existing_url, new_params)
+        self.assertIn("sort=name", updated_url)
+        
+        # 测试合并请求头
+        base_headers = {"Content-Type": "application/json"}
+        additional_headers = {"Authorization": "Bearer token123"}
+        merged_headers = merge_headers(base_headers, additional_headers)
+        self.assertEqual(merged_headers["Content-Type"], "application/json")
+        self.assertEqual(merged_headers["Authorization"], "Bearer token123")
+
+    def test_anti_crawler_tools(self):
+        """测试反爬虫应对工具"""
+        # 测试随机User-Agent
+        user_agent = get_random_user_agent()
+        self.assertIsInstance(user_agent, str)
+        self.assertGreater(len(user_agent), 0)
+        
+        # 测试代理轮换
+        proxy = rotate_proxy()
+        self.assertIsInstance(proxy, dict)
+
+    def test_distributed_coordinator_tools(self):
+        """测试分布式协调工具"""
+        # 测试任务ID生成
+        task_id = generate_task_id("https://example.com", "test_spider")
+        self.assertIsInstance(task_id, str)
+        self.assertEqual(len(task_id), 32)  # MD5 hash长度
+        
+        # 测试集群信息获取
+        cluster_info = get_cluster_info()
+        self.assertIsInstance(cluster_info, dict)
+        self.assertIn("worker_count", cluster_info)
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/tools_example.py

@@ -0,0 +1,258 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+Crawlo框架工具包使用示例
+"""
+from crawlo.tools import (
+    # 日期工具
+    parse_time,
+    format_time,
+    time_diff,
+    
+    # 数据清洗工具
+    clean_text,
+    format_currency,
+    extract_emails,
+    
+    # 数据验证工具
+    validate_email,
+    validate_url,
+    validate_phone,
+    
+    # 请求处理工具
+    build_url,
+    add_query_params,
+    merge_headers,
+    
+    # 反爬虫应对工具
+    get_random_user_agent,
+    rotate_proxy,
+    
+    # 带认证代理工具
+    AuthenticatedProxy,
+    create_proxy_config,
+    get_proxy_info,
+    
+    # 分布式协调工具
+    generate_task_id,
+    get_cluster_info
+)
+
+
+def demo_date_tools():
+    """演示日期工具的使用"""
+    print("=== 日期工具演示 ===\n")
+    
+    # 解析时间
+    time_str = "2025-09-10 14:30:00"
+    parsed_time = parse_time(time_str)
+    print(f"解析时间: {time_str} -> {parsed_time}")
+    
+    # 格式化时间
+    formatted_time = format_time(parsed_time, "%Y年%m月%d日 %H:%M:%S")
+    print(f"格式化时间: {parsed_time} -> {formatted_time}")
+    
+    # 时间差计算
+    time_str2 = "2025-09-11 16:45:30"
+    parsed_time2 = parse_time(time_str2)
+    diff = time_diff(parsed_time2, parsed_time)
+    print(f"时间差: {time_str2} - {time_str} = {diff} 秒")
+    
+    print()
+
+
+def demo_data_cleaning_tools():
+    """演示数据清洗工具的使用"""
+    print("=== 数据清洗工具演示 ===\n")
+    
+    # 清洗文本
+    dirty_text = "<p>这是一个&nbsp;<b>测试</b>&amp;文本</p>"
+    clean_result = clean_text(dirty_text)
+    print(f"清洗文本: {dirty_text} -> {clean_result}")
+    
+    # 格式化货币
+    price = 1234.567
+    formatted_price = format_currency(price, "¥", 2)
+    print(f"格式化货币: {price} -> {formatted_price}")
+    
+    # 提取邮箱
+    text_with_email = "联系邮箱: test@example.com, support@crawler.com"
+    emails = extract_emails(text_with_email)
+    print(f"提取邮箱: {text_with_email} -> {emails}")
+    
+    print()
+
+
+def demo_data_validation_tools():
+    """演示数据验证工具的使用"""
+    print("=== 数据验证工具演示 ===\n")
+    
+    # 验证邮箱
+    email = "test@example.com"
+    is_valid_email = validate_email(email)
+    print(f"验证邮箱: {email} -> {'有效' if is_valid_email else '无效'}")
+    
+    # 验证无效邮箱
+    invalid_email = "invalid-email"
+    is_valid_invalid = validate_email(invalid_email)
+    print(f"验证邮箱: {invalid_email} -> {'有效' if is_valid_invalid else '无效'}")
+    
+    # 验证URL
+    url = "https://example.com/path?param=value"
+    is_valid_url = validate_url(url)
+    print(f"验证URL: {url} -> {'有效' if is_valid_url else '无效'}")
+    
+    # 验证电话号码
+    phone = "13812345678"
+    is_valid_phone = validate_phone(phone)
+    print(f"验证电话: {phone} -> {'有效' if is_valid_phone else '无效'}")
+    
+    print()
+
+
+def demo_request_handling_tools():
+    """演示请求处理工具的使用"""
+    print("=== 请求处理工具演示 ===\n")
+    
+    # 构建URL
+    base_url = "https://api.example.com"
+    path = "/v1/users"
+    query_params = {"page": 1, "limit": 10}
+    full_url = build_url(base_url, path, query_params)
+    print(f"构建URL: {base_url} + {path} + {query_params} -> {full_url}")
+    
+    # 添加查询参数
+    existing_url = "https://api.example.com/v1/users?page=1"
+    new_params = {"sort": "name", "order": "asc"}
+    updated_url = add_query_params(existing_url, new_params)
+    print(f"添加参数: {existing_url} + {new_params} -> {updated_url}")
+    
+    # 合并请求头
+    base_headers = {"Content-Type": "application/json", "Accept": "application/json"}
+    additional_headers = {"Authorization": "Bearer token123", "User-Agent": "Crawlo/1.0"}
+    merged_headers = merge_headers(base_headers, additional_headers)
+    print(f"合并请求头:")
+    print(f"  基础头: {base_headers}")
+    print(f"  额外头: {additional_headers}")
+    print(f"  合并后: {merged_headers}")
+    
+    print()
+
+
+def demo_anti_crawler_tools():
+    """演示反爬虫应对工具的使用"""
+    print("=== 反爬虫应对工具演示 ===\n")
+    
+    # 获取随机User-Agent
+    user_agent = get_random_user_agent()
+    print(f"随机User-Agent: {user_agent[:50]}...")
+    
+    # 轮换代理
+    proxy = rotate_proxy()
+    print(f"轮换代理: {proxy}")
+    
+    print()
+
+
+def demo_authenticated_proxy_tools():
+    """演示带认证代理工具的使用"""
+    print("=== 带认证代理工具演示 ===\n")
+    
+    # 创建带认证的代理
+    proxy_url = "http://username:password@proxy.example.com:8080"
+    proxy = AuthenticatedProxy(proxy_url)
+    
+    print(f"代理URL: {proxy}")
+    print(f"清洁URL: {proxy.clean_url}")
+    print(f"用户名: {proxy.username}")
+    print(f"密码: {proxy.password}")
+    print(f"代理字典: {proxy.proxy_dict}")
+    print(f"认证凭据: {proxy.get_auth_credentials()}")
+    print(f"认证头: {proxy.get_auth_header()}")
+    print(f"是否有效: {proxy.is_valid()}")
+    
+    # 创建代理配置
+    proxy_config = create_proxy_config(proxy_url)
+    print(f"\n代理配置: {proxy_config}")
+    
+    # 获取代理信息
+    proxy_info = get_proxy_info(proxy_url)
+    print(f"代理信息: {proxy_info}")
+    
+    print()
+
+
+def demo_distributed_coordinator_tools():
+    """演示分布式协调工具的使用"""
+    print("=== 分布式协调工具演示 ===\n")
+    
+    # 生成任务ID
+    url = "https://example.com/page/1"
+    spider_name = "example_spider"
+    task_id = generate_task_id(url, spider_name)
+    print(f"生成任务ID: URL={url}, Spider={spider_name} -> {task_id}")
+    
+    # 获取集群信息
+    cluster_info = get_cluster_info()
+    print(f"集群信息: {cluster_info}")
+    
+    print()
+
+
+if __name__ == '__main__':
+    # 运行演示
+    demo_date_tools()
+    demo_data_cleaning_tools()
+    demo_data_validation_tools()
+    demo_request_handling_tools()
+    demo_anti_crawler_tools()
+    demo_authenticated_proxy_tools()
+    demo_distributed_coordinator_tools()
+    
+    print("=== 在爬虫中使用工具包 ===\n")
+    print("在爬虫项目中，您可以这样使用工具包:")
+    print("""
+from crawlo import Spider, Request
+from crawlo.tools import (
+    clean_text, 
+    validate_email, 
+    get_random_user_agent,
+    build_url,
+    AuthenticatedProxy
+)
+
+class ExampleSpider(Spider):
+    def start_requests(self):
+        headers = {"User-Agent": get_random_user_agent()}
+        
+        # 使用带认证的代理
+        proxy_url = "http://username:password@proxy.example.com:8080"
+        proxy = AuthenticatedProxy(proxy_url)
+        
+        request = Request("https://example.com", headers=headers)
+        # 根据下载器类型设置代理
+        downloader_type = self.crawler.settings.get("DOWNLOADER_TYPE", "aiohttp")
+        if downloader_type == "aiohttp":
+            request.proxy = proxy.clean_url
+            auth = proxy.get_auth_credentials()
+            if auth:
+                request.meta["proxy_auth"] = auth
+        else:
+            request.proxy = proxy.proxy_dict
+            
+        yield request
+    
+    def parse(self, response):
+        # 提取数据
+        title = response.css('h1::text').get()
+        email = response.css('.email::text').get()
+        
+        # 清洗和验证数据
+        clean_title = clean_text(title) if title else None
+        is_valid_email = validate_email(email) if email else False
+        
+        # 构建下一页URL
+        next_page_url = build_url("https://example.com", "/page/2")
+        
+        # 处理数据...
+    """)

crawlo/core/enhanced_engine.py

@@ -1,190 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-增强的引擎实现
-解决大规模请求生成时的并发控制和背压问题
-"""
-import asyncio
-
-from crawlo.core.engine import Engine as BaseEngine
-from crawlo.utils.log import get_logger
-
-
-class EnhancedEngine(BaseEngine):
-    """
-    增强的引擎实现
-    
-    主要改进：
-    1. 智能的请求生成控制
-    2. 背压感知的调度
-    3. 动态并发调整
-    """
-    
-    def __init__(self, crawler):
-        super().__init__(crawler)
-        
-        # 增强控制参数
-        self.max_queue_size = self.settings.get_int('SCHEDULER_MAX_QUEUE_SIZE', 200)
-        self.generation_batch_size = 10
-        self.generation_interval = 0.05
-        self.backpressure_ratio = 0.8  # 队列达到80%时启动背压
-        
-        # 状态跟踪
-        self._generation_paused = False
-        self._last_generation_time = 0
-        self._generation_stats = {
-            'total_generated': 0,
-            'backpressure_events': 0
-        }
-        
-        self.logger = get_logger(self.__class__.__name__)
-    
-    async def crawl(self):
-        """
-        增强的爬取循环
-        支持智能请求生成和背压控制
-        """
-        generation_task = None
-        
-        try:
-            # 启动请求生成任务
-            if self.start_requests:
-                generation_task = asyncio.create_task(
-                    self._controlled_request_generation()
-                )
-            
-            # 主爬取循环
-            while self.running:
-                # 获取并处理请求
-                if request := await self._get_next_request():
-                    await self._crawl(request)
-                
-                # 检查退出条件
-                if await self._should_exit():
-                    break
-                
-                # 短暂休息避免忙等
-                await asyncio.sleep(0.001)
-        
-        finally:
-            # 清理生成任务
-            if generation_task and not generation_task.done():
-                generation_task.cancel()
-                try:
-                    await generation_task
-                except asyncio.CancelledError:
-                    pass
-            
-            await self.close_spider()
-    
-    async def _controlled_request_generation(self):
-        """受控的请求生成"""
-        self.logger.info("🎛️ 启动受控请求生成")
-        
-        batch = []
-        total_generated = 0
-        
-        try:
-            for request in self.start_requests:
-                batch.append(request)
-                
-                # 批量处理
-                if len(batch) >= self.generation_batch_size:
-                    generated = await self._process_generation_batch(batch)
-                    total_generated += generated
-                    batch = []
-                
-                # 背压检查
-                if await self._should_pause_generation():
-                    await self._wait_for_capacity()
-            
-            # 处理剩余请求
-            if batch:
-                generated = await self._process_generation_batch(batch)
-                total_generated += generated
-        
-        except Exception as e:
-            self.logger.error(f"❌ 请求生成失败: {e}")
-        
-        finally:
-            self.start_requests = None
-            self.logger.info(f"🎉 请求生成完成，总计: {total_generated}")
-    
-    async def _process_generation_batch(self, batch) -> int:
-        """处理一批请求"""
-        generated = 0
-        
-        for request in batch:
-            if not self.running:
-                break
-            
-            # 等待队列有空间
-            while await self._is_queue_full() and self.running:
-                await asyncio.sleep(0.1)
-            
-            if self.running:
-                await self.enqueue_request(request)
-                generated += 1
-                self._generation_stats['total_generated'] += 1
-            
-            # 控制生成速度
-            if self.generation_interval > 0:
-                await asyncio.sleep(self.generation_interval)
-        
-        return generated
-    
-    async def _should_pause_generation(self) -> bool:
-        """判断是否应该暂停生成"""
-        # 检查队列大小
-        if await self._is_queue_full():
-            return True
-        
-        # 检查任务管理器负载
-        if self.task_manager:
-            current_tasks = len(self.task_manager.current_task)
-            if hasattr(self.task_manager, 'semaphore'):
-                max_concurrency = getattr(self.task_manager.semaphore, '_initial_value', 8)
-                if current_tasks >= max_concurrency * self.backpressure_ratio:
-                    return True
-        
-        return False
-    
-    async def _is_queue_full(self) -> bool:
-        """检查队列是否已满"""
-        if not self.scheduler:
-            return False
-        
-        queue_size = len(self.scheduler)
-        return queue_size >= self.max_queue_size * self.backpressure_ratio
-    
-    async def _wait_for_capacity(self):
-        """等待系统有足够容量"""
-        self._generation_stats['backpressure_events'] += 1
-        self.logger.debug("⏸️ 触发背压，暂停请求生成")
-        
-        wait_time = 0.1
-        max_wait = 2.0
-        
-        while await self._should_pause_generation() and self.running:
-            await asyncio.sleep(wait_time)
-            wait_time = min(wait_time * 1.1, max_wait)
-    
-    async def _should_exit(self) -> bool:
-        """检查是否应该退出"""
-        # 没有启动请求，且所有队列都空闲
-        if (self.start_requests is None and 
-            self.scheduler.idle() and 
-            self.downloader.idle() and 
-            self.task_manager.all_done() and 
-            self.processor.idle()):
-            return True
-        
-        return False
-    
-    def get_generation_stats(self) -> dict:
-        """获取生成统计"""
-        return {
-            **self._generation_stats,
-            'queue_size': len(self.scheduler) if self.scheduler else 0,
-            'active_tasks': len(self.task_manager.current_task) if self.task_manager else 0
-        }