crawlo 1.1.4__py3-none-any.whl → 1.1.6__py3-none-any.whl

tests/tools_example.py

@@ -0,0 +1,258 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+Crawlo框架工具包使用示例
+"""
+from crawlo.tools import (
+    # 日期工具
+    parse_time,
+    format_time,
+    time_diff,
+    
+    # 数据清洗工具
+    clean_text,
+    format_currency,
+    extract_emails,
+    
+    # 数据验证工具
+    validate_email,
+    validate_url,
+    validate_phone,
+    
+    # 请求处理工具
+    build_url,
+    add_query_params,
+    merge_headers,
+    
+    # 反爬虫应对工具
+    get_random_user_agent,
+    rotate_proxy,
+    
+    # 带认证代理工具
+    AuthenticatedProxy,
+    create_proxy_config,
+    get_proxy_info,
+    
+    # 分布式协调工具
+    generate_task_id,
+    get_cluster_info
+)
+
+
+def demo_date_tools():
+    """演示日期工具的使用"""
+    print("=== 日期工具演示 ===\n")
+    
+    # 解析时间
+    time_str = "2025-09-10 14:30:00"
+    parsed_time = parse_time(time_str)
+    print(f"解析时间: {time_str} -> {parsed_time}")
+    
+    # 格式化时间
+    formatted_time = format_time(parsed_time, "%Y年%m月%d日 %H:%M:%S")
+    print(f"格式化时间: {parsed_time} -> {formatted_time}")
+    
+    # 时间差计算
+    time_str2 = "2025-09-11 16:45:30"
+    parsed_time2 = parse_time(time_str2)
+    diff = time_diff(parsed_time2, parsed_time)
+    print(f"时间差: {time_str2} - {time_str} = {diff} 秒")
+    
+    print()
+
+
+def demo_data_cleaning_tools():
+    """演示数据清洗工具的使用"""
+    print("=== 数据清洗工具演示 ===\n")
+    
+    # 清洗文本
+    dirty_text = "<p>这是一个&nbsp;<b>测试</b>&amp;文本</p>"
+    clean_result = clean_text(dirty_text)
+    print(f"清洗文本: {dirty_text} -> {clean_result}")
+    
+    # 格式化货币
+    price = 1234.567
+    formatted_price = format_currency(price, "¥", 2)
+    print(f"格式化货币: {price} -> {formatted_price}")
+    
+    # 提取邮箱
+    text_with_email = "联系邮箱: test@example.com, support@crawler.com"
+    emails = extract_emails(text_with_email)
+    print(f"提取邮箱: {text_with_email} -> {emails}")
+    
+    print()
+
+
+def demo_data_validation_tools():
+    """演示数据验证工具的使用"""
+    print("=== 数据验证工具演示 ===\n")
+    
+    # 验证邮箱
+    email = "test@example.com"
+    is_valid_email = validate_email(email)
+    print(f"验证邮箱: {email} -> {'有效' if is_valid_email else '无效'}")
+    
+    # 验证无效邮箱
+    invalid_email = "invalid-email"
+    is_valid_invalid = validate_email(invalid_email)
+    print(f"验证邮箱: {invalid_email} -> {'有效' if is_valid_invalid else '无效'}")
+    
+    # 验证URL
+    url = "https://example.com/path?param=value"
+    is_valid_url = validate_url(url)
+    print(f"验证URL: {url} -> {'有效' if is_valid_url else '无效'}")
+    
+    # 验证电话号码
+    phone = "13812345678"
+    is_valid_phone = validate_phone(phone)
+    print(f"验证电话: {phone} -> {'有效' if is_valid_phone else '无效'}")
+    
+    print()
+
+
+def demo_request_handling_tools():
+    """演示请求处理工具的使用"""
+    print("=== 请求处理工具演示 ===\n")
+    
+    # 构建URL
+    base_url = "https://api.example.com"
+    path = "/v1/users"
+    query_params = {"page": 1, "limit": 10}
+    full_url = build_url(base_url, path, query_params)
+    print(f"构建URL: {base_url} + {path} + {query_params} -> {full_url}")
+    
+    # 添加查询参数
+    existing_url = "https://api.example.com/v1/users?page=1"
+    new_params = {"sort": "name", "order": "asc"}
+    updated_url = add_query_params(existing_url, new_params)
+    print(f"添加参数: {existing_url} + {new_params} -> {updated_url}")
+    
+    # 合并请求头
+    base_headers = {"Content-Type": "application/json", "Accept": "application/json"}
+    additional_headers = {"Authorization": "Bearer token123", "User-Agent": "Crawlo/1.0"}
+    merged_headers = merge_headers(base_headers, additional_headers)
+    print(f"合并请求头:")
+    print(f"  基础头: {base_headers}")
+    print(f"  额外头: {additional_headers}")
+    print(f"  合并后: {merged_headers}")
+    
+    print()
+
+
+def demo_anti_crawler_tools():
+    """演示反爬虫应对工具的使用"""
+    print("=== 反爬虫应对工具演示 ===\n")
+    
+    # 获取随机User-Agent
+    user_agent = get_random_user_agent()
+    print(f"随机User-Agent: {user_agent[:50]}...")
+    
+    # 轮换代理
+    proxy = rotate_proxy()
+    print(f"轮换代理: {proxy}")
+    
+    print()
+
+
+def demo_authenticated_proxy_tools():
+    """演示带认证代理工具的使用"""
+    print("=== 带认证代理工具演示 ===\n")
+    
+    # 创建带认证的代理
+    proxy_url = "http://username:password@proxy.example.com:8080"
+    proxy = AuthenticatedProxy(proxy_url)
+    
+    print(f"代理URL: {proxy}")
+    print(f"清洁URL: {proxy.clean_url}")
+    print(f"用户名: {proxy.username}")
+    print(f"密码: {proxy.password}")
+    print(f"代理字典: {proxy.proxy_dict}")
+    print(f"认证凭据: {proxy.get_auth_credentials()}")
+    print(f"认证头: {proxy.get_auth_header()}")
+    print(f"是否有效: {proxy.is_valid()}")
+    
+    # 创建代理配置
+    proxy_config = create_proxy_config(proxy_url)
+    print(f"\n代理配置: {proxy_config}")
+    
+    # 获取代理信息
+    proxy_info = get_proxy_info(proxy_url)
+    print(f"代理信息: {proxy_info}")
+    
+    print()
+
+
+def demo_distributed_coordinator_tools():
+    """演示分布式协调工具的使用"""
+    print("=== 分布式协调工具演示 ===\n")
+    
+    # 生成任务ID
+    url = "https://example.com/page/1"
+    spider_name = "example_spider"
+    task_id = generate_task_id(url, spider_name)
+    print(f"生成任务ID: URL={url}, Spider={spider_name} -> {task_id}")
+    
+    # 获取集群信息
+    cluster_info = get_cluster_info()
+    print(f"集群信息: {cluster_info}")
+    
+    print()
+
+
+if __name__ == '__main__':
+    # 运行演示
+    demo_date_tools()
+    demo_data_cleaning_tools()
+    demo_data_validation_tools()
+    demo_request_handling_tools()
+    demo_anti_crawler_tools()
+    demo_authenticated_proxy_tools()
+    demo_distributed_coordinator_tools()
+    
+    print("=== 在爬虫中使用工具包 ===\n")
+    print("在爬虫项目中，您可以这样使用工具包:")
+    print("""
+from crawlo import Spider, Request
+from crawlo.tools import (
+    clean_text, 
+    validate_email, 
+    get_random_user_agent,
+    build_url,
+    AuthenticatedProxy
+)
+
+class ExampleSpider(Spider):
+    def start_requests(self):
+        headers = {"User-Agent": get_random_user_agent()}
+        
+        # 使用带认证的代理
+        proxy_url = "http://username:password@proxy.example.com:8080"
+        proxy = AuthenticatedProxy(proxy_url)
+        
+        request = Request("https://example.com", headers=headers)
+        # 根据下载器类型设置代理
+        downloader_type = self.crawler.settings.get("DOWNLOADER_TYPE", "aiohttp")
+        if downloader_type == "aiohttp":
+            request.proxy = proxy.clean_url
+            auth = proxy.get_auth_credentials()
+            if auth:
+                request.meta["proxy_auth"] = auth
+        else:
+            request.proxy = proxy.proxy_dict
+            
+        yield request
+    
+    def parse(self, response):
+        # 提取数据
+        title = response.css('h1::text').get()
+        email = response.css('.email::text').get()
+        
+        # 清洗和验证数据
+        clean_title = clean_text(title) if title else None
+        is_valid_email = validate_email(email) if email else False
+        
+        # 构建下一页URL
+        next_page_url = build_url("https://example.com", "/page/2")
+        
+        # 处理数据...
+    """)

crawlo/core/enhanced_engine.py

@@ -1,190 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-增强的引擎实现
-解决大规模请求生成时的并发控制和背压问题
-"""
-import asyncio
-
-from crawlo.core.engine import Engine as BaseEngine
-from crawlo.utils.log import get_logger
-
-
-class EnhancedEngine(BaseEngine):
-    """
-    增强的引擎实现
-    
-    主要改进：
-    1. 智能的请求生成控制
-    2. 背压感知的调度
-    3. 动态并发调整
-    """
-    
-    def __init__(self, crawler):
-        super().__init__(crawler)
-        
-        # 增强控制参数
-        self.max_queue_size = self.settings.get_int('SCHEDULER_MAX_QUEUE_SIZE', 200)
-        self.generation_batch_size = 10
-        self.generation_interval = 0.05
-        self.backpressure_ratio = 0.8  # 队列达到80%时启动背压
-        
-        # 状态跟踪
-        self._generation_paused = False
-        self._last_generation_time = 0
-        self._generation_stats = {
-            'total_generated': 0,
-            'backpressure_events': 0
-        }
-        
-        self.logger = get_logger(self.__class__.__name__)
-    
-    async def crawl(self):
-        """
-        增强的爬取循环
-        支持智能请求生成和背压控制
-        """
-        generation_task = None
-        
-        try:
-            # 启动请求生成任务
-            if self.start_requests:
-                generation_task = asyncio.create_task(
-                    self._controlled_request_generation()
-                )
-            
-            # 主爬取循环
-            while self.running:
-                # 获取并处理请求
-                if request := await self._get_next_request():
-                    await self._crawl(request)
-                
-                # 检查退出条件
-                if await self._should_exit():
-                    break
-                
-                # 短暂休息避免忙等
-                await asyncio.sleep(0.001)
-        
-        finally:
-            # 清理生成任务
-            if generation_task and not generation_task.done():
-                generation_task.cancel()
-                try:
-                    await generation_task
-                except asyncio.CancelledError:
-                    pass
-            
-            await self.close_spider()
-    
-    async def _controlled_request_generation(self):
-        """受控的请求生成"""
-        self.logger.info("🎛️ 启动受控请求生成")
-        
-        batch = []
-        total_generated = 0
-        
-        try:
-            for request in self.start_requests:
-                batch.append(request)
-                
-                # 批量处理
-                if len(batch) >= self.generation_batch_size:
-                    generated = await self._process_generation_batch(batch)
-                    total_generated += generated
-                    batch = []
-                
-                # 背压检查
-                if await self._should_pause_generation():
-                    await self._wait_for_capacity()
-            
-            # 处理剩余请求
-            if batch:
-                generated = await self._process_generation_batch(batch)
-                total_generated += generated
-        
-        except Exception as e:
-            self.logger.error(f"❌ 请求生成失败: {e}")
-        
-        finally:
-            self.start_requests = None
-            self.logger.info(f"🎉 请求生成完成，总计: {total_generated}")
-    
-    async def _process_generation_batch(self, batch) -> int:
-        """处理一批请求"""
-        generated = 0
-        
-        for request in batch:
-            if not self.running:
-                break
-            
-            # 等待队列有空间
-            while await self._is_queue_full() and self.running:
-                await asyncio.sleep(0.1)
-            
-            if self.running:
-                await self.enqueue_request(request)
-                generated += 1
-                self._generation_stats['total_generated'] += 1
-            
-            # 控制生成速度
-            if self.generation_interval > 0:
-                await asyncio.sleep(self.generation_interval)
-        
-        return generated
-    
-    async def _should_pause_generation(self) -> bool:
-        """判断是否应该暂停生成"""
-        # 检查队列大小
-        if await self._is_queue_full():
-            return True
-        
-        # 检查任务管理器负载
-        if self.task_manager:
-            current_tasks = len(self.task_manager.current_task)
-            if hasattr(self.task_manager, 'semaphore'):
-                max_concurrency = getattr(self.task_manager.semaphore, '_initial_value', 8)
-                if current_tasks >= max_concurrency * self.backpressure_ratio:
-                    return True
-        
-        return False
-    
-    async def _is_queue_full(self) -> bool:
-        """检查队列是否已满"""
-        if not self.scheduler:
-            return False
-        
-        queue_size = len(self.scheduler)
-        return queue_size >= self.max_queue_size * self.backpressure_ratio
-    
-    async def _wait_for_capacity(self):
-        """等待系统有足够容量"""
-        self._generation_stats['backpressure_events'] += 1
-        self.logger.debug("⏸️ 触发背压，暂停请求生成")
-        
-        wait_time = 0.1
-        max_wait = 2.0
-        
-        while await self._should_pause_generation() and self.running:
-            await asyncio.sleep(wait_time)
-            wait_time = min(wait_time * 1.1, max_wait)
-    
-    async def _should_exit(self) -> bool:
-        """检查是否应该退出"""
-        # 没有启动请求，且所有队列都空闲
-        if (self.start_requests is None and 
-            self.scheduler.idle() and 
-            self.downloader.idle() and 
-            self.task_manager.all_done() and 
-            self.processor.idle()):
-            return True
-        
-        return False
-    
-    def get_generation_stats(self) -> dict:
-        """获取生成统计"""
-        return {
-            **self._generation_stats,
-            'queue_size': len(self.scheduler) if self.scheduler else 0,
-            'active_tasks': len(self.task_manager.current_task) if self.task_manager else 0
-        }

crawlo-1.1.4.dist-info/RECORD

@@ -1,117 +0,0 @@
-crawlo/__init__.py,sha256=esOolburYDjtF43D5N9Kh6TSQW2yKcz888ilhBSinBc,825
-crawlo/__version__.py,sha256=XxXhu8-QnuD9hA8Ah0WX5rgpt_DwOQmAwcK-FtpngyQ,22
-crawlo/cli.py,sha256=CtR2Pfa7SyRxEKPaXqt-6E6K5Vq5z3rfdAI95UO4cbU,1166
-crawlo/config.py,sha256=i0Amz6wNPgv_aVcdCBRRlcwuZLSa87cH9OEmTQvB97Q,8329
-crawlo/crawler.py,sha256=v6i5tjgSOtbMoqiw1qdgKx1cY4kcVcd5l5bUTWtJNNU,36461
-crawlo/event.py,sha256=7-y6HNv_EIJSYQNzsj0mVK-Gg4ON3wdQeMdQjfFJPlw,313
-crawlo/exceptions.py,sha256=sMay0wnWLfc_FXWslqxm60qz6b66LXs3EdN_w8ygE9k,1166
-crawlo/mode_manager.py,sha256=WIxrq9S3EAH0D71LH1AxvcqXomeABqoXgtUN4A--DKY,6702
-crawlo/project.py,sha256=xWN2eTAjf_Pza-wWvvV4JjScQRWxe9hXlztX81ccUMc,5182
-crawlo/stats_collector.py,sha256=NkO09CB-220qz5rxFcD_dedGfr2VPFrDo4hya0Zh8Qc,1577
-crawlo/subscriber.py,sha256=gioTIqRdEwVG-bwIiQonbk1vWWAqTh9hzVkrqZ1AfP0,5006
-crawlo/task_manager.py,sha256=19InAxS9oJ9EMj20Aw2urN-v6BeC22dkgPuW-B9-4UI,819
-crawlo/commands/__init__.py,sha256=AMYjXG7ulE8dPVmgWVo0uqXsaCYUUZYmmu2-7kFzH1M,342
-crawlo/commands/check.py,sha256=172OiAxnX5wwSlszUsyPgMZwAoIbGDTdfhtRz309ilc,22843
-crawlo/commands/genspider.py,sha256=-jGJdfXLsefX_H1ydQ2wirdu6p6wmhClzVXY_0L-1aE,5050
-crawlo/commands/list.py,sha256=yByqQeZBgvjewOKxpnOobpeJ7Hnbs-CWsoyITqZu2ZY,5781
-crawlo/commands/run.py,sha256=8Qngjsl8Q4RBdO39a__wKGsheY2PFuPit2hds_jwEbM,10524
-crawlo/commands/startproject.py,sha256=bzNgpkKzUEggY2m7Iw810mSPe8wOPFBqSCO0jZX3z_g,7138
-crawlo/commands/stats.py,sha256=6pAgkEi8MBnCer2rWmKpaTYr1jaM6HeMG9owAvEzJyY,6064
-crawlo/commands/utils.py,sha256=nohMvUU2zLvX0XzXk6KeCNxP0EvSWj9DiVLxM_7tD5o,5106
-crawlo/core/__init__.py,sha256=PnFyJdVNHBoPmV1sW0AHQXijeoSTQ8cMYrbNM1JK8kA,41
-crawlo/core/engine.py,sha256=8Dcew1XyxChW5Fz1wFEWpJlPrQb2hKDWKul8e61S-Q0,6662
-crawlo/core/enhanced_engine.py,sha256=9I9Uxdy2oAz8zDGTzEiytuKu__VDVmIN8zwZKfrD8bw,6254
-crawlo/core/processor.py,sha256=qmCqAeqhwYu-UE86evYesaGt9qpuSIfH-ZIZKcXFCZc,1140
-crawlo/core/scheduler.py,sha256=fiU-Q-lzyC3B6ih8NSWqjP1Xw_ryNVb_4dLUARtWRBE,5804
-crawlo/downloader/__init__.py,sha256=tl0mE54reR-PuJYSsXsKP2VY5uzvq4lITxZwKKjNzPs,7663
-crawlo/downloader/aiohttp_downloader.py,sha256=UKupGYPOWrscAVsjhFgKYElTa9tbEeltqV7nuWqjIeE,8005
-crawlo/downloader/cffi_downloader.py,sha256=-GVfSIhi1Ip56suSiGf8jnUE2EBF1P56vw0uxLh_T6I,10440
-crawlo/downloader/httpx_downloader.py,sha256=7jfQfvAtfk8yD_mvwUbWLhYOxMM7r1nudiU7m_Jl9wc,12037
-crawlo/extension/__init__.py,sha256=Sg588p6UhyrwFNTiD2wqGW-i3xgLX6HlLuQPKT7mayE,1526
-crawlo/extension/health_check.py,sha256=IVaaVo_0CcZtf1LoCAYXIBvs3wZ7hdmT6U4-NYWAgP0,5527
-crawlo/extension/log_interval.py,sha256=VCIeNqXcWDnxj4m6l77cjqgRzV8LfsPMb22X0Xc1Vwc,2417
-crawlo/extension/log_stats.py,sha256=Ssxz6R1YpWIj5WJvQ2cJ9F5oR7FUFdj-ITc9lV92SSU,2908
-crawlo/extension/logging_extension.py,sha256=ET6VAu1J2qNMz4NnG1G3zQLRhbsvV7l6xRIuQLE6DaE,1626
-crawlo/extension/memory_monitor.py,sha256=gg-GK5RD9XhnrN_zp3KTmPKyWDmKLMv_OTY-HxSxBNI,3664
-crawlo/extension/performance_profiler.py,sha256=NvQuuvE83dXJ-1URpN8OF9za9J1l7xhVbV22JynPQpA,4235
-crawlo/extension/request_recorder.py,sha256=RC23yzXClnVv9j2ljQvjBkUfWznfnDHsrQejKhE9y5E,4074
-crawlo/filters/__init__.py,sha256=XC_Q4ykZtSNYizYlAcehVwBBNO3lZ2zuWwafzXiuWyQ,4241
-crawlo/filters/aioredis_filter.py,sha256=WhkFZcVAym9wLSUa8WTVctYfEibjxG42umtmacO1IY0,8370
-crawlo/filters/memory_filter.py,sha256=VJO0UFRYGxmV8dj4G1subsQ-FtvPcGLbvd7IVtqXnOs,9260
-crawlo/items/__init__.py,sha256=bqekZrRlDhxfWie0UbCs656TptYseoe9QJ67I4E7Elk,386
-crawlo/items/base.py,sha256=tAYrPJgblp3ZEihDXvappdYc6pGdim6x2_9QSmMKI2o,577
-crawlo/items/fields.py,sha256=wMlakQTsEwyrlLzMt1gI4pScLQZMqd3E1xcfH4dbSqk,1801
-crawlo/items/items.py,sha256=e-3nXI9ckD64vcDxxQiAU6ufbtJMs09gbZQcYjxgwHY,3374
-crawlo/middleware/__init__.py,sha256=ldaGFNbiJnK9Fx12Vdf9fDNfzXxoETtShp5r-vodtw0,549
-crawlo/middleware/default_header.py,sha256=i_Uj07JObyeZFxL7ZAZmvZsHvA1HGtkNab1sA0d-nWI,1067
-crawlo/middleware/download_delay.py,sha256=2M-TchDA7MwyTfYy0Hzh_bW9wlHlpiP-oQlys7crTj0,966
-crawlo/middleware/middleware_manager.py,sha256=j1hkWRFB5rnC5SnB7oXWE5eUNv8blS9krDIDM5fIDs8,6213
-crawlo/middleware/proxy.py,sha256=m2ZZ50En9hUtgrqSqA6hItGT74xMqccHFPhZshutIco,9811
-crawlo/middleware/request_ignore.py,sha256=QI2z4fUnJ-4xvPTZAmsL-GqR4RFHS1xq9iDr5KFrMco,997
-crawlo/middleware/response_code.py,sha256=tmef2QVl3JCiTMii6VQkASlOY2OyqmOPoOfNxIK1eF8,659
-crawlo/middleware/response_filter.py,sha256=ep8ZxDlfIefi9YqK8dPASEp5TTDRo9QEY_jMceC411s,837
-crawlo/middleware/retry.py,sha256=-7zpRURugiTTm4QYUSUlbnURD5mcT2Ji0yHvCgY1wGc,4124
-crawlo/network/__init__.py,sha256=BLPERYPo22g1BXrW--wUnlolrdFUmOPjgOB8XQQJlck,397
-crawlo/network/request.py,sha256=tPAiOVJyF3Kk-midqydTGXgv5M5tsYJRtwUUJTrUsrE,11075
-crawlo/network/response.py,sha256=cUvdjsB2cQ-qWEKHNGIkwWGgCg-EnQ81xTrjrUOVno0,9738
-crawlo/pipelines/__init__.py,sha256=lrdVDjeHLNkA4_MAwI1auk_I9xfeU1SlBWXiammb6lc,616
-crawlo/pipelines/bloom_dedup_pipeline.py,sha256=QQxGFGEoMHN4Vx2kq7G_i1o9pmuXp8clZebilOar3fk,5642
-crawlo/pipelines/console_pipeline.py,sha256=KABkR3J-rqO0Awox7lizxKR2XuHfVhWPiVRgIybwwu4,1248
-crawlo/pipelines/csv_pipeline.py,sha256=6FBT2AoU6iNU-5NfgWRq7-JpF9dK2nBokjxx-y4jIas,12174
-crawlo/pipelines/database_dedup_pipeline.py,sha256=wVBXEGArFR3uxoN7yfJSOarBmtGrJpOqowAqa7OUs98,8000
-crawlo/pipelines/json_pipeline.py,sha256=vlu1nqbD2mtqtExt9cL5nibx1CwJM1RNqd4WGjZRHAY,8367
-crawlo/pipelines/memory_dedup_pipeline.py,sha256=5jeL2jEq7sioYmXlzfkx-LNSbWyChrXeWx8d15YEZOA,3839
-crawlo/pipelines/mongo_pipeline.py,sha256=k7gNqAO-g2MtIfArphC6z5ZzkKVRkBKcv-2ImziPFA0,5706
-crawlo/pipelines/mysql_pipeline.py,sha256=cwgJvRORTRea_Eep2coBaMf3G8PQVTQA1qrnIlDZApc,13480
-crawlo/pipelines/pipeline_manager.py,sha256=VrbebOYiqrobtKhp5II18w-odCICdWkmRg5WPK0Emz4,2112
-crawlo/pipelines/redis_dedup_pipeline.py,sha256=TaokJ4wP5-Cxf-ueFJdh4SX58hchT0QzZ5RBDXHDN64,6003
-crawlo/queue/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-crawlo/queue/pqueue.py,sha256=yzF900ko2RReVNZtfk_VV3PzSXEUALI6SHf97geuu6s,1058
-crawlo/queue/queue_manager.py,sha256=NMR0Fo8XFBg6_g7htq4D80cS6Ilo0EKt5QtyF-KxNuM,11467
-crawlo/queue/redis_priority_queue.py,sha256=boJCKqcKxRw9XCCzaHy5qmrX9DvzPiQBzOkBHI5odfE,8116
-crawlo/settings/__init__.py,sha256=xsukVKn_h2Hopm1Nj-bXkhbfyS62QTTvJi7fhZUwR9M,123
-crawlo/settings/default_settings.py,sha256=B4_61tNJvqzVvyqt9AtRV7Iq5q8G4pJOExcN0ve7S_A,11559
-crawlo/settings/setting_manager.py,sha256=SxKB1aCWh4OySM_bH9cYng9I3PAmrSP-Q8XOZEWEwbI,2899
-crawlo/spider/__init__.py,sha256=Z_rK23l5yt-DuwJPg8bcqodM_FIs4-iHLaKOimGumcE,20452
-crawlo/templates/crawlo.cfg.tmpl,sha256=9BAmwEibS5Tvy6HIcGXPb0BGeuesmibebmTW0iAEkmo,230
-crawlo/templates/project/__init__.py.tmpl,sha256=f3ETIXw_O6K-lkL6lXM5znMPJW1FZYGFrwDs2BnHcnQ,58
-crawlo/templates/project/items.py.tmpl,sha256=mt1Mm--H2Ouos3r7JPkYh0r33rgYJf1YOMz0OZy8TYs,297
-crawlo/templates/project/middlewares.py.tmpl,sha256=jpmj7b7Zb7d3nVyxcaVNdp4KqSts6l2cPSqn_oJUSrM,3775
-crawlo/templates/project/pipelines.py.tmpl,sha256=k_4MJnwZ6GPqVwJSEDURUlTxWybmts4vHrF0de2vgAk,2620
-crawlo/templates/project/run.py.tmpl,sha256=ktkYOslcCh9mpklg6yE5VqfATx3Frj_jNT5z5gHjQ4o,8177
-crawlo/templates/project/settings.py.tmpl,sha256=O_teIARjzRD3aMvPnuIgjaDHdjwW-3beyzfo1QH-Hr8,9580
-crawlo/templates/project/spiders/__init__.py.tmpl,sha256=j_YKsw6HQMJyqlk3WUouP3bsr-XVxshRoSNakHBc00g,106
-crawlo/templates/spider/spider.py.tmpl,sha256=a8S9j43z5gE4auMhf_OnnuVHSZN3JbMDu8Bczu8zIZY,4944
-crawlo/utils/__init__.py,sha256=BDORpyjMN7VGPKImnCDKSkprS-petgD7ezc9rMlBvb0,123
-crawlo/utils/controlled_spider_mixin.py,sha256=VjT30pNW_YIgmTD0nb7DDl2D3HvpnAYFzgSgV3fxFN0,16475
-crawlo/utils/date_tools.py,sha256=0yG0tzGb1VFgWDJJ_cow2LJfz3kj_w2MqSjmfKKESl8,6961
-crawlo/utils/db_helper.py,sha256=3ib5-agrlwf2t5S_QtLRYH75wvJDlYbRqRmDEbpH5Bo,10559
-crawlo/utils/func_tools.py,sha256=WUZEGpWMuDDX7g-QySM7iaiC74erW2SSkZoUvDw1NjM,2369
-crawlo/utils/large_scale_config.py,sha256=j7wQ5ty7pQlBRygw2vhRJ7OI19RYBZKPfYMP3WeF2WI,8154
-crawlo/utils/large_scale_helper.py,sha256=JJqcGSI6VaVe3MSL6IWjmCp8XQIu6T4U-BvBLSttr_s,12157
-crawlo/utils/log.py,sha256=A3lPyhD8kD88cV23KOL-_eT8g69xGQ5L1toDB2AO0mc,4005
-crawlo/utils/queue_helper.py,sha256=xpUUTOqlU1xz2Pb9NKAVGo3AfAO-7Xvx8Lm1q65Dgck,4743
-crawlo/utils/request.py,sha256=yoLB2rY8d78vgPjIWpdhY5SalIKjyLIvTG_UH6EMdVI,8798
-crawlo/utils/request_serializer.py,sha256=bPoSQqE2ksiMyP3WiPB3w3UqZs4f_LgkAw4Pj0qyBDo,8565
-crawlo/utils/spider_loader.py,sha256=pEDUsYOTGjszA6KgjiMlYN4GS5fP4uakkhcp3JTFFQY,2187
-crawlo/utils/system.py,sha256=HvWV1acxou0Rn0L7pNq4CnV_GWFeU0Tmjy3_nLD8M64,237
-crawlo/utils/tools.py,sha256=5Uv25Wy4m_ndZY0-n-eX-t3PxvaZ6wR3-Wvx-o7_Vrs,271
-crawlo/utils/url.py,sha256=rlgX2VlJv6JvLmCDTsbxzMSXE6R5ZL_0dLEqprsA-JU,1482
-examples/__init__.py,sha256=6i631BPnS_TR_BWUjtjB5CBO-zv9kRkwQTQvSya2wHE,123
-examples/controlled_spider_example.py,sha256=SP_k4mdKPvD1JCPs9UCm68jcy2Frg84vvXv9-14RC6I,7776
-tests/__init__.py,sha256=scL1IPVT1iucth7v8ffrjRdeW7QrC_Y7AMmFVMdTY1Y,129
-tests/test_final_validation.py,sha256=fBxf_6YcAEa_HyV_oGAXmmVHY4i6FdA4J6klCmc36hQ,4925
-tests/test_proxy_health_check.py,sha256=xo3QMP1YNw9hu7JDHZOYCUZmFFKLJpHSh4SbxXhCRPQ,1091
-tests/test_proxy_middleware_integration.py,sha256=zcl7fR9Toc-I-stSUTzKZPwcfh3kgrpjI5SbkZ6AVmE,4305
-tests/test_proxy_providers.py,sha256=XwWZCywTYguSsUxSm6fsbaoH1p9dKjqSIx9-sqKZehA,1693
-tests/test_proxy_stats.py,sha256=Til_yksrRz2yBVw-yJi5-36LhNW3vTwpXTm4BdR9PUM,507
-tests/test_proxy_strategies.py,sha256=ZkziozkvZd3KWOQnpHQ8Upd3WpyoX7gN0qFGluNm348,1809
-tests/test_redis_config.py,sha256=TqzFRojc6esGXjGhUCvSLYQDUTAgEJsty9vRVuNraMU,893
-tests/test_redis_queue.py,sha256=o6xViXxJcdx-1eMcG3vhAQEIm8h346HnZb7JXs7ZjwM,6622
-tests/test_request_serialization.py,sha256=8sVdppAsohJ5u-m1WvablCndwL-M_36YPLdGKwgeznM,2289
-tests/test_scheduler.py,sha256=-FOkTWzaMdr6yfO1Msu74hI_GgSfD7iRxO-cFA-9Iyk,7442
-crawlo-1.1.4.dist-info/METADATA,sha256=2I2NA0BR-MWoPZmRUkWrUQYMjuPiUi9mrogIYPWpASU,19781
-crawlo-1.1.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-crawlo-1.1.4.dist-info/entry_points.txt,sha256=5HoVoTSPxI8SCa5B7pQYxLSrkOdiunyO9tqNsLMv52g,43
-crawlo-1.1.4.dist-info/top_level.txt,sha256=keG_67pbZ_wZL2dmDRA9RMaNHTaV_x_oxZ9DKNgwvR0,22
-crawlo-1.1.4.dist-info/RECORD,,