crawlo 1.1.4__py3-none-any.whl → 1.1.5__py3-none-any.whl

{examples → tests}/controlled_spider_example.py

@@ -1,205 +1,205 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-受控爬虫混入类使用示例
-演示如何使用 ControlledRequestMixin 和 AsyncControlledRequestMixin 来控制大规模请求生成
-"""
-
-import asyncio
-from crawlo.spider import Spider
-from crawlo.network import Request
-from crawlo.utils.controlled_spider_mixin import ControlledRequestMixin, AsyncControlledRequestMixin
-
-
-class LargeScaleSpider(Spider, ControlledRequestMixin):
-    """
-    同步版本的受控爬虫示例
-    适用于需要生成大量请求但要控制并发的场景
-    """
-    name = 'large_scale_spider'
-    
-    def __init__(self):
-        Spider.__init__(self)
-        ControlledRequestMixin.__init__(self)
-        
-        # 配置受控生成参数
-        self.max_pending_requests = 150     # 最大待处理请求数
-        self.batch_size = 75               # 每批生成请求数
-        self.generation_interval = 0.02     # 生成间隔（秒）
-        self.backpressure_threshold = 300   # 背压阈值
-    
-    def _original_start_requests(self):
-        """
-        提供原始的大量请求
-        这里模拟爬取一个电商网站的商品页面
-        """
-        # 模拟爬取 10,000 个商品页面
-        base_urls = [
-            "https://example-shop.com/products",
-            "https://example-shop.com/electronics", 
-            "https://example-shop.com/clothing",
-            "https://example-shop.com/books",
-            "https://example-shop.com/home"
-        ]
-        
-        for category in base_urls:
-            # 每个分类爬取 2000 页
-            for page in range(1, 2001):
-                yield Request(
-                    url=f"{category}?page={page}",
-                    callback=self.parse_product_list,
-                    meta={'category': category.split('/')[-1], 'page': page}
-                )
-    
-    def _process_request_before_yield(self, request):
-        """
-        在 yield 请求前进行处理
-        可以添加去重、优先级设置、请求头设置等逻辑
-        """
-        # 设置请求头
-        request.headers.setdefault('User-Agent', 
-            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36')
-        
-        # 根据分类设置优先级
-        category = request.meta.get('category', '')
-        if category == 'electronics':
-            request.priority = 10  # 电子产品优先级最高
-        elif category == 'clothing':
-            request.priority = 8
-        else:
-            request.priority = 5
-        
-        # 可以在这里添加去重逻辑
-        # if self.is_duplicate_request(request):
-        #     return None  # 跳过重复请求
-        
-        return request
-    
-    async def parse_product_list(self, response):
-        """解析商品列表页面"""
-        # 提取商品链接
-        product_links = response.css('a.product-link::attr(href)').getall()
-        
-        for link in product_links:
-            # 生成商品详情页请求
-            yield Request(
-                url=response.urljoin(link),
-                callback=self.parse_product_detail,
-                meta={'category': response.meta['category']}
-            )
-        
-        # 提取分页信息
-        next_page = response.css('a.next-page::attr(href)').get()
-        if next_page:
-            yield Request(
-                url=response.urljoin(next_page),
-                callback=self.parse_product_list,
-                meta=response.meta
-            )
-    
-    async def parse_product_detail(self, response):
-        """解析商品详情页面"""
-        yield {
-            'title': response.css('h1.product-title::text').get(),
-            'price': response.css('.price::text').re_first(r'\d+\.?\d*'),
-            'description': response.css('.product-description::text').get(),
-            'category': response.meta['category'],
-            'url': response.url,
-            'in_stock': 'in-stock' in response.css('.availability::attr(class)').get(''),
-            'rating': response.css('.rating::attr(data-rating)').get(),
-            'reviews_count': response.css('.reviews-count::text').re_first(r'\d+')
-        }
-
-
-class AsyncLargeScaleSpider(Spider, AsyncControlledRequestMixin):
-    """
-    异步版本的受控爬虫示例
-    使用异步控制来实现更精确的并发管理
-    """
-    name = 'async_large_scale_spider'
-    
-    def __init__(self):
-        Spider.__init__(self)
-        AsyncControlledRequestMixin.__init__(self)
-        
-        # 配置异步控制参数
-        self.max_concurrent_generations = 15    # 最大同时生成数
-        self.queue_monitor_interval = 0.5       # 队列监控间隔
-    
-    def _original_start_requests(self):
-        """
-        提供原始的大量请求
-        这里模拟爬取新闻网站的文章
-        """
-        # 模拟爬取 50,000 篇新闻文章
-        news_sites = [
-            "https://news-site-1.com",
-            "https://news-site-2.com", 
-            "https://news-site-3.com",
-            "https://tech-news.com",
-            "https://finance-news.com"
-        ]
-        
-        categories = ['tech', 'finance', 'sports', 'politics', 'entertainment']
-        
-        for site in news_sites:
-            for category in categories:
-                # 每个分类爬取 2000 页
-                for page in range(1, 2001):
-                    yield Request(
-                        url=f"{site}/{category}?page={page}",
-                        callback=self.parse_article_list,
-                        meta={'site': site, 'category': category, 'page': page}
-                    )
-    
-    def _process_request_before_yield(self, request):
-        """异步版本的请求预处理"""
-        # 设置请求头
-        request.headers.setdefault('User-Agent', 
-            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36')
-        
-        # 根据新闻类型设置优先级
-        category = request.meta.get('category', '')
-        if category in ['tech', 'finance']:
-            request.priority = 10  # 科技和财经新闻优先级最高
-        elif category in ['sports', 'politics']:
-            request.priority = 8
-        else:
-            request.priority = 5
-        
-        # 设置延迟（避免对服务器造成过大压力）
-        site = request.meta.get('site', '')
-        if 'tech-news.com' in site:
-            request.meta['download_delay'] = 0.5  # 科技新闻站点较敏感，增加延迟
-        
-        return request
-    
-    async def parse_article_list(self, response):
-        """解析文章列表页面"""
-        # 提取文章链接
-        article_links = response.css('a.article-link::attr(href)').getall()
-        
-        for link in article_links:
-            yield Request(
-                url=response.urljoin(link),
-                callback=self.parse_article_detail,
-                meta={
-                    'site': response.meta['site'],
-                    'category': response.meta['category']
-                }
-            )
-    
-    async def parse_article_detail(self, response):
-        """解析文章详情页面"""
-        yield {
-            'title': response.css('h1.article-title::text').get(),
-            'content': ' '.join(response.css('.article-content p::text').getall()),
-            'author': response.css('.author::text').get(),
-            'publish_date': response.css('.publish-date::attr(datetime)').get(),
-            'category': response.meta['category'],
-            'site': response.meta['site'],
-            'url': response.url,
-            'tags': response.css('.tags a::text').getall(),
-            'views': response.css('.views-count::text').re_first(r'\d+'),
-            'comments': response.css('.comments-count::text').re_first(r'\d+')
-        }
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+受控爬虫混入类使用示例
+演示如何使用 ControlledRequestMixin 和 AsyncControlledRequestMixin 来控制大规模请求生成
+"""
+
+import asyncio
+from crawlo.spider import Spider
+from crawlo.network import Request
+from crawlo.utils.controlled_spider_mixin import ControlledRequestMixin, AsyncControlledRequestMixin
+
+
+class LargeScaleSpider(Spider, ControlledRequestMixin):
+    """
+    同步版本的受控爬虫示例
+    适用于需要生成大量请求但要控制并发的场景
+    """
+    name = 'large_scale_spider'
+    
+    def __init__(self):
+        Spider.__init__(self)
+        ControlledRequestMixin.__init__(self)
+        
+        # 配置受控生成参数
+        self.max_pending_requests = 150     # 最大待处理请求数
+        self.batch_size = 75               # 每批生成请求数
+        self.generation_interval = 0.02     # 生成间隔（秒）
+        self.backpressure_threshold = 300   # 背压阈值
+    
+    def _original_start_requests(self):
+        """
+        提供原始的大量请求
+        这里模拟爬取一个电商网站的商品页面
+        """
+        # 模拟爬取 10,000 个商品页面
+        base_urls = [
+            "https://example-shop.com/products",
+            "https://example-shop.com/electronics", 
+            "https://example-shop.com/clothing",
+            "https://example-shop.com/books",
+            "https://example-shop.com/home"
+        ]
+        
+        for category in base_urls:
+            # 每个分类爬取 2000 页
+            for page in range(1, 2001):
+                yield Request(
+                    url=f"{category}?page={page}",
+                    callback=self.parse_product_list,
+                    meta={'category': category.split('/')[-1], 'page': page}
+                )
+    
+    def _process_request_before_yield(self, request):
+        """
+        在 yield 请求前进行处理
+        可以添加去重、优先级设置、请求头设置等逻辑
+        """
+        # 设置请求头
+        request.headers.setdefault('User-Agent', 
+            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36')
+        
+        # 根据分类设置优先级
+        category = request.meta.get('category', '')
+        if category == 'electronics':
+            request.priority = 10  # 电子产品优先级最高
+        elif category == 'clothing':
+            request.priority = 8
+        else:
+            request.priority = 5
+        
+        # 可以在这里添加去重逻辑
+        # if self.is_duplicate_request(request):
+        #     return None  # 跳过重复请求
+        
+        return request
+    
+    async def parse_product_list(self, response):
+        """解析商品列表页面"""
+        # 提取商品链接
+        product_links = response.css('a.product-link::attr(href)').getall()
+        
+        for link in product_links:
+            # 生成商品详情页请求
+            yield Request(
+                url=response.urljoin(link),
+                callback=self.parse_product_detail,
+                meta={'category': response.meta['category']}
+            )
+        
+        # 提取分页信息
+        next_page = response.css('a.next-page::attr(href)').get()
+        if next_page:
+            yield Request(
+                url=response.urljoin(next_page),
+                callback=self.parse_product_list,
+                meta=response.meta
+            )
+    
+    async def parse_product_detail(self, response):
+        """解析商品详情页面"""
+        yield {
+            'title': response.css('h1.product-title::text').get(),
+            'price': response.css('.price::text').re_first(r'\d+\.?\d*'),
+            'description': response.css('.product-description::text').get(),
+            'category': response.meta['category'],
+            'url': response.url,
+            'in_stock': 'in-stock' in response.css('.availability::attr(class)').get(''),
+            'rating': response.css('.rating::attr(data-rating)').get(),
+            'reviews_count': response.css('.reviews-count::text').re_first(r'\d+')
+        }
+
+
+class AsyncLargeScaleSpider(Spider, AsyncControlledRequestMixin):
+    """
+    异步版本的受控爬虫示例
+    使用异步控制来实现更精确的并发管理
+    """
+    name = 'async_large_scale_spider'
+    
+    def __init__(self):
+        Spider.__init__(self)
+        AsyncControlledRequestMixin.__init__(self)
+        
+        # 配置异步控制参数
+        self.max_concurrent_generations = 15    # 最大同时生成数
+        self.queue_monitor_interval = 0.5       # 队列监控间隔
+    
+    def _original_start_requests(self):
+        """
+        提供原始的大量请求
+        这里模拟爬取新闻网站的文章
+        """
+        # 模拟爬取 50,000 篇新闻文章
+        news_sites = [
+            "https://news-site-1.com",
+            "https://news-site-2.com", 
+            "https://news-site-3.com",
+            "https://tech-news.com",
+            "https://finance-news.com"
+        ]
+        
+        categories = ['tech', 'finance', 'sports', 'politics', 'entertainment']
+        
+        for site in news_sites:
+            for category in categories:
+                # 每个分类爬取 2000 页
+                for page in range(1, 2001):
+                    yield Request(
+                        url=f"{site}/{category}?page={page}",
+                        callback=self.parse_article_list,
+                        meta={'site': site, 'category': category, 'page': page}
+                    )
+    
+    def _process_request_before_yield(self, request):
+        """异步版本的请求预处理"""
+        # 设置请求头
+        request.headers.setdefault('User-Agent', 
+            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36')
+        
+        # 根据新闻类型设置优先级
+        category = request.meta.get('category', '')
+        if category in ['tech', 'finance']:
+            request.priority = 10  # 科技和财经新闻优先级最高
+        elif category in ['sports', 'politics']:
+            request.priority = 8
+        else:
+            request.priority = 5
+        
+        # 设置延迟（避免对服务器造成过大压力）
+        site = request.meta.get('site', '')
+        if 'tech-news.com' in site:
+            request.meta['download_delay'] = 0.5  # 科技新闻站点较敏感，增加延迟
+        
+        return request
+    
+    async def parse_article_list(self, response):
+        """解析文章列表页面"""
+        # 提取文章链接
+        article_links = response.css('a.article-link::attr(href)').getall()
+        
+        for link in article_links:
+            yield Request(
+                url=response.urljoin(link),
+                callback=self.parse_article_detail,
+                meta={
+                    'site': response.meta['site'],
+                    'category': response.meta['category']
+                }
+            )
+    
+    async def parse_article_detail(self, response):
+        """解析文章详情页面"""
+        yield {
+            'title': response.css('h1.article-title::text').get(),
+            'content': ' '.join(response.css('.article-content p::text').getall()),
+            'author': response.css('.author::text').get(),
+            'publish_date': response.css('.publish-date::attr(datetime)').get(),
+            'category': response.meta['category'],
+            'site': response.meta['site'],
+            'url': response.url,
+            'tags': response.css('.tags a::text').getall(),
+            'views': response.css('.views-count::text').re_first(r'\d+'),
+            'comments': response.css('.comments-count::text').re_first(r'\d+')
+        }

tests/date_tools_example.py

@@ -0,0 +1,181 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+Crawlo框架日期工具使用示例
+"""
+from crawlo.tools import (
+    TimeUtils,
+    parse_time,
+    format_time,
+    time_diff,
+    to_timestamp,
+    to_datetime,
+    now,
+    to_timezone,
+    to_utc,
+    to_local
+)
+
+
+def demo_date_parsing():
+    """演示日期解析功能"""
+    print("=== 日期解析演示 ===\n")
+    
+    # 解析各种格式的日期字符串
+    date_strings = [
+        "2025-09-10 14:30:00",
+        "September 10, 2025 2:30 PM",
+        "2025/09/10 14:30:00",
+        "10-09-2025 14:30:00",
+        "2025年9月10日 14时30分00秒"
+    ]
+    
+    for date_str in date_strings:
+        parsed = parse_time(date_str)
+        print(f"解析 '{date_str}' -> {parsed}")
+    
+    print()
+
+
+def demo_date_formatting():
+    """演示日期格式化功能"""
+    print("=== 日期格式化演示 ===\n")
+    
+    # 获取当前时间
+    current_time = now()
+    print(f"当前时间: {current_time}")
+    
+    # 使用不同格式进行格式化
+    formats = [
+        "%Y-%m-%d",
+        "%Y年%m月%d日",
+        "%Y-%m-%d %H:%M:%S",
+        "%B %d, %Y at %I:%M %p",
+        "%A, %B %d, %Y"
+    ]
+    
+    for fmt in formats:
+        formatted = format_time(current_time, fmt)
+        print(f"格式化为 '{fmt}' -> {formatted}")
+    
+    print()
+
+
+def demo_time_calculations():
+    """演示时间计算功能"""
+    print("=== 时间计算演示 ===\n")
+    
+    # 计算时间差
+    start_time = "2025-09-10 10:00:00"
+    end_time = "2025-09-10 14:30:45"
+    
+    diff_seconds = time_diff(start_time, end_time, "seconds")
+    diff_minutes = time_diff(start_time, end_time, "minutes")
+    diff_hours = time_diff(start_time, end_time, "hours")
+    
+    print(f"起始时间: {start_time}")
+    print(f"结束时间: {end_time}")
+    print(f"时间差 - 秒: {diff_seconds}")
+    print(f"时间差 - 分钟: {diff_minutes}")
+    print(f"时间差 - 小时: {diff_hours}")
+    
+    print()
+    
+    # 日期加减
+    base_date = "2025-09-10"
+    plus_30_days = TimeUtils.add_days(base_date, 30)
+    minus_15_days = TimeUtils.add_days(base_date, -15)
+    plus_3_months = TimeUtils.add_months(base_date, 3)
+    
+    print(f"基础日期: {base_date}")
+    print(f"加30天: {plus_30_days}")
+    print(f"减15天: {minus_15_days}")
+    print(f"加3个月: {plus_3_months}")
+    
+    print()
+
+
+def demo_timezone_conversion():
+    """演示时区转换功能"""
+    print("=== 时区转换演示 ===\n")
+    
+    # 创建一个时间
+    dt = parse_time("2025-09-10 14:30:00")
+    print(f"原始时间: {dt}")
+    
+    # 转换为UTC时区
+    utc_time = to_utc(dt)
+    print(f"UTC时间: {utc_time}")
+    
+    # 转换为本地时区
+    local_time = to_local(dt)
+    print(f"本地时间: {local_time}")
+    
+    # 转换为纽约时区
+    ny_time = to_timezone(dt, "America/New_York")
+    print(f"纽约时间: {ny_time}")
+    
+    # 转换为伦敦时区
+    london_time = to_timezone(dt, "Europe/London")
+    print(f"伦敦时间: {london_time}")
+    
+    print()
+
+
+def demo_timestamp_conversion():
+    """演示时间戳转换功能"""
+    print("=== 时间戳转换演示 ===\n")
+    
+    # 获取当前时间戳
+    current_timestamp = to_timestamp(now())
+    print(f"当前时间戳: {current_timestamp}")
+    
+    # 从时间戳转换为datetime
+    dt_from_timestamp = to_datetime(current_timestamp)
+    print(f"从时间戳转换: {dt_from_timestamp}")
+    
+    # 从时间戳创建带时区的datetime
+    dt_with_tz = to_timezone(to_datetime(current_timestamp), "Asia/Shanghai")
+    print(f"带时区的时间: {dt_with_tz}")
+    
+    print()
+
+
+def demo_in_spider():
+    """演示在爬虫中使用日期工具"""
+    print("=== 在爬虫中使用日期工具 ===\n")
+    print("在爬虫项目中，您可以这样使用日期工具:")
+    print("""
+from crawlo import Spider
+from crawlo.tools import parse_time, format_time
+
+class NewsSpider(Spider):
+    def parse(self, response):
+        # 提取文章发布时间
+        publish_time_str = response.css('.publish-time::text').get()
+        
+        # 解析发布时间
+        publish_time = parse_time(publish_time_str)
+        
+        # 格式化时间为标准格式
+        formatted_time = format_time(publish_time, "%Y-%m-%d %H:%M:%S")
+        
+        # 计算文章发布多久了
+        current_time = self.tools.now()
+        hours_since_publish = self.tools.time_diff(publish_time, current_time, "hours")
+        
+        # 根据发布时间决定是否继续处理
+        if hours_since_publish < 24:  # 只处理24小时内的文章
+            # 处理文章...
+            pass
+    """)
+
+
+if __name__ == '__main__':
+    # 运行演示
+    demo_date_parsing()
+    demo_date_formatting()
+    demo_time_calculations()
+    demo_timezone_conversion()
+    demo_timestamp_conversion()
+    demo_in_spider()