crawlo 1.0.9__py3-none-any.whl → 1.1.1__py3-none-any.whl

examples/baidu_spider/middleware.py

@@ -0,0 +1,49 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+# @Time    :    2025-05-17 11:21
+# @Author  :   crawl-coder
+# @Desc    :   None
+"""
+import asyncio
+import random
+
+from crawlo.exceptions import IgnoreRequestError
+from crawlo.middleware import BaseMiddleware
+
+
+class TestMiddleWare(BaseMiddleware):
+
+    async def process_request(self, request, spider):
+        # 请求预处理
+        # print('process_request', request, spider)
+        # if random.randint(1, 5) == 1:
+        #     raise IgnoreRequestError('url不正确')
+        pass
+
+    def process_response(self, request, response, spider):
+        # 响应预处理
+        # print('process_response', response, response, spider)
+        return response
+
+    def process_exception(self, request, exception, spider):
+        # 异常预处理
+        # print('process_exception', request, exception, spider)
+        pass
+
+
+class TestMiddleWare2(BaseMiddleware):
+    def process_request(self, request, spider):
+        # 请求预处理
+        # print('process_request2', request, spider)
+        pass
+
+    def process_response(self, request, response, spider):
+        # 响应预处理
+        # print('process_response2', response, response, spider)
+        return response
+
+    def process_exception(self, request, exception, spider):
+        # 异常预处理
+        # print('process_exception2', request, exception, spider)
+        pass

examples/baidu_spider/pipeline.py

@@ -0,0 +1,55 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+import pymongo
+
+from motor.motor_asyncio import AsyncIOMotorClient
+
+from random import randint
+
+from crawlo.event import spider_closed
+from crawlo.exceptions import ItemDiscard
+from crawlo.utils.log import get_logger
+
+
+class TestPipeline(object):
+
+    async def process_item(self, item, spider):
+        if randint(1, 3) == 1:
+            raise ItemDiscard('重复数据')
+        return item
+
+    @classmethod
+    def create_instance(cls, *args, **kwargs):
+        return cls()
+
+
+class MongoPipeline(object):
+
+    def __init__(self, conn, col):
+        self.conn = conn
+        self.col = col
+
+        self.logger = get_logger(self.__class__.__name__)
+
+    @classmethod
+    def create_instance(cls, crawler):
+        settings = crawler.settings
+        mongo_params = settings.get('MONGODB_PARAMS', None)
+        db_name = settings.get('MONGODB_DB', None)
+        project_name = settings.get('PROJECT_NAME', None)
+
+        conn = AsyncIOMotorClient(**mongo_params) if mongo_params else AsyncIOMotorClient()
+
+        col = conn[db_name][project_name]
+        o = cls(conn, col)
+        crawler.subscriber.subscribe(o.spider_closed, event=spider_closed)
+        return o
+
+    async def process_item(self, item, spider):
+        await self.col.insert_one(item.to_dict())
+        return item
+
+    async def spider_closed(self):
+        self.logger.info('MongoDB closed.')
+        self.conn.close()
+

examples/baidu_spider/run.py

@@ -0,0 +1,27 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+# @Time    :    2025-02-05 13:12
+# @Author  :   oscar
+# @Desc    :   None
+"""
+import asyncio
+from crawlo.crawler import CrawlerProcess
+
+from examples.baidu_spider.spiders.miit import MiitDeviceSpider
+from examples.baidu_spider.spiders.sina import SinaSpider
+
+
+async def main():
+    process = CrawlerProcess()
+    # await process.crawl(
+    #     [
+    #         # SinaSpider,
+    #         MiitDeviceSpider
+    #     ]
+    # )
+    await process.crawl('miit_device')
+
+
+if __name__ == '__main__':
+    asyncio.run(main())

examples/baidu_spider/settings.py

@@ -0,0 +1,121 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+
+PROJECT_NAME = 'baidu_spider'
+
+CONCURRENCY = 30
+
+MAX_RUNNING_SPIDERS = 8
+
+USE_SESSION = True
+
+# 下载延迟
+DOWNLOAD_DELAY = 0.1
+RANDOMNESS = False
+
+# --------------------------------------------------- 公共MySQL配置 -----------------------------------------------------
+MYSQL_HOST = '43.139.14.225'
+MYSQL_PORT = 3306
+MYSQL_USER = 'picker'
+MYSQL_PASSWORD = 'kmcNbbz6TbSihttZ'
+MYSQL_DB = 'stock_pro'
+MYSQL_TABLE = 'articles'  # 可选，默认使用spider名称
+MYSQL_BATCH_SIZE = 500
+
+# asyncmy专属配置
+MYSQL_POOL_MIN = 5  # 连接池最小连接数
+MYSQL_POOL_MAX = 20  # 连接池最大连接数
+
+# 选择下载器
+DOWNLOADER = "crawlo.downloader.httpx_downloader.HttpXDownloader"
+# DOWNLOADER = "crawlo.downloader.cffi_downloader.CurlCffiDownloader"
+
+# MIDDLEWARES = [
+#     'crawlo.middleware.download_delay.DownloadDelayMiddleware',
+#     'crawlo.middleware.default_header.DefaultHeaderMiddleware',
+#     'crawlo.middleware.response_filter.ResponseFilterMiddleware',
+#     'crawlo.middleware.retry.RetryMiddleware',
+#     'crawlo.middleware.response_code.ResponseCodeMiddleware',
+#     'crawlo.middleware.request_ignore.RequestIgnoreMiddleware',
+# ]
+
+EXTENSIONS = [
+    'crawlo.extension.log_interval.LogIntervalExtension',
+    'crawlo.extension.log_stats.LogStats',
+]
+
+PIPELINES = [
+    'crawlo.pipelines.console_pipeline.ConsolePipeline',
+    # 'crawlo.pipelines.mysql_pipeline.AsyncmyMySQLPipeline',  # 或 AiomysqlMySQLPipeline
+    # 'crawlo.pipelines.mysql_batch_pipline.AsyncmyMySQLPipeline',  # 或 AiomysqlMySQLPipeline
+    # 'baidu_spider.pipeline.TestPipeline',
+    # 'baidu_spider.pipeline.MongoPipeline',
+]
+
+USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36'
+DEFAULT_HEADERS = {
+    "accept": "application/json, text/javascript, */*; q=0.01",
+    "accept-language": "zh-CN,zh;q=0.9,en;q=0.8",
+    "cache-control": "no-cache",
+    "pragma": "no-cache",
+    "priority": "u=1, i",
+    "sec-ch-ua": "\"Chromium\";v=\"136\", \"Google Chrome\";v=\"136\", \"Not.A/Brand\";v=\"99\"",
+    "sec-ch-ua-mobile": "?0",
+    "sec-ch-ua-platform": "\"macOS\"",
+    "sec-fetch-dest": "empty",
+    "sec-fetch-mode": "cors",
+    "sec-fetch-site": "same-origin",
+    # "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
+    "x-requested-with": "XMLHttpRequest"
+}
+
+# --------------------------------------DB ---------------------------------------------
+Mongo_Params = ''
+MONGODB_DB = 'news'
+
+REDIS_TTL = 0
+CLEANUP_FP = False
+
+LOG_LEVEL = 'DEBUG'
+
+FILTER_CLASS = 'crawlo.filters.aioredis_filter.AioRedisFilter'
+# FILTER_CLASS = 'crawlo.filters.redis_filter.RedisFilter'
+# FILTER_CLASS = 'crawlo.filters.memory_filter.MemoryFileFilter'
+
+# PROXY_POOL_API = 'http://123.56.42.142:5000/proxy/getitem/'
+#
+# PROXY_FETCH_FUNC = "crawlo.utils.proxy.get_proxies"
+
+
+# PROXY_ENABLED = True
+#
+# # 使用 API 提供者
+# PROXY_PROVIDERS = [
+#     {
+#         'class': 'crawlo.proxy.providers.APIProxyProvider',
+#         'config': {
+#             'url': 'http://123.56.42.142:5000/proxy/getitem/',
+#             'method': 'GET',
+#             'timeout': 10.0
+#         }
+#     }
+# ]
+#
+# # 代理选择策略：使用最少的代理（避免单 IP 过载）
+# PROXY_SELECTION_STRATEGY = 'least_used'
+#
+# # 请求延迟：0.5~1.5 秒，避免请求过快
+# PROXY_REQUEST_DELAY_ENABLED = True
+# PROXY_REQUEST_DELAY = 1.0
+#
+# # 健康检查
+# PROXY_HEALTH_CHECK_ENABLED = True
+# PROXY_HEALTH_CHECK_INTERVAL = 10  # 每 5 分钟检查一次
+#
+# # 代理池更新
+# PROXY_POOL_UPDATE_INTERVAL = 5  # 每 5 分钟从 API 拉取新代理
+#
+# # 失败处理
+# PROXY_MAX_FAILURES = 3  # 失败 3 次后禁用代理
+# PROXY_COOLDOWN_PERIOD = 600  # 禁用 10 分钟后恢复
+# PROXY_MAX_RETRY_COUNT = 2  # 每个请求最多重试 2 次

examples/baidu_spider/spiders/__init__.py

@@ -0,0 +1,7 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+# @Time    :    2025-05-11 12:20
+# @Author  :   oscar
+# @Desc    :   None
+"""

examples/baidu_spider/spiders/bai_du.py

@@ -0,0 +1,61 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+# @Time    :    2025-02-05 13:05
+# @Author  :   oscar
+# @Desc    :   None
+"""
+import asyncio
+from crawlo import Request
+from crawlo.spider import Spider
+
+from items import BauDuItem
+
+
+class BaiDuSpider(Spider):
+    start_urls = ["https://www.baidu.com/", "https://www.baidu.com/"]
+
+    custom_settings = {
+        'CONCURRENCY': 1
+    }
+
+    name = "bai_du"
+
+    # headers = {
+    #     "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36"
+    # }
+    #
+    user_gent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36"
+
+    async def parse(self, response):
+        for i in range(5):
+            url = f"https://www.baidu.com"
+            # url = f"https://www.httpbin.org/404"
+            r = Request(url=url, callback=self.parse_page, dont_filter=True)
+            yield r
+
+    async def parse_page(self, response):
+        for i in range(5):
+            url = f"https://www.baidu.com"
+            meta = {'test': 'hhhh'}
+            r = Request(url=url, callback=self.parse_detail, meta=meta, dont_filter=False)
+            yield r
+
+    def parse_detail(self, response):
+        item = BauDuItem()
+        item['title'] = response.xpath('//title/text()').get()
+
+        item['url'] = response.url
+
+        yield item
+
+    async def spider_opened(self):
+        pass
+
+    async def spider_closed(self):
+        pass
+
+
+if __name__ == '__main__':
+    b = BaiDuSpider()
+    b.start_requests()

examples/baidu_spider/spiders/miit.py

@@ -0,0 +1,159 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+# @Time    :    2025-08-22 14:00
+# @Author  :   oscar
+# @Desc    :   爬取工信部无线电设备核准信息（支持全量34652页）
+"""
+
+import json
+import asyncio
+import random
+
+from crawlo import Request
+from crawlo.spider import Spider
+from crawlo.utils.log import get_logger
+from crawlo.utils.date_tools import to_datetime
+
+# 引入定义好的 Item
+from examples.baidu_spider.items import MiitDeviceItem
+
+
+logger = get_logger(__name__)
+
+
+class MiitDeviceSpider(Spider):
+    name = 'miit_device'
+    allowed_domains = ['ythzxfw.miit.gov.cn']
+
+    # 字段映射表
+    FIELD_MAPPING = {
+        "articleField01": ("核准证编号", "approval_certificate_no"),
+        "articleField02": ("设备名称", "device_name"),
+        "articleField03": ("设备型号", "model_number"),
+        "articleField04": ("申请单位", "applicant"),
+        "articleField05": ("备注", "remarks"),
+        "articleField06": ("有效期", "validity_period"),
+        "articleField07": ("频率容限", "frequency_tolerance"),
+        "articleField08": ("频率范围", "frequency_range"),
+        "articleField09": ("发射功率", "transmission_power"),
+        "articleField10": ("占用带宽", "occupied_bandwidth"),
+        "articleField11": ("杂散发射限制", "spurious_emission_limit"),
+        "articleField12": ("发证日期", "issue_date"),
+        "articleField13": ("核准代码", "approval_code"),
+        "articleField14": ("CMIIT ID", "cmiit_id"),
+        "articleField15": ("调制方式", "modulation_scheme"),
+        "articleField16": ("技术体制/功能模块", "technology_module"),
+        "createTime": ("createTime", "create_time"),
+        "articleId": ("articleId", "article_id")
+    }
+
+    headers = {
+        "Accept": "application/json, text/plain, */*",
+        "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
+        "Authorization": "null",
+        "Cache-Control": "no-cache",
+        "Connection": "keep-alive",
+        "Content-Type": "application/json;charset=UTF-8",
+        "Origin": "https://ythzxfw.miit.gov.cn",
+        "Pragma": "no-cache",
+        "Referer": "https://ythzxfw.miit.gov.cn/oldyth/resultQuery",
+        "Sec-Fetch-Dest": "empty",
+        "Sec-Fetch-Mode": "cors",
+        "Sec-Fetch-Site": "same-origin",
+        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36",
+        "sec-ch-ua": '"Not;A=Brand";v="99", "Google Chrome";v="139", "Chromium";v="139"',
+        "sec-ch-ua-mobile": "?0",
+        "sec-ch-ua-platform": '"macOS"'
+    }
+
+    cookies = {
+        "wzws_sessionid": "gjdjYmMyNYFkZjRiZjCgaKkOx4AyNDBlOjQ3ZTozMmUwOmQ5MmI6ZjFjZTphNWJiOjk5ZmU6OTU4OQ==",
+        "ariauseGraymode": "false",
+        "Hm_lvt_a73626d298a849004aacc34159f68abd": "1755909833",
+        "Hm_lpvt_a73626d298a849004aacc34159f68abd": "1755909833",
+        "HMACCOUNT": "6C5E4C6C47DC62FF"
+    }
+
+    # 分页配置
+    start_page = 1           # 起始页
+    end_page = 34652         # 总页数
+    current_page = 1
+    page_size = 5            # 每页条数
+
+    # 请求间隔（秒），防止被封
+    min_delay = 1.5
+    max_delay = 3.0
+
+    def start_requests(self):
+        # 从起始页开始
+        yield self.make_request(self.start_page)
+
+    def make_request(self, page):
+        """封装请求创建"""
+        data = {
+            "categoryId": "352",
+            "currentPage": page,
+            "pageSize": self.page_size,
+            "searchContent": ""
+        }
+        return Request(
+            method='POST',
+            url='https://ythzxfw.miit.gov.cn/oldyth/user-center/tbAppSearch/selectResult',
+            headers=self.headers,
+            cookies=self.cookies,
+            body=json.dumps(data, separators=(',', ':'), ensure_ascii=False),
+            callback=self.parse,
+            dont_filter=True,
+            meta={'page': page}  # 记录当前页码，便于日志和调试
+        )
+
+    async def parse(self, response):
+        page = response.meta.get('page', 'unknown')
+        try:
+            json_data = response.json()
+            success = json_data.get("success")
+            code = json_data.get("code")
+
+            if not success or code != 200:
+                logger.error(f"第 {page} 页请求失败: code={code}, msg={json_data.get('msg')}")
+                return
+
+            tb_app_article = json_data.get('params', {}).get('tbAppArticle', {})
+            records = tb_app_article.get('list', [])
+            total_count = tb_app_article.get('total', 0)  # 总数据条数，例如 173256
+
+            logger.info(f"✅ 第 {page} 页解析成功，共 {len(records)} 条数据。总计: {total_count} 条")
+
+            for raw_item in records:
+                item = MiitDeviceItem()
+                for field_key, (chinese_name, english_field) in self.FIELD_MAPPING.items():
+                    value = raw_item.get(field_key)
+                    if english_field == 'issue_date' and value:
+                        value = to_datetime(value.split()[0])
+                    item[english_field] = value
+                yield item
+
+            # ✅ 核心修复：根据 total_count 和 page_size 计算真实总页数
+            # 注意：需要向上取整，例如 173256 / 5 = 34651.2，应该有 34652 页
+            import math
+            calculated_total_pages = math.ceil(total_count / self.page_size)
+
+            # 现在使用 calculated_total_pages 来判断是否继续翻页
+            next_page = page + 1
+            if next_page <= calculated_total_pages:
+                delay = random.uniform(self.min_delay, self.max_delay)
+                logger.debug(f"等待 {delay:.2f}s 后请求第 {next_page} 页...")
+                await asyncio.sleep(delay)
+                yield self.make_request(next_page)
+            else:
+                logger.info(f"🎉 爬取完成！已到达最后一页 {calculated_total_pages}")
+
+        except Exception as e:
+            logger.error(f"❌ 解析第 {page} 页失败: {e}, 响应: {response.text[:500]}...")
+
+    async def spider_opened(self):
+        logger.info(f"MiitDeviceSpider 启动，准备爬取 {self.start_page} 至 {self.end_page} 页...")
+
+    async def spider_closed(self):
+        logger.info("MiitDeviceSpider 结束。")

examples/baidu_spider/spiders/sina.py

@@ -0,0 +1,79 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+# @Time    :    2025-02-05 13:05
+# @Author  :   oscar
+# @Desc    :   None
+"""
+import time
+
+from crawlo import Request
+from crawlo.spider import Spider
+from crawlo.utils.date_tools import to_datetime
+
+from examples.baidu_spider.items import ArticleItem
+
+
+class SinaSpider(Spider):
+    # 获取当前时间戳，并减去 10 分钟（600 秒）
+    current_time_minus_10min = int(time.time()) - 6000
+    # 构造 URL
+    url = f'https://news.10jqka.com.cn/tapp/news/push/stock/?page=1&tag=&track=website&ctime={current_time_minus_10min}'
+
+    start_urls = [url]
+    name = 'sina'
+    # mysql_table = 'news_10jqka'
+
+    allowed_domains = ['*']
+
+    def start_requests(self):
+        for url in self.start_urls:
+            yield Request(url=url, callback=self.parse, dont_filter=True)
+
+    async def parse(self, response):
+        jsonp_str = response.json()
+        rows = jsonp_str.get('data', {}).get('list', [])
+        for row in rows:
+            article_id = row.get('id')
+            title = row.get('title')
+            digest = row.get('digest')
+            short = row.get('short')
+            detail_url = row.get('url')
+            tag = row.get('tag')
+            ctime = row.get('ctime')
+            source = row.get('source')
+            meta = {
+                'article_id': article_id,
+                'title': title,
+                'digest': digest,
+                'short': short,
+                'detail_url': detail_url,
+                'source': source,
+                'tag': tag,
+                'ctime': to_datetime(int(ctime))
+            }
+
+            yield Request(url=detail_url, callback=self.parse_detail, encoding='gbk', meta=meta)
+
+    @staticmethod
+    async def parse_detail(response):
+        item = ArticleItem()
+        meta = response.meta
+        content = ''.join(response.xpath('//*[@id="contentApp"]/p/text()').extract()).strip()
+        ctime = meta.get('ctime')
+        item['article_id'] = meta.get('article_id')
+        item['title'] = meta.get('title')
+        item['digest'] = content
+        item['short'] = meta.get('short')
+        item['url'] = meta.get('detail_url')
+        item['tag'] = meta.get('tag').strip()
+        item['ctime'] = to_datetime(ctime)
+        item['source'] = meta.get('source')
+
+        yield item
+
+    async def spider_opened(self):
+        pass
+
+    async def spider_closed(self):
+        pass

tests/__init__.py

@@ -1,7 +1,7 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-"""
-# @Time    :    2025-08-24 12:36
-# @Author  :   crawl-coder
-# @Desc    :   None
-"""
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+# @Time    :    2025-08-24 12:36
+# @Author  :   crawl-coder
+# @Desc    :   None
+"""

tests/test_proxy_health_check.py

@@ -1,33 +1,33 @@
-# tests/test_proxy_health_check.py
-import pytest
-from unittest.mock import AsyncMock, patch
-from crawlo.proxy.health_check import check_single_proxy
-import httpx
-
-
-@pytest.mark.asyncio
-@patch('httpx.AsyncClient')
-async def test_health_check_success(mock_client_class):
-    """测试健康检查：成功"""
-    mock_resp = AsyncMock()
-    mock_resp.status_code = 200
-    mock_client_class.return_value.__aenter__.return_value.get.return_value = mock_resp
-
-    proxy_info = {'url': 'http://good:8080', 'healthy': False}
-    await check_single_proxy(proxy_info)
-
-    assert proxy_info['healthy'] is True
-    assert proxy_info['failures'] == 0
-
-
-@pytest.mark.asyncio
-@patch('httpx.AsyncClient')
-async def test_health_check_failure(mock_client_class):
-    """测试健康检查：失败"""
-    mock_client_class.return_value.__aenter__.return_value.get.side_effect = httpx.ConnectError("Failed")
-
-    proxy_info = {'url': 'http://bad:8080', 'healthy': True, 'failures': 0}
-    await check_single_proxy(proxy_info)
-
-    assert proxy_info['healthy'] is False
+# tests/test_proxy_health_check.py
+import pytest
+from unittest.mock import AsyncMock, patch
+from crawlo.proxy.health_check import check_single_proxy
+import httpx
+
+
+@pytest.mark.asyncio
+@patch('httpx.AsyncClient')
+async def test_health_check_success(mock_client_class):
+    """测试健康检查：成功"""
+    mock_resp = AsyncMock()
+    mock_resp.status_code = 200
+    mock_client_class.return_value.__aenter__.return_value.get.return_value = mock_resp
+
+    proxy_info = {'url': 'http://good:8080', 'healthy': False}
+    await check_single_proxy(proxy_info)
+
+    assert proxy_info['healthy'] is True
+    assert proxy_info['failures'] == 0
+
+
+@pytest.mark.asyncio
+@patch('httpx.AsyncClient')
+async def test_health_check_failure(mock_client_class):
+    """测试健康检查：失败"""
+    mock_client_class.return_value.__aenter__.return_value.get.side_effect = httpx.ConnectError("Failed")
+
+    proxy_info = {'url': 'http://bad:8080', 'healthy': True, 'failures': 0}
+    await check_single_proxy(proxy_info)
+
+    assert proxy_info['healthy'] is False
     assert proxy_info['failures'] == 1