crawlo 1.0.4__py3-none-any.whl → 1.0.5__py3-none-any.whl

examples/gxb/settings.py

@@ -0,0 +1,71 @@
+import platform
+
+PROXY_ENABLED = False
+
+# API 地址
+PROXY_API_URL = 'http://123.56.42.142:5000/proxy/getitem/'
+
+# 提取方式（根据实际返回结构选择）
+PROXY_EXTRACTOR = "proxy"
+# 或
+# from utils.proxy_extractors import custom_extractor_proxy
+# PROXY_EXTRACTOR = custom_extractor_proxy
+
+# 刷新间隔
+PROXY_REFRESH_INTERVAL = 5
+
+CONCURRENCY = 3
+
+# 超时时间
+PROXY_API_TIMEOUT = 10
+
+if platform.system() == "Windows":
+    MYSQL_HOST = "pc-2ze9oh2diu5e5firh.rwlb.rds.aliyuncs.com"
+else:
+    MYSQL_HOST = "tianmai-k8s-dmadmin-x.rwlb.rds.aliyuncs.com"
+
+# 数据库端口
+MYSQL_PORT = 3306
+# 数据库用户名
+MYSQL_USER = "data_collection"
+# 数据库密码
+MYSQL_PASSWORD = "CRNabzFQ2H"
+# 数据库名
+MYSQL_DB = "cxzx_xm"
+# 数据库编码
+MYSQL_TABLE = "telecom_device_licenses_v4"
+
+MYSQL_BATCH_SIZE = 100
+
+PIPELINES = [
+    'crawlo.pipelines.console_pipeline.ConsolePipeline',
+    # 'crawlo.pipelines.mysql_pipeline.AsyncmyMySQLPipeline', # 可选：存入 MySQL
+]
+
+
+HEADERS = {
+            "Accept": "application/json, text/plain, */*",
+            "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
+            "Authorization": "null",
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "Content-Type": "application/json;charset=UTF-8",
+            "Origin": "https://ythzxfw.miit.gov.cn",
+            "Pragma": "no-cache",
+            "Referer": "https://ythzxfw.miit.gov.cn/oldyth/resultQuery",
+            "Sec-Fetch-Dest": "empty",
+            "Sec-Fetch-Mode": "cors",
+            "Sec-Fetch-Site": "same-origin",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36",
+            "sec-ch-ua": '"Not;A=Brand";v="99", "Google Chrome";v="139", "Chromium";v="139"',
+            "sec-ch-ua-mobile": "?0",
+            "sec-ch-ua-platform": '"Windows"'
+        }
+
+COOKIES = {
+    "wzws_sessionid": "oGivsIOAMjQwZTozYjM6MzBiMjo3MWMwOjg0NmY6MzQ4OTozNWZjOjEyMTGBOGY2OTQ2gjdjYmMyNQ==",
+    "ariauseGraymode": "false",
+    "Hm_lvt_a73626d298a849004aacc34159f68abd": "1755909741,1756084244,1756256541,1756344453",
+    "Hm_lpvt_a73626d298a849004aacc34159f68abd": "1756344453",
+    "HMACCOUNT": "08DF0D235A291EAA"
+}

examples/gxb/spider/miit_spider.py

@@ -0,0 +1,180 @@
+import json
+import logging
+import re
+
+from crawlo import Request, Spider
+
+from examples.gxb.items import RadioApprovalItem, TelecomLicenseItem
+
+logger = logging.getLogger(__name__)
+
+# 基础配置
+BASE_URL = "https://ythzxfw.miit.gov.cn"
+API_URL = BASE_URL + "/oldyth/user-center/tbAppSearch/selectResult"
+
+# 任务配置
+TASKS = {
+    "radio_approval": {
+        "name": "无线电设备型号核准",
+        "category_id": "352",
+        "item_class": RadioApprovalItem,
+        "table": "radio_equipment_approval_new",
+        "field_mapping": {
+            'articleField01': 'approval_number',
+            'articleField02': 'device_name',
+            'articleField03': 'device_model',
+            'articleField04': 'applicant',
+            'articleField05': 'remarks',
+            'articleField06': 'validity_period',
+            'articleField07': 'frequency_tolerance',
+            'articleField08': 'frequency_range',
+            'articleField09': 'transmit_power',
+            'articleField10': 'occupied_bandwidth',
+            'articleField11': 'spurious_emission_limit',
+            'articleField12': 'issue_date',
+            'articleField13': 'approval_code',
+            'articleField14': 'cmiit_id',
+            'articleField15': 'modulation_mode',
+            'articleField16': 'technology_system',
+        }
+    },
+    "telecom_license": {
+        "name": "电信设备进网许可证",
+        "category_id": "144",
+        "item_class": TelecomLicenseItem,
+        "table": "telecom_device_licenses_new",
+        "field_mapping": {
+            'articleField01': 'license_number',
+            'articleField02': 'device_name',
+            'articleField03': 'device_model',
+            'articleField04': 'applicant',
+            'articleField05': 'manufacturer',
+            'articleField06': 'issue_date',
+            'articleField07': 'expiry_date',
+            'articleField08': 'certificate_type',
+            'articleField09': 'remarks',
+            'articleField10': 'certificate_status',
+            'articleField11': 'origin',
+        }
+    }
+}
+
+def strip_html(text: str) -> str:
+    """去除 HTML 标签"""
+    if not text or not isinstance(text, str):
+        return text
+    return re.sub(r'<[^>]+>', '', text)
+
+class MiitSpider(Spider):
+    name = "miit_spider"
+    custom_settings = {
+        'DOWNLOAD_DELAY': 0.5,
+        'CONCURRENT_REQUESTS': 5,
+        'CONCURRENT_REQUESTS_PER_DOMAIN': 5,
+        'COOKIES_ENABLED': True,
+        'RETRY_TIMES': 3,
+        'DEFAULT_REQUEST_HEADERS': {
+            "Accept": "application/json, text/plain, */*",
+            "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
+            "Authorization": "null",
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "Content-Type": "application/json;charset=UTF-8",
+            "Origin": BASE_URL,
+            "Pragma": "no-cache",
+            "Referer": f"{BASE_URL}/oldyth/resultQuery",
+            "Sec-Fetch-Dest": "empty",
+            "Sec-Fetch-Mode": "cors",
+            "Sec-Fetch-Site": "same-origin",
+            "sec-ch-ua": '"Not;A=Brand";v="99", "Google Chrome";v="139", "Chromium";v="139"',
+            "sec-ch-ua-mobile": "?0",
+            "sec-ch-ua-platform": '"Windows"'
+        },
+        'COOKIES_DEBUG': False,
+        'LOG_LEVEL': 'INFO',
+        'ITEM_PIPELINES': {
+            'kyqb_scrapy.pipelines.DedupAndMySQLPipeline': 300,
+        },
+        'DOWNLOADER_MIDDLEWARES': {
+            'kyqb_scrapy.middlewares.RandomUserAgentMiddleware': 400,
+        }
+    }
+
+    def __init__(self, task='telecom_license', start_page=1, end_page=100, *args, **kwargs):
+        super(MiitSpider, self).__init__(*args, **kwargs)
+
+        if task not in TASKS:
+            raise ValueError(f"不支持的任务: {task}")
+
+        self.task_config = TASKS[task]
+        self.category_id = self.task_config["category_id"]
+        self.item_class = self.task_config["item_class"]
+        self.table_name = self.task_config["table"]
+        self.field_mapping = self.task_config["field_mapping"]
+
+        self.start_page = int(start_page)
+        self.end_page = int(end_page)
+        self.page_size = 5
+
+        # 设置 custom_settings 中的表名（动态）
+        self.custom_settings['MYSQL_TABLE'] = self.table_name
+
+        logger.info(f"🚀 启动任务: {self.task_config['name']}，页码 {self.start_page} ~ {self.end_page}")
+
+    def start_requests(self):
+        for page in range(self.start_page, self.end_page + 1):
+            data = {
+                "categoryId": self.category_id,
+                "currentPage": page,
+                "pageSize": self.page_size,
+                "searchContent": ""
+            }
+            yield Request(
+                url=API_URL,
+                method='POST',
+                body=json.dumps(data, separators=(',', ':')),
+                headers={'Content-Type': 'application/json;charset=UTF-8'},
+                callback=self.parse,
+                meta={'page': page},
+                dont_filter=True
+            )
+
+    def parse(self, response):
+        page = response.meta['page']
+
+        # 检查响应
+        if response.status_code != 200:
+            self.logger.error(f"❌ 第 {page} 页请求失败: HTTP {response.status}")
+            return
+
+        try:
+            result = json.loads(response.text)
+        except json.JSONDecodeError:
+            text = response.text
+            if "升级浏览器" in text or "请尝试" in text:
+                self.logger.error(f"⚠️ 检测到反爬: 请升级浏览器。响应片段: {text[:300]}")
+            else:
+                self.logger.error(f"JSON解析失败: {text[:300]}")
+            return
+
+        if not result.get("success"):
+            msg = result.get("msg", "未知错误")
+            if "升级浏览器" in msg or "请尝试" in msg:
+                self.logger.error(f"⚠️ 反爬提示: {msg}")
+            else:
+                self.logger.error(f"接口失败: {msg}")
+            return
+
+        raw_records = result["params"]["tbAppArticle"]["list"]
+        self.logger.info(f"✅ 第 {page} 页获取 {len(raw_records)} 条数据")
+
+        for record in raw_records:
+            item = self.item_class()
+
+            for src_key, dst_key in self.field_mapping.items():
+                value = record.get(src_key, '')
+                if isinstance(value, str):
+                    value = strip_html(value)
+                item[dst_key] = value
+
+            yield item

examples/gxb/spider/telecom_device_licenses.py

@@ -0,0 +1,129 @@
+# -*- coding: utf-8 -*-
+import json
+from crawlo import Spider, Request
+from crawlo.utils.log import get_logger
+
+from examples.gxb.items import TelecomLicenseItem
+from examples.gxb.settings import HEADERS, COOKIES
+
+
+logger = get_logger(__name__)
+
+class TelecomDeviceLicensesSpider(Spider):
+    name = 'telecom_device_licenses'
+    allowed_domains = ['ythzxfw.miit.gov.cn']
+    # API 的基础 URL
+    base_api_url = 'https://ythzxfw.miit.gov.cn/oldyth/user-center/tbAppSearch/selectResult'
+
+    # 配置：起始页码和结束页码
+    start_page = 1
+    end_page = 26405
+    data = {
+        "categoryId": "144",
+        "currentPage": 1,
+        "pageSize": 5,
+        "searchContent": ""
+    }
+
+
+    def start_requests(self):
+        """从第一页开始，逐页发起请求"""
+
+        yield Request(
+            url=self.base_api_url,
+            method='POST',
+            headers=HEADERS,
+            cookies=COOKIES,
+            body=json.dumps(self.data),
+            callback=self.parse,
+            meta={'page': 1},
+            dont_filter=True
+        )
+
+
+    def parse(self, response):
+        """
+        解析 API 响应
+        :param response: Scrapy Response 对象
+        """
+        page = response.meta['page']
+        self.logger.info(f"正在解析第 {page} 页，状态码: {response.status_code}")
+
+        try:
+            json_data = response.json()
+
+            if not json_data.get('success'):
+                self.logger.error(f"第 {page} 页请求失败: {json_data.get('msg', 'Unknown error')}")
+                return
+
+            # 提取总页数和总记录数（可选，用于验证）
+            total_records = json_data.get("params", {}).get("tbAppArticle", {}).get("total", 0)
+            self.logger.info(f"第 {page} 页，总记录数: {total_records}")
+
+            article_list = json_data.get("params", {}).get("tbAppArticle", {}).get("list", [])
+
+            if not article_list:
+                self.logger.warning(f"第 {page} 页未找到数据")
+                return
+
+            self.logger.info(f"第 {page} 页成功解析到 {len(article_list)} 条记录")
+
+            # 将每条记录作为独立的 item yield 出去
+            for item in article_list:
+                # 清洗数据：移除 HTML 标签
+                cleaned_item = self.clean_item(item)
+                item = TelecomLicenseItem()
+                item['license_number'] = cleaned_item.get('articleField01')
+                item['device_name'] = cleaned_item.get('articleField02')
+                item['device_model'] = cleaned_item.get('articleField03')
+                item['applicant'] = cleaned_item.get('articleField04')
+                item['manufacturer'] = cleaned_item.get('articleField05')
+                item['issue_date'] = cleaned_item.get('articleField06')
+                item['expiry_date'] = cleaned_item.get('articleField07')
+                item['certificate_type'] = cleaned_item.get('articleField08')
+                item['remarks'] = cleaned_item.get('articleField09')
+                item['certificate_status'] = cleaned_item.get('articleField10')
+                item['origin'] = cleaned_item.get('articleField11')
+                item['article_id'] = cleaned_item.get('articleId')
+                item['article_edit_date'] = cleaned_item.get('articleEdate')
+                item['create_time'] = cleaned_item.get('createTime')
+                yield item
+
+            # --- 自动翻页逻辑 ---
+            # 检查是否还有下一页
+            # 方法1：根据当前页码和预设的总页数
+            if page < self.end_page:
+                next_page = page + 1
+                self.data['currentPage'] = next_page
+                self.logger.debug(f"准备爬取下一页: {next_page}")
+                yield Request(
+                    url=self.base_api_url,
+                    method='POST',
+                    headers=HEADERS,
+                    cookies=COOKIES,
+                    body=json.dumps(self.data),
+                    callback=self.parse,
+                    meta={'page': next_page},
+                    dont_filter=True
+                )
+
+        except Exception as e:
+            self.logger.error(f"解析第 {page} 页响应失败: {e}", exc_info=True)
+
+    @staticmethod
+    def clean_item(item: dict) -> dict:
+        """
+        清洗单条记录，移除 HTML 标签等
+        :param item: 原始字典
+        :return: 清洗后的字典
+        """
+        import re
+        html_tag_re = re.compile(r'<[^>]+>')
+        cleaned = {}
+        for k, v in item.items():
+            if isinstance(v, str):
+                # 移除 HTML 标签并去除首尾空白
+                cleaned[k] = html_tag_re.sub('', v).strip()
+            else:
+                cleaned[k] = v
+        return cleaned

tests/__init__.py

@@ -1,7 +1,7 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-"""
-# @Time    :    2025-02-05 12:36
-# @Author  :   oscar
-# @Desc    :   None
-"""
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+# @Time    :    2025-08-24 12:36
+# @Author  :   crawl-coder
+# @Desc    :   None
+"""

tests/test_proxy_health_check.py

@@ -0,0 +1,33 @@
+# tests/test_proxy_health_check.py
+import pytest
+from unittest.mock import AsyncMock, patch
+from crawlo.proxy.health_check import check_single_proxy
+import httpx
+
+
+@pytest.mark.asyncio
+@patch('httpx.AsyncClient')
+async def test_health_check_success(mock_client_class):
+    """测试健康检查：成功"""
+    mock_resp = AsyncMock()
+    mock_resp.status_code = 200
+    mock_client_class.return_value.__aenter__.return_value.get.return_value = mock_resp
+
+    proxy_info = {'url': 'http://good:8080', 'healthy': False}
+    await check_single_proxy(proxy_info)
+
+    assert proxy_info['healthy'] is True
+    assert proxy_info['failures'] == 0
+
+
+@pytest.mark.asyncio
+@patch('httpx.AsyncClient')
+async def test_health_check_failure(mock_client_class):
+    """测试健康检查：失败"""
+    mock_client_class.return_value.__aenter__.return_value.get.side_effect = httpx.ConnectError("Failed")
+
+    proxy_info = {'url': 'http://bad:8080', 'healthy': True, 'failures': 0}
+    await check_single_proxy(proxy_info)
+
+    assert proxy_info['healthy'] is False
+    assert proxy_info['failures'] == 1

tests/test_proxy_middleware_integration.py

@@ -0,0 +1,137 @@
+# tests/test_proxy_middleware_integration.py
+import pytest
+import asyncio
+import time
+from unittest.mock import Mock, AsyncMock, patch
+from crawlo import Request, Response, Spider
+from crawlo.proxy.middleware import ProxyMiddleware
+from crawlo.proxy.stats import ProxyStats
+
+
+@pytest.fixture
+def crawler():
+    class MockSettings:
+        def get(self, key, default=None):
+            defaults = {
+                'PROXY_ENABLED': True,
+                'PROXIES': ['http://p1:8080', 'http://p2:8080'],
+                'PROXY_SELECTION_STRATEGY': 'random',
+                'PROXY_REQUEST_DELAY_ENABLED': False,
+                'PROXY_MAX_RETRY_COUNT': 1,
+            }
+            return defaults.get(key, default)
+
+        def get_bool(self, key, default=None):
+            return self.get(key, default)
+
+        def get_int(self, key, default=None):
+            return self.get(key, default)
+
+        def get_float(self, key, default=None):
+            return self.get(key, default)
+
+        def get_list(self, key, default=None):
+            return self.get(key, default)
+
+    class MockCrawler:
+        def __init__(self):
+            self.settings = MockSettings()
+
+    return MockCrawler()
+
+
+@pytest.fixture
+def middleware(crawler):
+    mw = ProxyMiddleware.create_instance(crawler)
+    mw._load_providers = Mock()
+    mw._update_proxy_pool = AsyncMock()
+    mw._health_check = AsyncMock()
+    mw.scheduler = None
+
+    mw.proxies = [
+        {
+            'url': 'http://p1:8080',
+            'healthy': True,
+            'failures': 0,
+            'last_health_check': 0,
+            'unhealthy_since': 0
+        },
+        {
+            'url': 'http://p2:8080',
+            'healthy': True,
+            'failures': 0,
+            'last_health_check': 0,
+            'unhealthy_since': 0
+        },
+    ]
+    mw.stats = ProxyStats()
+    for p in mw.proxies:
+        mw.stats.record(p['url'], 'total')
+
+    asyncio.get_event_loop().run_until_complete(mw._initial_setup())
+    return mw
+
+
+@pytest.fixture
+def spider():
+    return Mock(spec=Spider, logger=Mock())
+
+
+def test_process_request_sets_proxy(middleware, spider):
+    request = Request("https://example.com")
+    result = asyncio.get_event_loop().run_until_complete(
+        middleware.process_request(request, spider)
+    )
+    assert result is None
+    assert hasattr(request, 'proxy')
+    assert request.proxy in ['http://p1:8080', 'http://p2:8080']
+
+
+def test_process_response_records_success(middleware, spider):
+    request = Request("https://example.com")
+    request.proxy = 'http://p1:8080'
+    response = Response("https://example.com", body=b"ok", headers={})
+    middleware.stats.record(request.proxy, 'total')
+    middleware.process_response(request, response, spider)
+    assert middleware.stats.get(request.proxy)['success'] == 1
+
+
+def test_process_exception_switches_proxy(middleware, spider):
+    request = Request("https://example.com")
+    request.proxy = 'http://p1:8080'
+    request.meta['proxy_retry_count'] = 0
+
+    result = middleware.process_exception(request, Exception("Timeout"), spider)
+    assert result is not None
+    assert result.proxy != 'http://p1:8080'
+    assert result.meta['proxy_retry_count'] == 1
+
+    final = middleware.process_exception(result, Exception("Timeout"), spider)
+    assert final is None
+
+
+def test_mark_failure_disables_proxy(middleware):
+    proxy_url = 'http://p1:8080'
+    p = next(p for p in middleware.proxies if p['url'] == proxy_url)
+    p['failures'] = 2
+
+    middleware._mark_failure(proxy_url)
+    assert p['failures'] == 3
+    assert p['healthy'] is False
+    assert p['unhealthy_since'] > 0
+
+
+@pytest.mark.asyncio
+async def test_request_delay(middleware, spider):
+    """测试请求延迟功能：验证是否调用了 asyncio.sleep"""
+    with patch("crawlo.proxy.middleware.asyncio.sleep", new_callable=AsyncMock) as mock_sleep:
+        middleware.delay_enabled = True  # 注意：这里应该是 delay_enabled 而不是 request_delay_enabled
+        middleware.request_delay = 0.1
+        middleware._last_req_time = time.time() - 0.05  # 50ms 前
+
+        request = Request("https://a.com")
+        await middleware.process_request(request, spider)
+
+        mock_sleep.assert_called_once()
+        delay = mock_sleep.call_args[0][0]
+        assert 0.04 <= delay <= 0.06

tests/test_proxy_providers.py

@@ -0,0 +1,57 @@
+# tests/test_proxy_providers.py
+import pytest
+import pytest
+import respx
+from httpx import Response
+from crawlo.proxy.providers import StaticProxyProvider, FileProxyProvider, APIProxyProvider
+import tempfile
+import os
+
+
+@pytest.mark.asyncio
+async def test_static_provider():
+    """测试静态代理提供者"""
+    provider = StaticProxyProvider(['http://1.1.1.1:8080', 'http://2.2.2.2:8080'])
+    proxies = await provider.fetch_proxies()
+    assert len(proxies) == 2
+    assert 'http://1.1.1.1:8080' in proxies
+    assert 'http://2.2.2.2:8080' in proxies
+
+
+@pytest.mark.asyncio
+async def test_file_provider():
+    """测试文件代理提供者"""
+    with tempfile.NamedTemporaryFile(mode='w', delete=False) as f:
+        f.write("http://a.com:8080\nhttp://b.com:8080\n")
+        temp_path = f.name
+    try:
+        provider = FileProxyProvider(temp_path)
+        proxies = await provider.fetch_proxies()
+        assert len(proxies) == 2
+        assert 'http://a.com:8080' in proxies
+        assert 'http://b.com:8080' in proxies
+    finally:
+        os.unlink(temp_path)
+
+
+@pytest.mark.asyncio
+@respx.mock
+async def test_api_provider():
+    """使用 respx 拦截 HTTP 请求，更简洁可靠"""
+    # 拦截 GET 请求
+    respx.get("https://api.example.com").mock(
+        return_value=Response(
+            200,
+            json=[
+                {"ip": "1.1.1.1", "port": 8080},
+                {"ip": "2.2.2.2", "port": 8080}
+            ]
+        )
+    )
+
+    provider = APIProxyProvider(url="https://api.example.com")
+    proxies = await provider.fetch_proxies()
+
+    assert len(proxies) == 2
+    assert "http://1.1.1.1:8080" in proxies
+    assert "http://2.2.2.2:8080" in proxies

tests/test_proxy_stats.py

@@ -0,0 +1,20 @@
+# tests/test_proxy_stats.py
+from crawlo.proxy.stats import ProxyStats
+
+
+def test_proxy_stats():
+    """测试代理统计功能"""
+    stats = ProxyStats()
+    url = 'http://proxy1:8080'
+
+    stats.record(url, 'success')
+    stats.record(url, 'success')
+    stats.record(url, 'failure')
+
+    assert stats.get(url)['success'] == 2
+    assert stats.get(url)['failure'] == 1
+    assert stats.get(url)['total'] == 3
+
+    all_data = stats.all()
+    assert url in all_data
+    assert all_data[url]['success'] == 2