crawlo 1.0.9__py3-none-any.whl → 1.1.1__py3-none-any.whl

crawlo/utils/project.py

@@ -1,197 +0,0 @@
-#!/usr/bin/python
-# -*- coding: UTF-8 -*-
-"""
-自动发现项目并创建 SettingManager 实例
-
-该模块负责：
-1.  向上搜索项目根目录（通过 crawlo.cfg 或 settings.py）
-2.  将项目根目录加入 Python 路径 (sys.path)
-3.  加载指定的 settings 模块
-4.  返回一个已配置好的 SettingManager 实例
-"""
-import os
-import sys
-import configparser
-from importlib import import_module
-from inspect import iscoroutinefunction
-from typing import Callable, Optional
-
-from crawlo.utils.log import get_logger
-from crawlo.settings.setting_manager import SettingManager
-
-logger = get_logger(__name__)
-
-
-def _find_project_root(start_path: str = '.') -> Optional[str]:
-    """
-    从指定的起始路径开始，向上级目录递归搜索，寻找项目根目录。
-    搜索依据：
-        1. 优先查找 'crawlo.cfg' 文件。
-        2. 如果未找到 cfg 文件，则查找位于 Python 包内（即包含 __init__.py 的目录）的 'settings.py' 文件。
-
-    Args:
-        start_path (str): 搜索的起始路径，默认为当前工作目录 '.'。
-
-    Returns:
-        Optional[str]: 找到的项目根目录的绝对路径，如果未找到则返回 None。
-    """
-    path = os.path.abspath(start_path)
-
-    while True:
-        # 1. 检查是否存在 crawlo.cfg 文件
-        cfg_file = os.path.join(path, 'crawlo.cfg')
-        if os.path.isfile(cfg_file):
-            return path
-
-        # 2. 检查是否存在 settings.py 文件，并且它位于一个 Python 包中
-        settings_file = os.path.join(path, 'settings.py')
-        if os.path.isfile(settings_file):
-            init_file = os.path.join(path, '__init__.py')
-            if os.path.isfile(init_file):
-                return path
-            else:
-                logger.debug(f"在路径 {path} 找到 'settings.py'，但缺少 '__init__.py'，忽略。")
-
-        # 移动到上一级目录
-        parent = os.path.dirname(path)
-        if parent == path:
-            # 已经到达文件系统根目录
-            break
-        path = parent
-
-    logger.warning("向上搜索完毕，未找到项目根目录。")
-    return None
-
-
-def _get_settings_module_from_cfg(cfg_path: str) -> str:
-    """
-    从 crawlo.cfg 配置文件中读取 settings 模块的路径。
-
-    Args:
-        cfg_path (str): crawlo.cfg 文件的完整路径。
-
-    Returns:
-        str: settings 模块的导入路径，例如 'myproject.settings'。
-
-    Raises:
-        RuntimeError: 当读取文件或解析配置出错时抛出。
-    """
-    logger.info(f"正在读取配置文件: {cfg_path}")
-    config = configparser.ConfigParser()
-    try:
-        config.read(cfg_path, encoding='utf-8')
-        if config.has_section('settings') and config.has_option('settings', 'default'):
-            module_path = config.get('settings', 'default')
-            logger.debug(f"从 'crawlo.cfg' 中读取到 settings 模块路径: {module_path}")
-            return module_path
-        else:
-            error_msg = f"配置文件 '{cfg_path}' 缺少 '[settings]' 或 'default' 配置项。"
-            logger.error(error_msg)
-            raise RuntimeError(error_msg)
-    except (configparser.Error, OSError) as e:
-        error_msg = f"读取或解析配置文件 '{cfg_path}' 时出错: {e}"
-        logger.error(error_msg)
-        raise RuntimeError(error_msg)
-
-
-def get_settings(custom_settings=None):
-    """
-    获取配置管理器实例的主函数。
-    此函数会自动发现项目，加载配置，并返回一个配置好的 SettingManager。
-
-    Args:
-        custom_settings (dict, optional): 运行时传入的自定义设置字典，会覆盖 settings.py 中的同名配置。
-
-    Returns:
-        SettingManager: 一个已加载所有配置的 SettingManager 实例。
-
-    Raises:
-        RuntimeError: 当无法找到项目或配置文件时。
-        ImportError: 当无法导入指定的 settings 模块时。
-    """
-    logger.debug("正在初始化配置管理器...")
-
-    # 1. 发现项目根目录
-    project_root = _find_project_root()
-    if not project_root:
-        error_msg = "未找到 Crawlo 项目。请确保您正在包含 'crawlo.cfg' 或 'settings.py' 的项目目录中运行。"
-        logger.error(error_msg)
-        raise RuntimeError(error_msg)
-
-    logger.debug(f"项目根目录已确定: {project_root}")
-
-    # 2. 确定 settings 模块的导入路径
-    settings_module_path = None
-
-    # 优先从 crawlo.cfg 中读取
-    cfg_file = os.path.join(project_root, 'crawlo.cfg')
-    if os.path.isfile(cfg_file):
-        settings_module_path = _get_settings_module_from_cfg(cfg_file)
-    else:
-        logger.debug("未找到 'crawlo.cfg'，尝试推断 settings 模块路径...")
-        # 推断：项目目录名.settings
-        project_name = os.path.basename(project_root)
-        settings_module_path = f"{project_name}.settings"
-        logger.debug(f"推断 settings 模块路径为: {settings_module_path}")
-
-    # 3. 将项目根目录添加到 Python 路径，确保可以成功导入
-    if project_root not in sys.path:
-        sys.path.insert(0, project_root)
-        logger.debug(f"已将项目根目录 '{project_root}' 添加到 Python 路径。")
-    else:
-        logger.debug(f"项目根目录 '{project_root}' 已在 Python 路径中。")
-
-    # 4. 创建 SettingManager 并加载配置
-    logger.debug(f"正在加载 settings 模块: {settings_module_path}")
-    settings = SettingManager()
-
-    try:
-        # 这会触发 SettingManager.set_settings()，从模块中加载所有大写常量
-        settings.set_settings(settings_module_path)
-        logger.debug("settings 模块加载成功。")
-    except Exception as e:
-        error_msg = f"加载 settings 模块 '{settings_module_path}' 失败: {e}"
-        logger.error(error_msg)
-        raise ImportError(error_msg)
-
-    # 5. 应用运行时自定义设置
-    if custom_settings:
-        logger.debug(f"正在应用运行时自定义设置: {custom_settings}")
-        settings.update_attributes(custom_settings)
-        logger.info("运行时自定义设置已应用。")
-
-    logger.debug("配置管理器初始化完成。")
-    return settings
-
-
-def load_class(_path):
-    if not isinstance(_path, str):
-        if callable(_path):
-            return _path
-        else:
-            raise TypeError(f"args expect str or object, got {_path}")
-
-    module_name, class_name = _path.rsplit('.', 1)
-    module = import_module(module_name)
-
-    try:
-        cls = getattr(module, class_name)
-    except AttributeError:
-        raise NameError(f"Module {module_name!r} has no class named {class_name!r}")
-    return cls
-
-
-def merge_settings(spider, settings):
-    spider_name = getattr(spider, 'name', 'UnknownSpider')
-    if hasattr(spider, 'custom_settings'):
-        custom_settings = getattr(spider, 'custom_settings')
-        settings.update_attributes(custom_settings)
-    else:
-        logger.debug(f"爬虫 '{spider_name}' 无 custom_settings，跳过合并")  # 添加日志
-
-
-async def common_call(func: Callable, *args, **kwargs):
-    if iscoroutinefunction(func):
-        return await func(*args, **kwargs)
-    else:
-        return func(*args, **kwargs)

crawlo-1.0.9.dist-info/METADATA

@@ -1,49 +0,0 @@
-Metadata-Version: 2.4
-Name: crawlo
-Version: 1.0.9
-Summary: Crawlo 是一款基于异步IO的高性能Python爬虫框架，支持分布式抓取。
-Home-page: https://github.com/crawl-coder/Crawlo.git
-Author: crawl-coder
-Author-email: crawlo@qq.com
-License: MIT
-Classifier: Programming Language :: Python :: 3
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Operating System :: OS Independent
-Requires-Python: >=3.6
-Description-Content-Type: text/markdown
-Requires-Dist: aiohttp>=3.12.14
-Requires-Dist: aiomysql>=0.2.0
-Requires-Dist: aioredis>=2.0.1
-Requires-Dist: asyncmy>=0.2.10
-Requires-Dist: cssselect>=1.2.0
-Requires-Dist: dateparser>=1.2.2
-Requires-Dist: httpx[http2]>=0.27.0
-Requires-Dist: curl-cffi>=0.13.0
-Requires-Dist: lxml>=5.2.1
-Requires-Dist: motor>=3.7.0
-Requires-Dist: parsel>=1.9.1
-Requires-Dist: pydantic>=2.11.7
-Requires-Dist: pymongo>=4.11
-Requires-Dist: PyMySQL>=1.1.1
-Requires-Dist: python-dateutil>=2.9.0.post0
-Requires-Dist: redis>=6.2.0
-Requires-Dist: requests>=2.32.4
-Requires-Dist: six>=1.17.0
-Requires-Dist: ujson>=5.9.0
-Requires-Dist: urllib3>=2.5.0
-Requires-Dist: w3lib>=2.1.2
-Provides-Extra: render
-Requires-Dist: webdriver-manager>=4.0.0; extra == "render"
-Requires-Dist: playwright; extra == "render"
-Requires-Dist: selenium>=3.141.0; extra == "render"
-Provides-Extra: all
-Requires-Dist: bitarray>=1.5.3; extra == "all"
-Requires-Dist: PyExecJS>=1.5.1; extra == "all"
-Requires-Dist: pymongo>=3.10.1; extra == "all"
-Requires-Dist: redis-py-cluster>=2.1.0; extra == "all"
-Requires-Dist: webdriver-manager>=4.0.0; extra == "all"
-Requires-Dist: playwright; extra == "all"
-Requires-Dist: selenium>=3.141.0; extra == "all"
-
-# Crawlo
-Crawlo 是一款基于异步IO的高性能Python爬虫框架，支持分布式抓取与数据管道。

crawlo-1.0.9.dist-info/RECORD

@@ -1,97 +0,0 @@
-crawlo/__init__.py,sha256=xpiIAZbSG3CzneJuDLPCbwfRcvw2wyHYl2kJjaNfNGY,584
-crawlo/__version__.py,sha256=5fdKqtSBBDvdwuTWCGoh62x6-wR269e8DEQnOPkCHWg,23
-crawlo/cli.py,sha256=hjAJKx9pba375sATvvcy-dtZyBIgXj8fRBq9RFIZHA4,1206
-crawlo/crawler.py,sha256=AyKxUyJvCwb1u4d3Zn3vFmjH28ExWKIygfTICps-3yY,20026
-crawlo/event.py,sha256=ZhoPW5CglCEuZNFEwviSCBIw0pT5O6jT98bqYrDFd3E,324
-crawlo/exceptions.py,sha256=xdyZkvVcLEJ-19sWMHvn9IJsu30-hAY2jJhA2kYIims,1207
-crawlo/stats_collector.py,sha256=v4jC9BAe-23w93hWzbeMCCgQ9VuFPyxw5JV9ItbGH8w,1636
-crawlo/subscriber.py,sha256=udlHeTR0ymGQhCDxVUGwUzeeeR4TYCEJrJwFnkgr0cU,3836
-crawlo/task_manager.py,sha256=D9m-nqnGj-FZPtGk4CdwZX3Gw7IWyYvTS7CHpRGWc_w,748
-crawlo/commands/__init__.py,sha256=kZ3qATqDPmMUCNUQSFfBfIA8fp_1dgBwIAWbmFN3_To,355
-crawlo/commands/check.py,sha256=Q8wFjIo43XW0wP93TTlM7HSShgytJsbSWHIlmkcNxz0,3585
-crawlo/commands/genspider.py,sha256=kSHYsAGHRoxU6Qf_MGpR_VS-Ua5NUGY2KGm_Wapn0sw,3529
-crawlo/commands/list.py,sha256=itR05muZlZs8FbRh88kOhcRbZc77OXiR6A86UnVhSMY,2974
-crawlo/commands/run.py,sha256=s6JJC8HNa-tBgPDB2BPUmj26D7PMckhlx4AOEz57ESY,6197
-crawlo/commands/startproject.py,sha256=1KOq_CALy01oklr0dAUYhGFzu4f7w45q2H0O3qafLX4,3494
-crawlo/commands/stats.py,sha256=rH0TlD0o-xUr9RxtvNYgnSjHHoRyma3rvx9Q9nIGDNg,1659
-crawlo/core/__init__.py,sha256=JYSAn15r8yWgRK_Nc69t_8tZCyb70MiPZKssA8wrYz0,43
-crawlo/core/engine.py,sha256=JFHooPp-5cfHSyxEh87nOOR5NMaPLVDfNSqAsbtx4PM,6030
-crawlo/core/processor.py,sha256=oHLs-cno0bJGTNc9NGD2S7_2-grI3ruvggO0SY2mf3Q,1180
-crawlo/core/scheduler.py,sha256=ZMPs4LSs69FsFfDTvaOMJKqpSQQGvIEE9pMyYVVAA64,1948
-crawlo/downloader/__init__.py,sha256=72u2Hef4HaMfs9VCqEjbMtiaRXbaXmgNiJn6qy09LHs,2384
-crawlo/downloader/aiohttp_downloader.py,sha256=YfvYCDp3y0OsVyfdYX1XJC3EcCrbNLKOcFY8b7JC3_w,7675
-crawlo/downloader/cffi_downloader.py,sha256=QthBmZOE0cjYNRTM-449EuaFuqxxdc19kp93iqOlwB8,12678
-crawlo/downloader/httpx_downloader.py,sha256=yshb1JZa4B6hcVwIT97SrxCXkj3HJqT9IDpxSxjkJm4,11754
-crawlo/extension/__init__.py,sha256=O2BVK1U3WwmurZb-PaYVz3g1tZ_iYUjCwilmUKf6844,1170
-crawlo/extension/log_interval.py,sha256=FOWeTOuWtOpCz2UPV5F_--QIa8yomltSpjxbw3F7bkU,1971
-crawlo/extension/log_stats.py,sha256=JFJsdK7UWkhP4TEAF-H-S7SpQbDpBryS0AT6e6jZCBo,1721
-crawlo/extension/logging_extension.py,sha256=rty2_up53KV05nCazuBuz2ZapHKq0ti7mGVBzMTr0ak,1236
-crawlo/filters/__init__.py,sha256=9fJQRVkxWWPChajYbAGe1O6UYB639xWt0hiLUGBs4hQ,1014
-crawlo/filters/aioredis_filter.py,sha256=phBFW9Z28oylbik9Kb2WHM65Wo5yRAH2w9Yz0_2HaOQ,5621
-crawlo/filters/memory_filter.py,sha256=L8XEJkObOxs4BzYpQvk9PVM969k2LE61VFsnEOTEf_E,6841
-crawlo/items/__init__.py,sha256=rFpx1qFBo0Ik7bSdnXC8EVTJUOQdoJYGVdhYjaH00nk,409
-crawlo/items/base.py,sha256=hwGJEdFWOdaZfalFX8umRkh_HUWLEbCjvq4j70fplMQ,598
-crawlo/items/fields.py,sha256=fpS0vlRPpZYjTaMDgI9Q8z_YQqruwf6fi4Dgm6R2oEk,1854
-crawlo/items/items.py,sha256=OmVEvMmgofMU95GkaiWkfNQ2fjsH2fY9sw3SKcmUhLs,3478
-crawlo/middleware/__init__.py,sha256=PSwpRLdBUopaQzBp1S0zK_TZbrRagQ4yzvgyLy4tBk8,570
-crawlo/middleware/default_header.py,sha256=OVW4vpRPp3Y6qYXtiEYlGqVjCYcbuv1Iecc7zEgwCsI,1099
-crawlo/middleware/download_delay.py,sha256=P2eyAJXwdLdC4yYuLhvKZVa1b5YQvQD0GpsR8aDW8-8,994
-crawlo/middleware/middleware_manager.py,sha256=Vfkasi8YaLxzGrOrFYfxOMEGRS8XocqeQMruLtVxL_c,6360
-crawlo/middleware/proxy.py,sha256=PiIfhRXfcMzBtW_p7jfR8rGxcM4VT68Mk54swbaV2H4,9801
-crawlo/middleware/request_ignore.py,sha256=jdybWFVXuA5YsAPfZJFzLTWkYhEAewNgxuhFqczPW9M,1027
-crawlo/middleware/response_code.py,sha256=vgXWv3mMu_v9URvhKA9myIFH4u6L4EwNme80wL4DCGc,677
-crawlo/middleware/response_filter.py,sha256=O2gkV_Yjart8kmmXTGzrtZnb_Uuefap4uL2Cu01iRs4,863
-crawlo/middleware/retry.py,sha256=a2EmigYFzL8oxd50JhrSe5XbYJyx8yDjOjE5fXAOFhY,3459
-crawlo/network/__init__.py,sha256=DVz1JpasjxCgOlXvm76gz-S18OXr4emG_J39yi5iVuA,130
-crawlo/network/request.py,sha256=qd50mmrXS6yZKmAb6ERAMHzm2Ln80Wu5NSMwx_t1IGc,7247
-crawlo/network/response.py,sha256=z2Owti_9ds567jLvfuX8hrfdQL8JKn5lkt2QOc-Gi3Y,6200
-crawlo/pipelines/__init__.py,sha256=IbXJ6B8LqxVVjeLNgL_12AxV6zbV8hNRQxAfMLjjSaw,273
-crawlo/pipelines/console_pipeline.py,sha256=bwe5hZgaVSWmh3R8XpOaaeAjJme-Ttrpo6G6f1cnLIg,1287
-crawlo/pipelines/mongo_pipeline.py,sha256=Yr48D0T61-_Y-EpgWXf7BUn9w8e-Pj5P07QDSPZ0pYU,4558
-crawlo/pipelines/mysql_batch_pipline.py,sha256=Mj3PReDRw22JhJ5hZxnka4cirKq3kEbOCNhgpq1gvfA,10611
-crawlo/pipelines/mysql_pipeline.py,sha256=bsAFqpxrCijzvX-IusxOtvTvQEUCt5uHNTyYMo_pIq4,8056
-crawlo/pipelines/pipeline_manager.py,sha256=k-Rg0os0Havrov99D-Jn3ROpnz154K30tf7aARE5W3k,2174
-crawlo/settings/__init__.py,sha256=NgYFLfk_Bw7h6KSoepJn_lMBSqVbCHebjKxaE3_eMgw,130
-crawlo/settings/default_settings.py,sha256=urj4XJ--ZpVRbbo3fWUT71bYQLmElx43AC9KeHtqHBs,7310
-crawlo/settings/setting_manager.py,sha256=4xXOzKwZCgAp8ybwvVcs2R--CsOD7c6dBIkj6DJHB3c,2998
-crawlo/spider/__init__.py,sha256=IyQd4ufbAIhA_cvWrsNReRv3tj76CHc5Aef9c8KR-9s,3983
-crawlo/templates/crawlo.cfg.tmpl,sha256=lwiUVe5sFixJgHFEjn1OtbAeyWsECOrz37uheuVtulk,240
-crawlo/templates/project/__init__.py.tmpl,sha256=aQnHaOjMSkTviOC8COUX0fKymuyf8lx2tGduxkMkXEE,61
-crawlo/templates/project/items.py.tmpl,sha256=bXx-oCldMr2EgBKUAH9LH5gMnbyLiWX-EySAaMzcu2g,318
-crawlo/templates/project/middlewares.py.tmpl,sha256=VAolmMTC6HBmJT5XvWB0ag6ig9iaDBS32adIQ1zPdw0,2177
-crawlo/templates/project/pipelines.py.tmpl,sha256=xK1Yl7wYxiyUCm07GZvMnCS_cxJ5LF7z1YBBdkLlWys,1880
-crawlo/templates/project/settings.py.tmpl,sha256=985Z-jiU6A31f5s1IVU4PvkC_QGlFlRRfTF6rZ_G4ek,1771
-crawlo/templates/project/spiders/__init__.py.tmpl,sha256=zMbePipgLsctQUEnda4WkHz8rDLUX--rc8ruI6zkpWc,111
-crawlo/templates/spider/spider.py.tmpl,sha256=SkNv1kOwet7ZdxoNXpj-o1iRETB30bcwPP16Uy8lyXg,869
-crawlo/utils/__init__.py,sha256=XCYumI8wJ1jU_Myn_K0LT-LVygPDUCdETCbXM3EWvlo,130
-crawlo/utils/concurrency_manager.py,sha256=o-_cfeUHdlBOM3eAXF857MtekSrRcVTBJ2jWZvY6weQ,5230
-crawlo/utils/date_tools.py,sha256=lcEFP2Z5b-6pUTHczrzCCuqiHP_4_2zamomMGPZrExo,7194
-crawlo/utils/db_helper.py,sha256=ZqOt1d3mErVv4TOvoWlov0niUxORB9aHByTmMoNFIDw,10902
-crawlo/utils/func_tools.py,sha256=y-TYP9H3X67MS_foWy9Z2LIS6GP7Y4Cy3T168ulq3Jc,2451
-crawlo/utils/log.py,sha256=YD2FfXuuE2MC9ZdQQZ0H7KysE7l_LHZqQepaTPlcApo,4133
-crawlo/utils/pqueue.py,sha256=HDgX4HAkc7RqYUtX6q51tzI1ZRTACf8P_4jLqC4-uC0,5559
-crawlo/utils/project.py,sha256=hXSKV55OBUFjJi7TXekB4X3MmAgsqAeVTj5wPUWOizc,7394
-crawlo/utils/request.py,sha256=ejdKpTwc-HE04HQybafhOVywzz57IV3pY0YMkSLyGUo,9065
-crawlo/utils/spider_loader.py,sha256=V0CBTicJBYBZafhwLfDEfuEc_hJ2mSoiptT6qKufI9U,2249
-crawlo/utils/system.py,sha256=24zGmtHNhDFMGVo7ftMV-Pqg6_5d63zsyNey9udvJJk,248
-crawlo/utils/tools.py,sha256=uy7qw5Z1BIhyEgiHENvtM7WoGCJxlS8EX3PmOA7ouCo,275
-crawlo/utils/url.py,sha256=RKe_iqdjafsNcp-P2GVLYpsL1qbxiuZLiFc-SqOQkcs,1521
-examples/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-examples/gxb/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-examples/gxb/items.py,sha256=3-1Lxpi7EqMzheDJoO0MPyHky5nHG_nqQGgKlm8y6mQ,989
-examples/gxb/run.py,sha256=9kJlR8f-tZ3BqP5PW7sCLTw6PAFWo3x4cG5lc-6GWqI,333
-examples/gxb/settings.py,sha256=_nbXj9HV2e0F6liUzK0ueygLcaMM_IUlkuwL6mJqUfc,2345
-examples/gxb/spider/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-examples/gxb/spider/miit_spider.py,sha256=tcQnuyUHfu-Re1QbKKSI9DXW3Sp1vyBW8qBzKLf_RC4,6666
-examples/gxb/spider/telecom_device.py,sha256=58iG6BQtQjjDHOF7-DXH0u5_XnppP5AJTQwaVJVyBEo,4929
-tests/__init__.py,sha256=409aRX8hsPffiZCVjOogtxwhACzBp8G2UTJyUQSxhK0,136
-tests/test_proxy_health_check.py,sha256=_tDlxa_6TdL3M5RLkHF82roXJ8WIuG5hELBp2GADyKQ,1123
-tests/test_proxy_middleware_integration.py,sha256=mTPK_XvbmLCV_QoVZzA3ybWOOX61493Ew78WfTp-bYQ,4441
-tests/test_proxy_providers.py,sha256=u_R2fhab90vqvQEaOAztpAOe9tJXvUMIdoDxmStmXJ4,1749
-tests/test_proxy_stats.py,sha256=ES00CEoDITYPFBGPk8pecFzD3ItYIv6NSpcqNd8-kvo,526
-tests/test_proxy_strategies.py,sha256=9Z1pXmTNyw-eIhGXlf2abZbJx6igLohYq-_3hldQ5uE,1868
-crawlo-1.0.9.dist-info/METADATA,sha256=yvso4RU1U2ntTLfw9hVj_AkZmD5ygve40Oo0haLkbHw,1825
-crawlo-1.0.9.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-crawlo-1.0.9.dist-info/entry_points.txt,sha256=5HoVoTSPxI8SCa5B7pQYxLSrkOdiunyO9tqNsLMv52g,43
-crawlo-1.0.9.dist-info/top_level.txt,sha256=keG_67pbZ_wZL2dmDRA9RMaNHTaV_x_oxZ9DKNgwvR0,22
-crawlo-1.0.9.dist-info/RECORD,,

examples/gxb/items.py

@@ -1,36 +0,0 @@
-from crawlo.items import Item, Field
-
-class RadioApprovalItem(Item):
-    approval_number = Field()
-    device_name = Field()
-    device_model = Field()
-    applicant = Field()
-    remarks = Field()
-    validity_period = Field()
-    frequency_tolerance = Field()
-    frequency_range = Field()
-    transmit_power = Field()
-    occupied_bandwidth = Field()
-    spurious_emission_limit = Field()
-    issue_date = Field()
-    approval_code = Field()
-    cmiit_id = Field()
-    modulation_mode = Field()
-    technology_system = Field()
-    mid = Field()
-
-class TelecomLicenseItem(Item):
-    license_number = Field()
-    device_name = Field()
-    device_model = Field()
-    applicant = Field()
-    manufacturer = Field()
-    issue_date = Field()
-    expiry_date = Field()
-    certificate_type = Field()
-    remarks = Field()
-    certificate_status = Field()
-    origin = Field()
-    article_id = Field()
-    article_edit_date = Field()
-    create_time = Field()

examples/gxb/run.py

@@ -1,16 +0,0 @@
-import asyncio
-
-from crawlo.crawler import CrawlerProcess
-from examples.gxb.spider.telecom_device import TelecomDeviceLicensesSpider
-
-async def main():
-    process = CrawlerProcess()
-    await process.crawl(
-        [TelecomDeviceLicensesSpider]
-    )
-
-
-
-if __name__ == '__main__':
-    asyncio.run(main())
-    # 132023

examples/gxb/settings.py

@@ -1,72 +0,0 @@
-import platform
-
-PROXY_ENABLED = True
-PROJECT_PACKAGE = 'gxb'
-
-# API 地址
-PROXY_API_URL = 'http://123.56.42.142:5000/proxy/getitem/'
-
-# 提取方式（根据实际返回结构选择）
-PROXY_EXTRACTOR = "proxy"
-# 或
-# from utils.proxy_extractors import custom_extractor_proxy
-# PROXY_EXTRACTOR = custom_extractor_proxy
-
-# 刷新间隔
-PROXY_REFRESH_INTERVAL = 5
-
-CONCURRENCY = 3
-
-# 超时时间
-PROXY_API_TIMEOUT = 10
-
-if platform.system() == "Windows":
-    MYSQL_HOST = "pc-2ze9oh2diu5e5firh.rwlb.rds.aliyuncs.com"
-else:
-    MYSQL_HOST = "tianmai-k8s-dmadmin-x.rwlb.rds.aliyuncs.com"
-
-# 数据库端口
-MYSQL_PORT = 3306
-# 数据库用户名
-MYSQL_USER = "data_collection"
-# 数据库密码
-MYSQL_PASSWORD = "CRNabzFQ2H"
-# 数据库名
-MYSQL_DB = "cxzx_xm"
-# 数据库编码
-MYSQL_TABLE = "telecom_device_licenses_v4"
-
-MYSQL_BATCH_SIZE = 100
-
-PIPELINES = [
-    'crawlo.pipelines.console_pipeline.ConsolePipeline',
-    # 'crawlo.pipelines.mysql_pipeline.AsyncmyMySQLPipeline', # 可选：存入 MySQL
-]
-
-
-HEADERS = {
-            "Accept": "application/json, text/plain, */*",
-            "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
-            "Authorization": "null",
-            "Cache-Control": "no-cache",
-            "Connection": "keep-alive",
-            "Content-Type": "application/json;charset=UTF-8",
-            "Origin": "https://ythzxfw.miit.gov.cn",
-            "Pragma": "no-cache",
-            "Referer": "https://ythzxfw.miit.gov.cn/oldyth/resultQuery",
-            "Sec-Fetch-Dest": "empty",
-            "Sec-Fetch-Mode": "cors",
-            "Sec-Fetch-Site": "same-origin",
-            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36",
-            "sec-ch-ua": '"Not;A=Brand";v="99", "Google Chrome";v="139", "Chromium";v="139"',
-            "sec-ch-ua-mobile": "?0",
-            "sec-ch-ua-platform": '"Windows"'
-        }
-
-COOKIES = {
-    "wzws_sessionid": "oGivsIOAMjQwZTozYjM6MzBiMjo3MWMwOjg0NmY6MzQ4OTozNWZjOjEyMTGBOGY2OTQ2gjdjYmMyNQ==",
-    "ariauseGraymode": "false",
-    "Hm_lvt_a73626d298a849004aacc34159f68abd": "1755909741,1756084244,1756256541,1756344453",
-    "Hm_lpvt_a73626d298a849004aacc34159f68abd": "1756344453",
-    "HMACCOUNT": "08DF0D235A291EAA"
-}

examples/gxb/spider/miit_spider.py

@@ -1,180 +0,0 @@
-import json
-import logging
-import re
-
-from crawlo import Request, Spider
-
-from examples.gxb.items import RadioApprovalItem, TelecomLicenseItem
-
-logger = logging.getLogger(__name__)
-
-# 基础配置
-BASE_URL = "https://ythzxfw.miit.gov.cn"
-API_URL = BASE_URL + "/oldyth/user-center/tbAppSearch/selectResult"
-
-# 任务配置
-TASKS = {
-    "radio_approval": {
-        "name": "无线电设备型号核准",
-        "category_id": "352",
-        "item_class": RadioApprovalItem,
-        "table": "radio_equipment_approval_new",
-        "field_mapping": {
-            'articleField01': 'approval_number',
-            'articleField02': 'device_name',
-            'articleField03': 'device_model',
-            'articleField04': 'applicant',
-            'articleField05': 'remarks',
-            'articleField06': 'validity_period',
-            'articleField07': 'frequency_tolerance',
-            'articleField08': 'frequency_range',
-            'articleField09': 'transmit_power',
-            'articleField10': 'occupied_bandwidth',
-            'articleField11': 'spurious_emission_limit',
-            'articleField12': 'issue_date',
-            'articleField13': 'approval_code',
-            'articleField14': 'cmiit_id',
-            'articleField15': 'modulation_mode',
-            'articleField16': 'technology_system',
-        }
-    },
-    "telecom_license": {
-        "name": "电信设备进网许可证",
-        "category_id": "144",
-        "item_class": TelecomLicenseItem,
-        "table": "telecom_device_licenses_new",
-        "field_mapping": {
-            'articleField01': 'license_number',
-            'articleField02': 'device_name',
-            'articleField03': 'device_model',
-            'articleField04': 'applicant',
-            'articleField05': 'manufacturer',
-            'articleField06': 'issue_date',
-            'articleField07': 'expiry_date',
-            'articleField08': 'certificate_type',
-            'articleField09': 'remarks',
-            'articleField10': 'certificate_status',
-            'articleField11': 'origin',
-        }
-    }
-}
-
-def strip_html(text: str) -> str:
-    """去除 HTML 标签"""
-    if not text or not isinstance(text, str):
-        return text
-    return re.sub(r'<[^>]+>', '', text)
-
-class MiitSpider(Spider):
-    name = "miit_spider"
-    custom_settings = {
-        'DOWNLOAD_DELAY': 0.5,
-        'CONCURRENT_REQUESTS': 5,
-        'CONCURRENT_REQUESTS_PER_DOMAIN': 5,
-        'COOKIES_ENABLED': True,
-        'RETRY_TIMES': 3,
-        'DEFAULT_REQUEST_HEADERS': {
-            "Accept": "application/json, text/plain, */*",
-            "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
-            "Authorization": "null",
-            "Cache-Control": "no-cache",
-            "Connection": "keep-alive",
-            "Content-Type": "application/json;charset=UTF-8",
-            "Origin": BASE_URL,
-            "Pragma": "no-cache",
-            "Referer": f"{BASE_URL}/oldyth/resultQuery",
-            "Sec-Fetch-Dest": "empty",
-            "Sec-Fetch-Mode": "cors",
-            "Sec-Fetch-Site": "same-origin",
-            "sec-ch-ua": '"Not;A=Brand";v="99", "Google Chrome";v="139", "Chromium";v="139"',
-            "sec-ch-ua-mobile": "?0",
-            "sec-ch-ua-platform": '"Windows"'
-        },
-        'COOKIES_DEBUG': False,
-        'LOG_LEVEL': 'INFO',
-        'ITEM_PIPELINES': {
-            'kyqb_scrapy.pipelines.DedupAndMySQLPipeline': 300,
-        },
-        'DOWNLOADER_MIDDLEWARES': {
-            'kyqb_scrapy.middlewares.RandomUserAgentMiddleware': 400,
-        }
-    }
-
-    def __init__(self, task='telecom_license', start_page=1, end_page=100, *args, **kwargs):
-        super(MiitSpider, self).__init__(*args, **kwargs)
-
-        if task not in TASKS:
-            raise ValueError(f"不支持的任务: {task}")
-
-        self.task_config = TASKS[task]
-        self.category_id = self.task_config["category_id"]
-        self.item_class = self.task_config["item_class"]
-        self.table_name = self.task_config["table"]
-        self.field_mapping = self.task_config["field_mapping"]
-
-        self.start_page = int(start_page)
-        self.end_page = int(end_page)
-        self.page_size = 5
-
-        # 设置 custom_settings 中的表名（动态）
-        self.custom_settings['MYSQL_TABLE'] = self.table_name
-
-        logger.info(f"🚀 启动任务: {self.task_config['name']}，页码 {self.start_page} ~ {self.end_page}")
-
-    def start_requests(self):
-        for page in range(self.start_page, self.end_page + 1):
-            data = {
-                "categoryId": self.category_id,
-                "currentPage": page,
-                "pageSize": self.page_size,
-                "searchContent": ""
-            }
-            yield Request(
-                url=API_URL,
-                method='POST',
-                body=json.dumps(data, separators=(',', ':')),
-                headers={'Content-Type': 'application/json;charset=UTF-8'},
-                callback=self.parse,
-                meta={'page': page},
-                dont_filter=True
-            )
-
-    def parse(self, response):
-        page = response.meta['page']
-
-        # 检查响应
-        if response.status_code != 200:
-            self.logger.error(f"❌ 第 {page} 页请求失败: HTTP {response.status}")
-            return
-
-        try:
-            result = json.loads(response.text)
-        except json.JSONDecodeError:
-            text = response.text
-            if "升级浏览器" in text or "请尝试" in text:
-                self.logger.error(f"⚠️ 检测到反爬: 请升级浏览器。响应片段: {text[:300]}")
-            else:
-                self.logger.error(f"JSON解析失败: {text[:300]}")
-            return
-
-        if not result.get("success"):
-            msg = result.get("msg", "未知错误")
-            if "升级浏览器" in msg or "请尝试" in msg:
-                self.logger.error(f"⚠️ 反爬提示: {msg}")
-            else:
-                self.logger.error(f"接口失败: {msg}")
-            return
-
-        raw_records = result["params"]["tbAppArticle"]["list"]
-        self.logger.info(f"✅ 第 {page} 页获取 {len(raw_records)} 条数据")
-
-        for record in raw_records:
-            item = self.item_class()
-
-            for src_key, dst_key in self.field_mapping.items():
-                value = record.get(src_key, '')
-                if isinstance(value, str):
-                    value = strip_html(value)
-                item[dst_key] = value
-
-            yield item