crawlo 1.0.5__py3-none-any.whl → 1.0.6__py3-none-any.whl

crawlo/utils/project.py

@@ -1,39 +1,171 @@
 #!/usr/bin/python
-# -*- coding:UTF-8 -*-
+# -*- coding: UTF-8 -*-
+"""
+自动发现项目并创建 SettingManager 实例
+
+该模块负责：
+1.  向上搜索项目根目录（通过 crawlo.cfg 或 settings.py）
+2.  将项目根目录加入 Python 路径 (sys.path)
+3.  加载指定的 settings 模块
+4.  返回一个已配置好的 SettingManager 实例
+"""
 import os
 import sys
+import configparser
 from importlib import import_module
 from inspect import iscoroutinefunction
-from typing import Callable
+from typing import Callable, Optional
 
+from crawlo.utils.log import get_logger
 from crawlo.settings.setting_manager import SettingManager
 
 
-def _get_closest(path='.'):
-    path = os.path.abspath(path)
-    return path
+logger =get_logger(__name__)
 
 
-def _init_env():
-    closest = _get_closest()
-    if closest:
-        sys.path.append(closest)
-        # project_dir = os.path.dirname(closest)
-        # sys.path.append(project_dir)
+def _find_project_root(start_path: str = '.') -> Optional[str]:
+    """
+    从指定的起始路径开始，向上级目录递归搜索，寻找项目根目录。
+    搜索依据：
+        1. 优先查找 'crawlo.cfg' 文件。
+        2. 如果未找到 cfg 文件，则查找位于 Python 包内（即包含 __init__.py 的目录）的 'settings.py' 文件。
 
+    Args:
+        start_path (str): 搜索的起始路径，默认为当前工作目录 '.'。
 
-def get_settings(settings='settings'):
-    _settings = SettingManager()
-    _init_env()
-    _settings.set_settings(settings)
-    return _settings
+    Returns:
+        Optional[str]: 找到的项目根目录的绝对路径，如果未找到则返回 None。
+    """
+    path = os.path.abspath(start_path)
+    logger.info(f"开始向上搜索项目根目录，起始路径: {path}")
 
+    while True:
+        # 1. 检查是否存在 crawlo.cfg 文件
+        cfg_file = os.path.join(path, 'crawlo.cfg')
+        if os.path.isfile(cfg_file):
+            logger.info(f"在路径 {path} 找到 'crawlo.cfg' 文件，确定为项目根目录。")
+            return path
 
-def merge_settings(spider, settings):
-    if hasattr(spider, 'custom_settings'):
-        custom_settings = getattr(spider, 'custom_settings')
+        # 2. 检查是否存在 settings.py 文件，并且它位于一个 Python 包中
+        settings_file = os.path.join(path, 'settings.py')
+        if os.path.isfile(settings_file):
+            init_file = os.path.join(path, '__init__.py')
+            if os.path.isfile(init_file):
+                logger.info(f"在路径 {path} 找到 'settings.py' 文件，确定为项目根目录。")
+                return path
+            else:
+                logger.debug(f"在路径 {path} 找到 'settings.py'，但缺少 '__init__.py'，忽略。")
+
+        # 移动到上一级目录
+        parent = os.path.dirname(path)
+        if parent == path:
+            # 已经到达文件系统根目录
+            break
+        path = parent
+
+    logger.warning("向上搜索完毕，未找到项目根目录。")
+    return None
+
+
+def _get_settings_module_from_cfg(cfg_path: str) -> str:
+    """
+    从 crawlo.cfg 配置文件中读取 settings 模块的路径。
+
+    Args:
+        cfg_path (str): crawlo.cfg 文件的完整路径。
+
+    Returns:
+        str: settings 模块的导入路径，例如 'myproject.settings'。
+
+    Raises:
+        RuntimeError: 当读取文件或解析配置出错时抛出。
+    """
+    logger.info(f"正在读取配置文件: {cfg_path}")
+    config = configparser.ConfigParser()
+    try:
+        config.read(cfg_path, encoding='utf-8')
+        if config.has_section('settings') and config.has_option('settings', 'default'):
+            module_path = config.get('settings', 'default')
+            logger.info(f"从 'crawlo.cfg' 中读取到 settings 模块路径: {module_path}")
+            return module_path
+        else:
+            error_msg = f"配置文件 '{cfg_path}' 缺少 '[settings]' 或 'default' 配置项。"
+            logger.error(error_msg)
+            raise RuntimeError(error_msg)
+    except (configparser.Error, OSError) as e:
+        error_msg = f"读取或解析配置文件 '{cfg_path}' 时出错: {e}"
+        logger.error(error_msg)
+        raise RuntimeError(error_msg)
+
+
+def get_settings(custom_settings=None):
+    """
+    获取配置管理器实例的主函数。
+    此函数会自动发现项目，加载配置，并返回一个配置好的 SettingManager。
+
+    Args:
+        custom_settings (dict, optional): 运行时传入的自定义设置字典，会覆盖 settings.py 中的同名配置。
+
+    Returns:
+        SettingManager: 一个已加载所有配置的 SettingManager 实例。
+
+    Raises:
+        RuntimeError: 当无法找到项目或配置文件时。
+        ImportError: 当无法导入指定的 settings 模块时。
+    """
+    logger.info("正在初始化配置管理器...")
+
+    # 1. 发现项目根目录
+    project_root = _find_project_root()
+    if not project_root:
+        error_msg = "未找到 Crawlo 项目。请确保您正在包含 'crawlo.cfg' 或 'settings.py' 的项目目录中运行。"
+        logger.error(error_msg)
+        raise RuntimeError(error_msg)
+
+    logger.info(f"项目根目录已确定: {project_root}")
+
+    # 2. 确定 settings 模块的导入路径
+    settings_module_path = None
+
+    # 优先从 crawlo.cfg 中读取
+    cfg_file = os.path.join(project_root, 'crawlo.cfg')
+    if os.path.isfile(cfg_file):
+        settings_module_path = _get_settings_module_from_cfg(cfg_file)
+    else:
+        logger.info("未找到 'crawlo.cfg'，尝试推断 settings 模块路径...")
+        # 推断：项目目录名.settings
+        project_name = os.path.basename(project_root)
+        settings_module_path = f"{project_name}.settings"
+        logger.info(f"推断 settings 模块路径为: {settings_module_path}")
+
+    # 3. 将项目根目录添加到 Python 路径，确保可以成功导入
+    if project_root not in sys.path:
+        sys.path.insert(0, project_root)
+        logger.info(f"已将项目根目录 '{project_root}' 添加到 Python 路径。")
+    else:
+        logger.debug(f"项目根目录 '{project_root}' 已在 Python 路径中。")
+
+    # 4. 创建 SettingManager 并加载配置
+    logger.info(f"正在加载 settings 模块: {settings_module_path}")
+    settings = SettingManager()
+
+    try:
+        # 这会触发 SettingManager.set_settings()，从模块中加载所有大写常量
+        settings.set_settings(settings_module_path)
+        logger.info("settings 模块加载成功。")
+    except Exception as e:
+        error_msg = f"加载 settings 模块 '{settings_module_path}' 失败: {e}"
+        logger.error(error_msg)
+        raise ImportError(error_msg)
+
+    # 5. 应用运行时自定义设置
+    if custom_settings:
+        logger.info(f"正在应用运行时自定义设置: {custom_settings}")
         settings.update_attributes(custom_settings)
+        logger.info("运行时自定义设置已应用。")
 
+    logger.info("配置管理器初始化完成。")
+    return settings
 
 def load_class(_path):
     if not isinstance(_path, str):
@@ -51,6 +183,14 @@ def load_class(_path):
         raise NameError(f"Module {module_name!r} has no class named {class_name!r}")
     return cls
 
+def merge_settings(spider, settings):
+    spider_name = getattr(spider, 'name', 'UnknownSpider')
+    if hasattr(spider, 'custom_settings'):
+        custom_settings = getattr(spider, 'custom_settings')
+        settings.update_attributes(custom_settings)
+    else:
+        logger.debug(f"爬虫 '{spider_name}' 无 custom_settings，跳过合并") # 添加日志
+
 
 async def common_call(func: Callable, *args, **kwargs):
     if iscoroutinefunction(func):

crawlo/utils/spider_loader.py

@@ -0,0 +1,63 @@
+import importlib
+import inspect
+from pathlib import Path
+from typing import List, Type, Optional, Dict
+
+from crawlo.spider import Spider
+from crawlo.utils.log import get_logger
+
+logger = get_logger(__name__)
+
+
+class SpiderLoader:
+    """爬虫加载器，负责发现和加载爬虫"""
+
+    def __init__(self, project_package: str):
+        self.project_package = project_package
+        self._spiders: Dict[str, Type[Spider]] = {}
+        self._load_spiders()
+
+    def _load_spiders(self):
+        """加载所有爬虫"""
+        spiders_dir = Path.cwd() / self.project_package / 'spiders'
+        if not spiders_dir.exists():
+            logger.warning(f"Spiders directory not found: {spiders_dir}")
+            return
+
+        for py_file in spiders_dir.glob("*.py"):
+            if py_file.name.startswith('_'):
+                continue
+
+            module_name = py_file.stem
+            spider_module_path = f"{self.project_package}.spiders.{module_name}"
+
+            try:
+                module = importlib.import_module(spider_module_path)
+            except ImportError as e:
+                logger.debug(f"Skip module {module_name}: {e}")
+                continue
+
+            # 查找所有 Spider 子类
+            for attr_name in dir(module):
+                attr_value = getattr(module, attr_name)
+                if (isinstance(attr_value, type) and
+                        issubclass(attr_value, Spider) and
+                        attr_value != Spider and
+                        hasattr(attr_value, 'name')):
+
+                    spider_name = getattr(attr_value, 'name')
+                    if spider_name in self._spiders:
+                        logger.warning(f"Duplicate spider name '{spider_name}' found")
+                    self._spiders[spider_name] = attr_value
+
+    def load(self, spider_name: str) -> Optional[Type[Spider]]:
+        """通过 name 加载爬虫"""
+        return self._spiders.get(spider_name)
+
+    def list(self) -> List[str]:
+        """列出所有可用的爬虫名称"""
+        return list(self._spiders.keys())
+
+    def get_all(self) -> Dict[str, Type[Spider]]:
+        """获取所有爬虫"""
+        return self._spiders.copy()

{crawlo-1.0.5.dist-info → crawlo-1.0.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: crawlo
-Version: 1.0.5
+Version: 1.0.6
 Summary: Crawlo 是一款基于异步IO的高性能Python爬虫框架，支持分布式抓取。
 Home-page: https://github.com/crawl-coder/Crawlo.git
 Author: crawl-coder

{crawlo-1.0.5.dist-info → crawlo-1.0.6.dist-info}/RECORD

@@ -1,11 +1,16 @@
 crawlo/__init__.py,sha256=xpiIAZbSG3CzneJuDLPCbwfRcvw2wyHYl2kJjaNfNGY,584
-crawlo/__version__.py,sha256=lfZikIZ2prlMV6RkxhMRZj5dAeD0TCswIWS46kSjXw0,23
-crawlo/crawler.py,sha256=2izxy7-0yD8n_FsLSL_NaoFYdqQWIhm0hsoSLKgnPcA,16919
+crawlo/__version__.py,sha256=1HqFYnow__4MUVRI_OMjvzTBzKkReNozOdA96kH53cA,23
+crawlo/cli.py,sha256=hjAJKx9pba375sATvvcy-dtZyBIgXj8fRBq9RFIZHA4,1206
+crawlo/crawler.py,sha256=nfuA_f8QnuIp2F4ZbaJv8Fceo_QPwqV1jYdD_edkMjg,8527
 crawlo/event.py,sha256=ZhoPW5CglCEuZNFEwviSCBIw0pT5O6jT98bqYrDFd3E,324
-crawlo/exceptions.py,sha256=VUFSOS00BPWMcH8EW5MgMDhXlUlaFeEcsgqbS_e8MoU,1119
+crawlo/exceptions.py,sha256=xdyZkvVcLEJ-19sWMHvn9IJsu30-hAY2jJhA2kYIims,1207
 crawlo/stats_collector.py,sha256=v4jC9BAe-23w93hWzbeMCCgQ9VuFPyxw5JV9ItbGH8w,1636
 crawlo/subscriber.py,sha256=udlHeTR0ymGQhCDxVUGwUzeeeR4TYCEJrJwFnkgr0cU,3836
 crawlo/task_manager.py,sha256=D9m-nqnGj-FZPtGk4CdwZX3Gw7IWyYvTS7CHpRGWc_w,748
+crawlo/commands/__init__.py,sha256=dRu3ipuhDM7M1eTb6zJtQZ_u7N_tZumGfH5_I92xno8,252
+crawlo/commands/genspider.py,sha256=kSHYsAGHRoxU6Qf_MGpR_VS-Ua5NUGY2KGm_Wapn0sw,3529
+crawlo/commands/run.py,sha256=Upv8K4sM0c0I1fIwTFK18VDcSHF7xabqfXtQ82fk56g,4628
+crawlo/commands/startproject.py,sha256=1KOq_CALy01oklr0dAUYhGFzu4f7w45q2H0O3qafLX4,3494
 crawlo/core/__init__.py,sha256=JYSAn15r8yWgRK_Nc69t_8tZCyb70MiPZKssA8wrYz0,43
 crawlo/core/engine.py,sha256=JFHooPp-5cfHSyxEh87nOOR5NMaPLVDfNSqAsbtx4PM,6030
 crawlo/core/processor.py,sha256=oHLs-cno0bJGTNc9NGD2S7_2-grI3ruvggO0SY2mf3Q,1180
@@ -21,8 +26,10 @@ crawlo/extension/logging_extension.py,sha256=rty2_up53KV05nCazuBuz2ZapHKq0ti7mGV
 crawlo/filters/__init__.py,sha256=9fJQRVkxWWPChajYbAGe1O6UYB639xWt0hiLUGBs4hQ,1014
 crawlo/filters/aioredis_filter.py,sha256=phBFW9Z28oylbik9Kb2WHM65Wo5yRAH2w9Yz0_2HaOQ,5621
 crawlo/filters/memory_filter.py,sha256=L8XEJkObOxs4BzYpQvk9PVM969k2LE61VFsnEOTEf_E,6841
-crawlo/items/__init__.py,sha256=o5BSpS1Byivr-bpdfFgc9GCoGi8ThNuPJiTW7lz85-I,2125
-crawlo/items/items.py,sha256=0jf-CdZFkgDAevYn8PmSgGhf6iYu3bx1sv87hJbFtF4,3891
+crawlo/items/__init__.py,sha256=HLDShSwAQUrgwt9_Ec2SIwzpIDZnNOCg9nSYqqEQdp8,407
+crawlo/items/base.py,sha256=DZG0qENdukJExRtKjqdNkSlzUoWR3ucjyF73LYLANFo,754
+crawlo/items/fields.py,sha256=fpS0vlRPpZYjTaMDgI9Q8z_YQqruwf6fi4Dgm6R2oEk,1854
+crawlo/items/items.py,sha256=OmVEvMmgofMU95GkaiWkfNQ2fjsH2fY9sw3SKcmUhLs,3478
 crawlo/middleware/__init__.py,sha256=PSwpRLdBUopaQzBp1S0zK_TZbrRagQ4yzvgyLy4tBk8,570
 crawlo/middleware/default_header.py,sha256=OVW4vpRPp3Y6qYXtiEYlGqVjCYcbuv1Iecc7zEgwCsI,1099
 crawlo/middleware/download_delay.py,sha256=P2eyAJXwdLdC4yYuLhvKZVa1b5YQvQD0GpsR8aDW8-8,994
@@ -42,15 +49,17 @@ crawlo/pipelines/mysql_batch_pipline.py,sha256=Mj3PReDRw22JhJ5hZxnka4cirKq3kEbOC
 crawlo/pipelines/mysql_pipeline.py,sha256=bsAFqpxrCijzvX-IusxOtvTvQEUCt5uHNTyYMo_pIq4,8056
 crawlo/pipelines/pipeline_manager.py,sha256=k-Rg0os0Havrov99D-Jn3ROpnz154K30tf7aARE5W3k,2174
 crawlo/settings/__init__.py,sha256=NgYFLfk_Bw7h6KSoepJn_lMBSqVbCHebjKxaE3_eMgw,130
-crawlo/settings/default_settings.py,sha256=CH95c2oBmxy6t6bGLUuqSL8vJ3Z5Psicdfpc9W0MG90,7309
+crawlo/settings/default_settings.py,sha256=urj4XJ--ZpVRbbo3fWUT71bYQLmElx43AC9KeHtqHBs,7310
 crawlo/settings/setting_manager.py,sha256=4xXOzKwZCgAp8ybwvVcs2R--CsOD7c6dBIkj6DJHB3c,2998
 crawlo/spider/__init__.py,sha256=lWi0bCR7HLT5bnj7_e9UIgFJjuqoeWtbwADfNkaajug,1139
-crawlo/templates/item_template.tmpl,sha256=0bGFnlwJRqstxMNEj1H_pEICybwoueRhs31QaDPXrS0,372
-crawlo/templates/spider_template.tmpl,sha256=JzphuA87Yl_F1xR9zOIi_ZSazyT8eSNPxYYPMv3Uiko,835
-crawlo/templates/project_template/main.py,sha256=BcCP294ycCPsHi_AMN7OAJtcrLvQdf91meH93PqbQgs,626
-crawlo/templates/project_template/setting.py,sha256=Ce4nMbrdhL1ioRdTcB0vV_vK_50cfnwVqSvt49QsNkA,9395
-crawlo/templates/project_template/items/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-crawlo/templates/project_template/spiders/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+crawlo/templates/crawlo.cfg.tmpl,sha256=lwiUVe5sFixJgHFEjn1OtbAeyWsECOrz37uheuVtulk,240
+crawlo/templates/project/__init__.py.tmpl,sha256=aQnHaOjMSkTviOC8COUX0fKymuyf8lx2tGduxkMkXEE,61
+crawlo/templates/project/items.py.tmpl,sha256=bXx-oCldMr2EgBKUAH9LH5gMnbyLiWX-EySAaMzcu2g,318
+crawlo/templates/project/middlewares.py.tmpl,sha256=VAolmMTC6HBmJT5XvWB0ag6ig9iaDBS32adIQ1zPdw0,2177
+crawlo/templates/project/pipelines.py.tmpl,sha256=xK1Yl7wYxiyUCm07GZvMnCS_cxJ5LF7z1YBBdkLlWys,1880
+crawlo/templates/project/settings.py.tmpl,sha256=985Z-jiU6A31f5s1IVU4PvkC_QGlFlRRfTF6rZ_G4ek,1771
+crawlo/templates/project/spiders/__init__.py.tmpl,sha256=zMbePipgLsctQUEnda4WkHz8rDLUX--rc8ruI6zkpWc,111
+crawlo/templates/spider/spider.py.tmpl,sha256=SkNv1kOwet7ZdxoNXpj-o1iRETB30bcwPP16Uy8lyXg,869
 crawlo/utils/__init__.py,sha256=XCYumI8wJ1jU_Myn_K0LT-LVygPDUCdETCbXM3EWvlo,130
 crawlo/utils/concurrency_manager.py,sha256=o-_cfeUHdlBOM3eAXF857MtekSrRcVTBJ2jWZvY6weQ,5230
 crawlo/utils/date_tools.py,sha256=lcEFP2Z5b-6pUTHczrzCCuqiHP_4_2zamomMGPZrExo,7194
@@ -58,27 +67,28 @@ crawlo/utils/db_helper.py,sha256=ZqOt1d3mErVv4TOvoWlov0niUxORB9aHByTmMoNFIDw,109
 crawlo/utils/func_tools.py,sha256=y-TYP9H3X67MS_foWy9Z2LIS6GP7Y4Cy3T168ulq3Jc,2451
 crawlo/utils/log.py,sha256=YD2FfXuuE2MC9ZdQQZ0H7KysE7l_LHZqQepaTPlcApo,4133
 crawlo/utils/pqueue.py,sha256=HDgX4HAkc7RqYUtX6q51tzI1ZRTACf8P_4jLqC4-uC0,5559
-crawlo/utils/project.py,sha256=FfBaMfxcau4yL59O-DfD7FAii8k6gXWQmQ1YU6aaUCE,1544
+crawlo/utils/project.py,sha256=qAiCmpIxiB7RxCLG-U5lGV6k4UCa21uRdykTfnAF834,7669
 crawlo/utils/request.py,sha256=ejdKpTwc-HE04HQybafhOVywzz57IV3pY0YMkSLyGUo,9065
+crawlo/utils/spider_loader.py,sha256=V0CBTicJBYBZafhwLfDEfuEc_hJ2mSoiptT6qKufI9U,2249
 crawlo/utils/system.py,sha256=24zGmtHNhDFMGVo7ftMV-Pqg6_5d63zsyNey9udvJJk,248
 crawlo/utils/tools.py,sha256=uy7qw5Z1BIhyEgiHENvtM7WoGCJxlS8EX3PmOA7ouCo,275
 crawlo/utils/url.py,sha256=RKe_iqdjafsNcp-P2GVLYpsL1qbxiuZLiFc-SqOQkcs,1521
 examples/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 examples/gxb/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-examples/gxb/items.py,sha256=s4uKo3kKlN2DC4Y4muwp_qzG6kdyhzOVLEjXv7Qvi7c,995
-examples/gxb/run.py,sha256=YLtlUB6GEAHLuLaTOt8HSOyAF1ZBdjSAwR9rJ2prUSs,340
-examples/gxb/settings.py,sha256=JqwnEkZ0wZZ1f43I2Ne9yu1LnEBBiH2rVG2iDKZC1Q8,2321
+examples/gxb/items.py,sha256=3-1Lxpi7EqMzheDJoO0MPyHky5nHG_nqQGgKlm8y6mQ,989
+examples/gxb/run.py,sha256=9kJlR8f-tZ3BqP5PW7sCLTw6PAFWo3x4cG5lc-6GWqI,333
+examples/gxb/settings.py,sha256=_nbXj9HV2e0F6liUzK0ueygLcaMM_IUlkuwL6mJqUfc,2345
 examples/gxb/spider/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 examples/gxb/spider/miit_spider.py,sha256=tcQnuyUHfu-Re1QbKKSI9DXW3Sp1vyBW8qBzKLf_RC4,6666
-examples/gxb/spider/telecom_device_licenses.py,sha256=t-XFai7e4itfGR4zeTJVJ1ulhfj-92gIgISqqdOwdag,4938
+examples/gxb/spider/telecom_device.py,sha256=58iG6BQtQjjDHOF7-DXH0u5_XnppP5AJTQwaVJVyBEo,4929
 tests/__init__.py,sha256=409aRX8hsPffiZCVjOogtxwhACzBp8G2UTJyUQSxhK0,136
 tests/test_proxy_health_check.py,sha256=_tDlxa_6TdL3M5RLkHF82roXJ8WIuG5hELBp2GADyKQ,1123
 tests/test_proxy_middleware_integration.py,sha256=mTPK_XvbmLCV_QoVZzA3ybWOOX61493Ew78WfTp-bYQ,4441
 tests/test_proxy_providers.py,sha256=u_R2fhab90vqvQEaOAztpAOe9tJXvUMIdoDxmStmXJ4,1749
 tests/test_proxy_stats.py,sha256=ES00CEoDITYPFBGPk8pecFzD3ItYIv6NSpcqNd8-kvo,526
 tests/test_proxy_strategies.py,sha256=9Z1pXmTNyw-eIhGXlf2abZbJx6igLohYq-_3hldQ5uE,1868
-crawlo-1.0.5.dist-info/METADATA,sha256=IC9lzZIPUOEZdBXsSZkkd0CpkFuYChtuNtSasgO-O6M,1825
-crawlo-1.0.5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-crawlo-1.0.5.dist-info/entry_points.txt,sha256=GD9PBhKQN83EaxPYtz7NhcGeZeh3bdr2jWbTixOs-lw,59
-crawlo-1.0.5.dist-info/top_level.txt,sha256=keG_67pbZ_wZL2dmDRA9RMaNHTaV_x_oxZ9DKNgwvR0,22
-crawlo-1.0.5.dist-info/RECORD,,
+crawlo-1.0.6.dist-info/METADATA,sha256=_TDAivxDg2R8omq5gG1kUiODY2tZ3UEp5aH0SwshOjI,1825
+crawlo-1.0.6.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+crawlo-1.0.6.dist-info/entry_points.txt,sha256=5HoVoTSPxI8SCa5B7pQYxLSrkOdiunyO9tqNsLMv52g,43
+crawlo-1.0.6.dist-info/top_level.txt,sha256=keG_67pbZ_wZL2dmDRA9RMaNHTaV_x_oxZ9DKNgwvR0,22
+crawlo-1.0.6.dist-info/RECORD,,

crawlo-1.0.6.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+crawlo = crawlo.cli:main

examples/gxb/items.py

@@ -1,4 +1,4 @@
-from crawlo.items.items import Item, Field
+from crawlo.items import Item, Field
 
 class RadioApprovalItem(Item):
     approval_number = Field()

examples/gxb/run.py

@@ -1,7 +1,7 @@
 import asyncio
 
 from crawlo.crawler import CrawlerProcess
-from examples.gxb.spider.telecom_device_licenses import TelecomDeviceLicensesSpider
+from examples.gxb.spider.telecom_device import TelecomDeviceLicensesSpider
 
 async def main():
     process = CrawlerProcess()
@@ -10,6 +10,7 @@ async def main():
     )
 
 
+
 if __name__ == '__main__':
     asyncio.run(main())
     # 132023

examples/gxb/settings.py

@@ -1,6 +1,7 @@
 import platform
 
-PROXY_ENABLED = False
+PROXY_ENABLED = True
+PROJECT_PACKAGE = 'gxb'
 
 # API 地址
 PROXY_API_URL = 'http://123.56.42.142:5000/proxy/getitem/'

examples/gxb/spider/{telecom_device_licenses.py → telecom_device.py}

@@ -10,7 +10,7 @@ from examples.gxb.settings import HEADERS, COOKIES
 logger = get_logger(__name__)
 
 class TelecomDeviceLicensesSpider(Spider):
-    name = 'telecom_device_licenses'
+    name = 'telecom_device'
     allowed_domains = ['ythzxfw.miit.gov.cn']
     # API 的基础 URL
     base_api_url = 'https://ythzxfw.miit.gov.cn/oldyth/user-center/tbAppSearch/selectResult'

crawlo/templates/item_template.tmpl

@@ -1,22 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
-Created on {DATE}
----------
-@summary:
----------
-@author: {USER}
-"""
-
-from crawlo import Item
-
-
-class ${item_name}Item(Item):
-    """
-    This class was generated by feapder
-    command: feapder create -i ${command}
-    """
-
-    __table_name__ = "${table_name}"
-
-    def __init__(self, *args, **kwargs):
-        ${propertys}

crawlo/templates/project_template/main.py

@@ -1,33 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
-Created on {DATE}
----------
-@summary: 爬虫入口
----------
-@author: {USER}
-"""
-
-from crawlo import ArgumentParser
-
-from spiders import *
-
-
-
-def crawl_xxx():
-    """
-    Spider爬虫
-    """
-    spider = xxx.XXXSpider(redis_key="xxx:xxx")
-    spider.start()
-
-
-
-if __name__ == "__main__":
-    parser = ArgumentParser(description="xxx爬虫")
-
-    parser.add_argument(
-        "--crawl_xxx", action="store_true", help="xxx爬虫", function=crawl_xxx
-    )
-    parser.start()
-
-    # main.py作为爬虫启动的统一入口，提供命令行的方式启动多个爬虫