crawlo 1.1.0__py3-none-any.whl → 1.1.1__py3-none-any.whl

crawlo/pipelines/mysql_pipeline.py

@@ -1,195 +1,195 @@
-# -*- coding: utf-8 -*-
-import asyncio
-import aiomysql
-from typing import Optional
-from asyncmy import create_pool
-from crawlo.utils.log import get_logger
-from crawlo.exceptions import ItemDiscard
-from crawlo.utils.db_helper import make_insert_sql, logger
-
-
-class AsyncmyMySQLPipeline:
-    def __init__(self, crawler):
-        self.crawler = crawler
-        self.settings = crawler.settings
-        self.logger = get_logger(self.__class__.__name__, self.settings.get('LOG_LEVEL'))
-
-        # 使用异步锁和初始化标志确保线程安全
-        self._pool_lock = asyncio.Lock()
-        self._pool_initialized = False
-        self.pool = None
-        self.table_name = (
-                self.settings.get('MYSQL_TABLE') or
-                getattr(crawler.spider, 'mysql_table', None) or
-                f"{crawler.spider.name}_items"
-        )
-
-        # 注册关闭事件
-        crawler.subscriber.subscribe(self.spider_closed, event='spider_closed')
-
-    @classmethod
-    def from_crawler(cls, crawler):
-        return cls(crawler)
-
-    async def _ensure_pool(self):
-        """确保连接池已初始化（线程安全）"""
-        if self._pool_initialized:
-            return
-
-        async with self._pool_lock:
-            if not self._pool_initialized:  # 双重检查避免竞争条件
-                try:
-                    self.pool = await create_pool(
-                        host=self.settings.get('MYSQL_HOST', 'localhost'),
-                        port=self.settings.get_int('MYSQL_PORT', 3306),
-                        user=self.settings.get('MYSQL_USER', 'root'),
-                        password=self.settings.get('MYSQL_PASSWORD', ''),
-                        db=self.settings.get('MYSQL_DB', 'scrapy_db'),
-                        minsize=self.settings.get_int('MYSQL_POOL_MIN', 3),
-                        maxsize=self.settings.get_int('MYSQL_POOL_MAX', 10),
-                        echo=self.settings.get_bool('MYSQL_ECHO', False)
-                    )
-                    self._pool_initialized = True
-                    self.logger.debug(f"MySQL连接池初始化完成（表: {self.table_name}）")
-                except Exception as e:
-                    self.logger.error(f"MySQL连接池初始化失败: {e}")
-                    raise
-
-    async def process_item(self, item, spider, kwargs=None) -> Optional[dict]:
-        """处理item的核心方法"""
-        kwargs = kwargs or {}
-        spider_name = getattr(spider, 'name', 'unknown')  # 获取爬虫名称
-        try:
-            await self._ensure_pool()
-            item_dict = dict(item)
-            sql = make_insert_sql(table=self.table_name, data=item_dict, **kwargs)
-
-            rowcount = await self._execute_sql(sql=sql)
-            if rowcount > 1:
-                self.logger.info(
-                    f"爬虫 {spider_name} 成功插入 {rowcount} 条记录到表 {self.table_name}"
-                )
-            elif rowcount == 1:
-                self.logger.debug(
-                    f"爬虫 {spider_name} 成功插入单条记录到表 {self.table_name}"
-                )
-            else:
-                self.logger.warning(
-                    f"爬虫 {spider_name}: SQL执行成功但未插入新记录 - {sql[:100]}..."
-                )
-
-            return item
-
-        except Exception as e:
-            self.logger.error(f"处理item时发生错误: {e}")
-            raise ItemDiscard(f"处理失败: {e}")
-
-    async def _execute_sql(self, sql: str, values: list = None) -> int:
-        """执行SQL语句并处理结果"""
-        async with self.pool.acquire() as conn:
-            async with conn.cursor() as cursor:
-                try:
-                    # 根据是否有参数值选择不同的执行方法
-                    if values is not None:
-                        rowcount = await cursor.execute(sql, values)
-                    else:
-                        rowcount = await cursor.execute(sql)
-
-                    await conn.commit()
-                    self.crawler.stats.inc_value('mysql/insert_success')
-                    return rowcount
-                except Exception as e:
-                    await conn.rollback()
-                    self.crawler.stats.inc_value('mysql/insert_failed')
-                    raise ItemDiscard(f"MySQL插入失败: {e}")
-
-    async def spider_closed(self):
-        """关闭爬虫时清理资源"""
-        if self.pool:
-            self.pool.close()
-            await self.pool.wait_closed()
-            self.logger.info("MySQL连接池已关闭")
-
-
-class AiomysqlMySQLPipeline:
-    def __init__(self, crawler):
-        self.crawler = crawler
-        self.settings = crawler.settings
-        self.logger = get_logger(self.__class__.__name__, self.settings.get('LOG_LEVEL'))
-
-        # 使用异步锁和初始化标志
-        self._pool_lock = asyncio.Lock()
-        self._pool_initialized = False
-        self.pool = None
-        self.table_name = (
-                self.settings.get('MYSQL_TABLE') or
-                getattr(crawler.spider, 'mysql_table', None) or
-                f"{crawler.spider.name}_items"
-        )
-
-        crawler.subscriber.subscribe(self.spider_closed, event='spider_closed')
-
-    @classmethod
-    def create_instance(cls, crawler):
-        return cls(crawler)
-
-    async def _init_pool(self):
-        """延迟初始化连接池（线程安全）"""
-        if self._pool_initialized:
-            return
-
-        async with self._pool_lock:
-            if not self._pool_initialized:
-                try:
-                    self.pool = await aiomysql.create_pool(
-                        host=self.settings.get('MYSQL_HOST', 'localhost'),
-                        port=self.settings.getint('MYSQL_PORT', 3306),
-                        user=self.settings.get('MYSQL_USER', 'root'),
-                        password=self.settings.get('MYSQL_PASSWORD', ''),
-                        db=self.settings.get('MYSQL_DB', 'scrapy_db'),
-                        minsize=self.settings.getint('MYSQL_POOL_MIN', 2),
-                        maxsize=self.settings.getint('MYSQL_POOL_MAX', 5),
-                        cursorclass=aiomysql.DictCursor,
-                        autocommit=False
-                    )
-                    self._pool_initialized = True
-                    self.logger.debug(f"aiomysql连接池已初始化（表: {self.table_name}）")
-                except Exception as e:
-                    self.logger.error(f"aiomysql连接池初始化失败: {e}")
-                    raise
-
-    async def process_item(self, item, spider) -> Optional[dict]:
-        """处理item方法"""
-        try:
-            await self._init_pool()
-
-            item_dict = dict(item)
-            sql = f"""
-            INSERT INTO `{self.table_name}` 
-            ({', '.join([f'`{k}`' for k in item_dict.keys()])})
-            VALUES ({', '.join(['%s'] * len(item_dict))})
-            """
-
-            async with self.pool.acquire() as conn:
-                async with conn.cursor() as cursor:
-                    try:
-                        await cursor.execute(sql, list(item_dict.values()))
-                        await conn.commit()
-                        self.crawler.stats.inc_value('mysql/insert_success')
-                    except aiomysql.Error as e:
-                        await conn.rollback()
-                        self.crawler.stats.inc_value('mysql/insert_failed')
-                        raise ItemDiscard(f"MySQL错误: {e.args[1]}")
-
-            return item
-
-        except Exception as e:
-            self.logger.error(f"Pipeline处理异常: {e}")
-            raise ItemDiscard(f"处理失败: {e}")
-
-    async def spider_closed(self):
-        """资源清理"""
-        if self.pool:
-            self.pool.close()
-            await self.pool.wait_closed()
-            self.logger.info("aiomysql连接池已释放")
+# -*- coding: utf-8 -*-
+import asyncio
+import aiomysql
+from typing import Optional
+from asyncmy import create_pool
+from crawlo.utils.log import get_logger
+from crawlo.exceptions import ItemDiscard
+from crawlo.utils.db_helper import make_insert_sql, logger
+
+
+class AsyncmyMySQLPipeline:
+    def __init__(self, crawler):
+        self.crawler = crawler
+        self.settings = crawler.settings
+        self.logger = get_logger(self.__class__.__name__, self.settings.get('LOG_LEVEL'))
+
+        # 使用异步锁和初始化标志确保线程安全
+        self._pool_lock = asyncio.Lock()
+        self._pool_initialized = False
+        self.pool = None
+        self.table_name = (
+                self.settings.get('MYSQL_TABLE') or
+                getattr(crawler.spider, 'mysql_table', None) or
+                f"{crawler.spider.name}_items"
+        )
+
+        # 注册关闭事件
+        crawler.subscriber.subscribe(self.spider_closed, event='spider_closed')
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        return cls(crawler)
+
+    async def _ensure_pool(self):
+        """确保连接池已初始化（线程安全）"""
+        if self._pool_initialized:
+            return
+
+        async with self._pool_lock:
+            if not self._pool_initialized:  # 双重检查避免竞争条件
+                try:
+                    self.pool = await create_pool(
+                        host=self.settings.get('MYSQL_HOST', 'localhost'),
+                        port=self.settings.get_int('MYSQL_PORT', 3306),
+                        user=self.settings.get('MYSQL_USER', 'root'),
+                        password=self.settings.get('MYSQL_PASSWORD', ''),
+                        db=self.settings.get('MYSQL_DB', 'scrapy_db'),
+                        minsize=self.settings.get_int('MYSQL_POOL_MIN', 3),
+                        maxsize=self.settings.get_int('MYSQL_POOL_MAX', 10),
+                        echo=self.settings.get_bool('MYSQL_ECHO', False)
+                    )
+                    self._pool_initialized = True
+                    self.logger.debug(f"MySQL连接池初始化完成（表: {self.table_name}）")
+                except Exception as e:
+                    self.logger.error(f"MySQL连接池初始化失败: {e}")
+                    raise
+
+    async def process_item(self, item, spider, kwargs=None) -> Optional[dict]:
+        """处理item的核心方法"""
+        kwargs = kwargs or {}
+        spider_name = getattr(spider, 'name', 'unknown')  # 获取爬虫名称
+        try:
+            await self._ensure_pool()
+            item_dict = dict(item)
+            sql = make_insert_sql(table=self.table_name, data=item_dict, **kwargs)
+
+            rowcount = await self._execute_sql(sql=sql)
+            if rowcount > 1:
+                self.logger.info(
+                    f"爬虫 {spider_name} 成功插入 {rowcount} 条记录到表 {self.table_name}"
+                )
+            elif rowcount == 1:
+                self.logger.debug(
+                    f"爬虫 {spider_name} 成功插入单条记录到表 {self.table_name}"
+                )
+            else:
+                self.logger.warning(
+                    f"爬虫 {spider_name}: SQL执行成功但未插入新记录 - {sql[:100]}..."
+                )
+
+            return item
+
+        except Exception as e:
+            self.logger.error(f"处理item时发生错误: {e}")
+            raise ItemDiscard(f"处理失败: {e}")
+
+    async def _execute_sql(self, sql: str, values: list = None) -> int:
+        """执行SQL语句并处理结果"""
+        async with self.pool.acquire() as conn:
+            async with conn.cursor() as cursor:
+                try:
+                    # 根据是否有参数值选择不同的执行方法
+                    if values is not None:
+                        rowcount = await cursor.execute(sql, values)
+                    else:
+                        rowcount = await cursor.execute(sql)
+
+                    await conn.commit()
+                    self.crawler.stats.inc_value('mysql/insert_success')
+                    return rowcount
+                except Exception as e:
+                    await conn.rollback()
+                    self.crawler.stats.inc_value('mysql/insert_failed')
+                    raise ItemDiscard(f"MySQL插入失败: {e}")
+
+    async def spider_closed(self):
+        """关闭爬虫时清理资源"""
+        if self.pool:
+            self.pool.close()
+            await self.pool.wait_closed()
+            self.logger.info("MySQL连接池已关闭")
+
+
+class AiomysqlMySQLPipeline:
+    def __init__(self, crawler):
+        self.crawler = crawler
+        self.settings = crawler.settings
+        self.logger = get_logger(self.__class__.__name__, self.settings.get('LOG_LEVEL'))
+
+        # 使用异步锁和初始化标志
+        self._pool_lock = asyncio.Lock()
+        self._pool_initialized = False
+        self.pool = None
+        self.table_name = (
+                self.settings.get('MYSQL_TABLE') or
+                getattr(crawler.spider, 'mysql_table', None) or
+                f"{crawler.spider.name}_items"
+        )
+
+        crawler.subscriber.subscribe(self.spider_closed, event='spider_closed')
+
+    @classmethod
+    def create_instance(cls, crawler):
+        return cls(crawler)
+
+    async def _init_pool(self):
+        """延迟初始化连接池（线程安全）"""
+        if self._pool_initialized:
+            return
+
+        async with self._pool_lock:
+            if not self._pool_initialized:
+                try:
+                    self.pool = await aiomysql.create_pool(
+                        host=self.settings.get('MYSQL_HOST', 'localhost'),
+                        port=self.settings.getint('MYSQL_PORT', 3306),
+                        user=self.settings.get('MYSQL_USER', 'root'),
+                        password=self.settings.get('MYSQL_PASSWORD', ''),
+                        db=self.settings.get('MYSQL_DB', 'scrapy_db'),
+                        minsize=self.settings.getint('MYSQL_POOL_MIN', 2),
+                        maxsize=self.settings.getint('MYSQL_POOL_MAX', 5),
+                        cursorclass=aiomysql.DictCursor,
+                        autocommit=False
+                    )
+                    self._pool_initialized = True
+                    self.logger.debug(f"aiomysql连接池已初始化（表: {self.table_name}）")
+                except Exception as e:
+                    self.logger.error(f"aiomysql连接池初始化失败: {e}")
+                    raise
+
+    async def process_item(self, item, spider) -> Optional[dict]:
+        """处理item方法"""
+        try:
+            await self._init_pool()
+
+            item_dict = dict(item)
+            sql = f"""
+            INSERT INTO `{self.table_name}` 
+            ({', '.join([f'`{k}`' for k in item_dict.keys()])})
+            VALUES ({', '.join(['%s'] * len(item_dict))})
+            """
+
+            async with self.pool.acquire() as conn:
+                async with conn.cursor() as cursor:
+                    try:
+                        await cursor.execute(sql, list(item_dict.values()))
+                        await conn.commit()
+                        self.crawler.stats.inc_value('mysql/insert_success')
+                    except aiomysql.Error as e:
+                        await conn.rollback()
+                        self.crawler.stats.inc_value('mysql/insert_failed')
+                        raise ItemDiscard(f"MySQL错误: {e.args[1]}")
+
+            return item
+
+        except Exception as e:
+            self.logger.error(f"Pipeline处理异常: {e}")
+            raise ItemDiscard(f"处理失败: {e}")
+
+    async def spider_closed(self):
+        """资源清理"""
+        if self.pool:
+            self.pool.close()
+            await self.pool.wait_closed()
+            self.logger.info("aiomysql连接池已释放")

crawlo/pipelines/pipeline_manager.py

@@ -1,56 +1,56 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-from typing import List
-from pprint import pformat
-from asyncio import create_task
-
-
-from crawlo.utils.log import get_logger
-from crawlo.event import item_successful, item_discard
-from crawlo.utils.project import load_class, common_call
-from crawlo.exceptions import PipelineInitError, ItemDiscard, InvalidOutputError
-
-
-class PipelineManager:
-
-    def __init__(self, crawler):
-        self.crawler = crawler
-        self.pipelines: List = []
-        self.methods: List = []
-
-        self.logger = get_logger(self.__class__.__name__, self.crawler.settings.get('LOG_LEVEL'))
-        pipelines = self.crawler.settings.get_list('PIPELINES')
-        self._add_pipelines(pipelines)
-        self._add_methods()
-
-    @classmethod
-    def from_crawler(cls, *args, **kwargs):
-        o = cls(*args, **kwargs)
-        return o
-
-    def _add_pipelines(self, pipelines):
-        for pipeline in pipelines:
-            pipeline_cls = load_class(pipeline)
-            if not hasattr(pipeline_cls, 'from_crawler'):
-                raise PipelineInitError(
-                    f"Pipeline init failed, must inherit from `BasePipeline` or have a `create_instance` method"
-                )
-            self.pipelines.append(pipeline_cls.from_crawler(self.crawler))
-        if pipelines:
-            self.logger.info(f"enabled pipelines: \n {pformat(pipelines)}")
-
-    def _add_methods(self):
-        for pipeline in self.pipelines:
-            if hasattr(pipeline, 'process_item'):
-                self.methods.append(pipeline.process_item)
-
-    async def process_item(self, item):
-        try:
-            for method in self.methods:
-                item = await common_call(method, item, self.crawler.spider)
-                if item is None:
-                    raise InvalidOutputError(f"{method.__qualname__} return None is not supported.")
-        except ItemDiscard as exc:
-            create_task(self.crawler.subscriber.notify(item_discard, item, exc, self.crawler.spider))
-        else:
-            create_task(self.crawler.subscriber.notify(item_successful, item, self.crawler.spider))
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+from typing import List
+from pprint import pformat
+from asyncio import create_task
+
+
+from crawlo.utils.log import get_logger
+from crawlo.event import item_successful, item_discard
+from crawlo.project import load_class, common_call
+from crawlo.exceptions import PipelineInitError, ItemDiscard, InvalidOutputError
+
+
+class PipelineManager:
+
+    def __init__(self, crawler):
+        self.crawler = crawler
+        self.pipelines: List = []
+        self.methods: List = []
+
+        self.logger = get_logger(self.__class__.__name__, self.crawler.settings.get('LOG_LEVEL'))
+        pipelines = self.crawler.settings.get_list('PIPELINES')
+        self._add_pipelines(pipelines)
+        self._add_methods()
+
+    @classmethod
+    def from_crawler(cls, *args, **kwargs):
+        o = cls(*args, **kwargs)
+        return o
+
+    def _add_pipelines(self, pipelines):
+        for pipeline in pipelines:
+            pipeline_cls = load_class(pipeline)
+            if not hasattr(pipeline_cls, 'from_crawler'):
+                raise PipelineInitError(
+                    f"Pipeline init failed, must inherit from `BasePipeline` or have a `create_instance` method"
+                )
+            self.pipelines.append(pipeline_cls.from_crawler(self.crawler))
+        if pipelines:
+            self.logger.info(f"enabled pipelines: \n {pformat(pipelines)}")
+
+    def _add_methods(self):
+        for pipeline in self.pipelines:
+            if hasattr(pipeline, 'process_item'):
+                self.methods.append(pipeline.process_item)
+
+    async def process_item(self, item):
+        try:
+            for method in self.methods:
+                item = await common_call(method, item, self.crawler.spider)
+                if item is None:
+                    raise InvalidOutputError(f"{method.__qualname__} return None is not supported.")
+        except ItemDiscard as exc:
+            create_task(self.crawler.subscriber.notify(item_discard, item, exc, self.crawler.spider))
+        else:
+            create_task(self.crawler.subscriber.notify(item_successful, item, self.crawler.spider))

crawlo/project.py

@@ -0,0 +1,153 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+Crawlo 项目初始化模块
+
+负责：
+1. 向上搜索项目根目录（通过 crawlo.cfg 或 settings.py）
+2. 将项目根目录加入 sys.path
+3. 加载 settings 模块
+4. 返回 SettingManager 实例
+"""
+import os
+import sys
+import configparser
+from importlib import import_module
+from inspect import iscoroutinefunction
+from typing import Callable, Optional, Tuple
+
+from crawlo.utils.log import get_logger
+from crawlo.settings.setting_manager import SettingManager
+
+logger = get_logger(__name__)
+
+
+def _find_project_root(start_path: str = ".") -> Optional[str]:
+    """
+    从指定路径向上查找项目根目录。
+    识别依据：
+        1. 存在 'crawlo.cfg'
+        2. 存在 '__init__.py' 和 'settings.py'（即 Python 包）
+    """
+    path = os.path.abspath(start_path)
+    while True:
+        cfg_file = os.path.join(path, "crawlo.cfg")
+        if os.path.isfile(cfg_file):
+            logger.info(f"✅ 找到项目配置文件: {cfg_file}")
+            return path
+
+        settings_file = os.path.join(path, "settings.py")
+        init_file = os.path.join(path, "__init__.py")
+        if os.path.isfile(settings_file) and os.path.isfile(init_file):
+            logger.info(f"✅ 找到项目模块: {path}")
+            return path
+
+        parent = os.path.dirname(path)
+        if parent == path:
+            break
+        path = parent
+
+    logger.warning("❌ 未找到 Crawlo 项目根目录。请确保在包含 'crawlo.cfg' 或 'settings.py' 的目录运行。")
+    return None
+
+
+def _get_settings_module_from_cfg(cfg_path: str) -> str:
+    """从 crawlo.cfg 读取 settings 模块路径"""
+    config = configparser.ConfigParser()
+    try:
+        config.read(cfg_path, encoding="utf-8")
+        if config.has_section("settings") and config.has_option("settings", "default"):
+            module_path = config.get("settings", "default")
+            logger.info(f"📄 从 crawlo.cfg 加载 settings 模块: {module_path}")
+            return module_path
+        else:
+            raise RuntimeError(f"配置文件缺少 [settings] 或 default 选项: {cfg_path}")
+    except Exception as e:
+        raise RuntimeError(f"解析 crawlo.cfg 失败: {e}")
+
+
+def get_settings(custom_settings: Optional[dict] = None) -> SettingManager:
+    """
+    获取配置管理器实例（主入口函数）
+
+    Args:
+        custom_settings: 运行时自定义配置，会覆盖 settings.py
+
+    Returns:
+        SettingManager: 已加载配置的实例
+    """
+    logger.info("🚀 正在初始化 Crawlo 项目配置...")
+
+    # 1. 查找项目根
+    project_root = _find_project_root()
+    if not project_root:
+        raise RuntimeError("未找到 Crawlo 项目，请检查项目结构")
+
+    # 2. 确定 settings 模块
+    settings_module_path = None
+    cfg_file = os.path.join(project_root, "crawlo.cfg")
+
+    if os.path.isfile(cfg_file):
+        settings_module_path = _get_settings_module_from_cfg(cfg_file)
+    else:
+        # 推断：项目目录名.settings
+        project_name = os.path.basename(project_root)
+        settings_module_path = f"{project_name}.settings"
+        logger.warning(f"⚠️ 未找到 crawlo.cfg，推断 settings 模块为: {settings_module_path}")
+
+    # 3. 注入 sys.path
+    project_root_str = os.path.abspath(project_root)
+    if project_root_str not in sys.path:
+        sys.path.insert(0, project_root_str)
+        logger.info(f"📁 项目根目录已加入 sys.path: {project_root_str}")
+
+    # 4. 加载 SettingManager
+    logger.info(f"⚙️ 正在加载配置模块: {settings_module_path}")
+    settings = SettingManager()
+
+    try:
+        settings.set_settings(settings_module_path)
+        logger.info("✅ settings 模块加载成功")
+    except Exception as e:
+        raise ImportError(f"加载 settings 模块失败 '{settings_module_path}': {e}")
+
+    # 5. 合并运行时配置
+    if custom_settings:
+        settings.update_attributes(custom_settings)
+        logger.info(f"🔧 已应用运行时自定义配置: {list(custom_settings.keys())}")
+
+    logger.info("🎉 Crawlo 项目配置初始化完成！")
+    return settings
+
+
+def load_class(_path):
+    if not isinstance(_path, str):
+        if callable(_path):
+            return _path
+        else:
+            raise TypeError(f"args expect str or object, got {_path}")
+
+    module_name, class_name = _path.rsplit('.', 1)
+    module = import_module(module_name)
+
+    try:
+        cls = getattr(module, class_name)
+    except AttributeError:
+        raise NameError(f"Module {module_name!r} has no class named {class_name!r}")
+    return cls
+
+
+def merge_settings(spider, settings):
+    spider_name = getattr(spider, 'name', 'UnknownSpider')
+    if hasattr(spider, 'custom_settings'):
+        custom_settings = getattr(spider, 'custom_settings')
+        settings.update_attributes(custom_settings)
+    else:
+        logger.debug(f"爬虫 '{spider_name}' 无 custom_settings，跳过合并")  # 添加日志
+
+
+async def common_call(func: Callable, *args, **kwargs):
+    if iscoroutinefunction(func):
+        return await func(*args, **kwargs)
+    else:
+        return func(*args, **kwargs)