crawlo 1.0.1__py3-none-any.whl → 1.0.2__py3-none-any.whl

crawlo/network/response.py

@@ -2,8 +2,8 @@
 # -*- coding:UTF-8 -*-
 import re
 import ujson
-from typing import Dict
-from parsel import Selector
+from typing import Dict, Any, List, Optional
+from parsel import Selector, SelectorList
 from http.cookies import SimpleCookie
 from urllib.parse import urljoin as _urljoin
 
@@ -11,17 +11,20 @@ from crawlo import Request
 from crawlo.exceptions import DecodeError
 
 
-class Response(object):
+class Response:
+    """
+    HTTP响应的封装，提供数据解析的便捷方法。
+    """
 
     def __init__(
-            self,
-            url: str,
-            *,
-            headers: Dict,
-            body: bytes = b"",
-            method: str = 'GET',
-            request: Request = None,
-            status_code: int = 200,
+        self,
+        url: str,
+        *,
+        headers: Dict[str, Any],
+        body: bytes = b"",
+        method: str = 'GET',
+        request: Request = None,
+        status_code: int = 200,
     ):
         self.url = url
         self.headers = headers
@@ -29,65 +32,131 @@ class Response(object):
         self.method = method
         self.request = request
         self.status_code = status_code
-        self.encoding = request.encoding
-        self._selector = None
+        self.encoding = self.request.encoding if self.request else None
         self._text_cache = None
+        self._selector_instance = None  # 修改变量名，避免与 @property 冲突
 
     @property
-    def text(self):
-        # 请求缓存
-        if self._text_cache:
+    def text(self) -> str:
+        """将响应体(body)以正确的编码解码为字符串，并缓存结果。"""
+        if self._text_cache is not None:
             return self._text_cache
+
+        encoding = self.encoding
         try:
-            self._text_cache = self.body.decode(self.encoding)
-        except UnicodeDecodeError:
-            try:
-                _encoding_re = re.compile(r"charset=([\w-]+)", flags=re.I)
-                _encoding_string = self.headers.get('Content-Type', '') or self.headers.get('content-type', '')
-                _encoding = _encoding_re.search(_encoding_string)
-                if _encoding:
-                    _encoding = _encoding.group(1)
-                    self._text_cache = self.body.decode(_encoding)
-                else:
-                    raise DecodeError(f"{self.request} {self.request.encoding} error.")
-            except UnicodeDecodeError as exp:
-                raise UnicodeDecodeError(
-                    exp.encoding, exp.object, exp.start, exp.end, f"{self.request} error."
-                )
-        return self._text_cache
-
-    def json(self):
+            # 优先使用 request 提供的编码
+            if encoding:
+                self._text_cache = self.body.decode(encoding)
+                return self._text_cache
+
+            # 从 Content-Type 中提取编码
+            content_type = self.headers.get("Content-Type", "")
+            charset_match = re.search(r"charset=([\w-]+)", content_type, re.I)
+            if charset_match:
+                encoding = charset_match.group(1)
+                self._text_cache = self.body.decode(encoding)
+                return self._text_cache
+
+            # 默认尝试 UTF-8
+            self._text_cache = self.body.decode("utf-8")
+            return self._text_cache
+
+        except UnicodeDecodeError as e:
+            raise DecodeError(f"Failed to decode response from {self.url}: {e}")
+
+    def json(self) -> Any:
+        """将响应文本解析为 JSON 对象。"""
         return ujson.loads(self.text)
 
-    def urljoin(self, url):
+    def urljoin(self, url: str) -> str:
+        """拼接 URL，自动处理相对路径。"""
         return _urljoin(self.url, url)
 
-    def xpath(self, xpath_str):
-        if self._selector is None:
-            self._selector = Selector(self.text)
-        return self._selector.xpath(xpath_str)
+    @property
+    def _selector(self) -> Selector:
+        """懒加载 Selector 实例"""
+        if self._selector_instance is None:
+            self._selector_instance = Selector(self.text)
+        return self._selector_instance
+
+    def xpath(self, query: str) -> SelectorList:
+        """使用 XPath 选择器查询文档。"""
+        return self._selector.xpath(query)
+
+    def css(self, query: str) -> SelectorList:
+        """使用 CSS 选择器查询文档。"""
+        return self._selector.css(query)
+
+    def xpath_text(self, query: str) -> str:
+        """使用 XPath 提取并返回纯文本。"""
+        fragments = self.xpath(f"{query}//text()").getall()
+        return " ".join(text.strip() for text in fragments if text.strip())
+
+    def css_text(self, query: str) -> str:
+        """使用 CSS 选择器提取并返回纯文本。"""
+        fragments = self.css(f"{query} ::text").getall()
+        return " ".join(text.strip() for text in fragments if text.strip())
+
+    def get_text(self, xpath_or_css: str, join_str: str = " ") -> str:
+        """
+        获取指定节点的纯文本(自动拼接子节点文本)
+
+        参数:
+            xpath_or_css: XPath或CSS选择器
+            join_str: 文本拼接分隔符(默认为空格)
+
+        返回:
+            拼接后的纯文本字符串
+        """
+        elements = self.xpath(xpath_or_css) if xpath_or_css.startswith(('/', '//', './')) else self.css(xpath_or_css)
+        texts = elements.xpath('.//text()').getall()
+        return join_str.join(text.strip() for text in texts if text.strip())
+
+    def get_all_text(self, xpath_or_css: str, join_str: str = " ") -> List[str]:
+        """
+        获取多个节点的纯文本列表
+
+        参数:
+            xpath_or_css: XPath或CSS选择器
+            join_str: 单个节点内文本拼接分隔符
 
-    def css(self, css_str):
-        if self._selector is None:
-            self._selector = Selector(self.text)
-        return self._selector.css(css_str)
+        返回:
+            纯文本列表(每个元素对应一个节点的文本)
+        """
+        elements = self.xpath(xpath_or_css) if xpath_or_css.startswith(('/', '//', './')) else self.css(xpath_or_css)
+        result = []
+        for element in elements:
+            texts = element.xpath('.//text()').getall()
+            clean_text = join_str.join(text.strip() for text in texts if text.strip())
+            if clean_text:
+                result.append(clean_text)
+        return result
 
-    def re_search(self, pattern, flags=re.DOTALL):
+    def re_search(self, pattern: str, flags: int = re.DOTALL) -> Optional[re.Match]:
+        """在响应文本上执行正则表达式搜索。"""
+        if not isinstance(pattern, str):
+            raise TypeError("Pattern must be a string")
         return re.search(pattern, self.text, flags=flags)
 
-    def re_findall(self, pattern, flags=re.DOTALL):
+    def re_findall(self, pattern: str, flags: int = re.DOTALL) -> List[Any]:
+        """在响应文本上执行正则表达式查找。"""
+        if not isinstance(pattern, str):
+            raise TypeError("Pattern must be a string")
         return re.findall(pattern, self.text, flags=flags)
 
-    def get_cookies(self):
-        cookie_headers = self.headers.getlist('Set-Cookie') or []
+    def get_cookies(self) -> Dict[str, str]:
+        """从响应头中解析并返回Cookies。"""
+        cookie_header = self.headers.get("Set-Cookie", "")
+        if isinstance(cookie_header, list):
+            cookie_header = ", ".join(cookie_header)
         cookies = SimpleCookie()
-        for header in cookie_headers:
-            cookies.load(header)
-        return {k: v.value for k, v in cookies.items()}
+        cookies.load(cookie_header)
+        return {key: morsel.value for key, morsel in cookies.items()}
 
     @property
-    def meta(self):
-        return self.request.meta
+    def meta(self) -> Dict:
+        """获取关联的 Request 对象的 meta 字典。"""
+        return self.request.meta if self.request else {}
 
     def __str__(self):
-        return f"{self.url} {self.status_code} {self.request.encoding} "
+        return f"<{self.status_code} {self.url}>"

crawlo/pipelines/mysql_pipeline.py

@@ -5,6 +5,7 @@ from typing import Optional
 from asyncmy import create_pool
 from crawlo.utils.log import get_logger
 from crawlo.exceptions import ItemDiscard
+from crawlo.utils.tools import make_insert_sql, logger
 
 
 class AsyncmyMySQLPipeline:
@@ -54,29 +55,28 @@ class AsyncmyMySQLPipeline:
                     self.logger.error(f"MySQL连接池初始化失败: {e}")
                     raise
 
-    async def process_item(self, item, spider) -> Optional[dict]:
+    async def process_item(self, item, spider, kwargs=None) -> Optional[dict]:
         """处理item的核心方法"""
+        kwargs = kwargs or {}
+        spider_name = getattr(spider, 'name', 'unknown')  # 获取爬虫名称
         try:
             await self._ensure_pool()
-
             item_dict = dict(item)
-            columns = ', '.join([f'`{k}`' for k in item_dict.keys()])
-            placeholders = ', '.join(['%s'] * len(item_dict))
-
-            async with self.pool.acquire() as conn:
-                async with conn.cursor() as cursor:
-                    try:
-                        await cursor.execute(
-                            f"INSERT INTO `{self.table_name}` ({columns}) VALUES ({placeholders})",
-                            list(item_dict.values())
-                        )
-                        await conn.commit()
-                        self.crawler.stats.inc_value('mysql/insert_success')
-                    except Exception as e:
-                        await conn.rollback()
-                        self.crawler.stats.inc_value('mysql/insert_failed')
-                        self.logger.error(f"MySQL插入失败: {e}")
-                        raise ItemDiscard(f"MySQL插入失败: {e}")
+            sql = make_insert_sql(table=self.table_name, data=item_dict, **kwargs)
+
+            rowcount = await self._execute_sql(sql=sql)
+            if rowcount > 1:
+                self.logger.info(
+                    f"爬虫 {spider_name} 成功插入 {rowcount} 条记录到表 {self.table_name}"
+                )
+            elif rowcount == 1:
+                self.logger.debug(
+                    f"爬虫 {spider_name} 成功插入单条记录到表 {self.table_name}"
+                )
+            else:
+                self.logger.warning(
+                    f"爬虫 {spider_name}: SQL执行成功但未插入新记录 - {sql[:100]}..."
+                )
 
             return item
 
@@ -84,6 +84,25 @@ class AsyncmyMySQLPipeline:
             self.logger.error(f"处理item时发生错误: {e}")
             raise ItemDiscard(f"处理失败: {e}")
 
+    async def _execute_sql(self, sql: str, values: list = None) -> int:
+        """执行SQL语句并处理结果"""
+        async with self.pool.acquire() as conn:
+            async with conn.cursor() as cursor:
+                try:
+                    # 根据是否有参数值选择不同的执行方法
+                    if values is not None:
+                        rowcount = await cursor.execute(sql, values)
+                    else:
+                        rowcount = await cursor.execute(sql)
+
+                    await conn.commit()
+                    self.crawler.stats.inc_value('mysql/insert_success')
+                    return rowcount
+                except Exception as e:
+                    await conn.rollback()
+                    self.crawler.stats.inc_value('mysql/insert_failed')
+                    raise ItemDiscard(f"MySQL插入失败: {e}")
+
     async def spider_closed(self):
         """关闭爬虫时清理资源"""
         if self.pool:

crawlo/subscriber.py

@@ -3,25 +3,104 @@
 import asyncio
 from collections import defaultdict
 from inspect import iscoroutinefunction
-from typing import Dict, Set, Callable, Coroutine
+from typing import Dict, Set, Callable, Coroutine, Any, TypeAlias, List
 
-from crawlo.exceptions import ReceiverTypeError
+
+class ReceiverTypeError(TypeError):
+    """当订阅的接收者不是一个协程函数时抛出。"""
+    pass
+
+
+ReceiverCoroutine: TypeAlias = Callable[..., Coroutine[Any, Any, Any]]
 
 
 class Subscriber:
+    """
+    一个支持异步协程的发布/订阅（Pub/Sub）模式实现。
+
+    这个类允许你注册（订阅）协程函数来监听特定事件，并在事件发生时
+    以并发的方式异步地通知所有订阅者。
+    """
 
     def __init__(self):
-        self._subscribers: Dict[str, Set[Callable[..., Coroutine]]] = defaultdict(set)
+        """初始化一个空的订阅者字典。"""
+        self._subscribers: Dict[str, Set[ReceiverCoroutine]] = defaultdict(set)
 
-    def subscribe(self, receiver: Callable[..., Coroutine], *, event: str) -> None:
+    def subscribe(self, receiver: ReceiverCoroutine, *, event: str) -> None:
+        """
+        订阅一个事件。
+
+        Args:
+            receiver: 一个协程函数 (例如 async def my_func(...))。
+            event: 要订阅的事件名称。
+
+        Raises:
+            ReceiverTypeError: 如果提供的 `receiver` 不是一个协程函数。
+        """
         if not iscoroutinefunction(receiver):
-            raise ReceiverTypeError(f"{receiver.__qualname__} must be a coroutine function")
+            raise ReceiverTypeError(f"接收者 '{receiver.__qualname__}' 必须是一个协程函数。")
         self._subscribers[event].add(receiver)
 
-    def unsubscribe(self, receiver: Callable[..., Coroutine], *, event: str) -> None:
-        self._subscribers[event].discard(receiver)
+    def unsubscribe(self, receiver: ReceiverCoroutine, *, event: str) -> None:
+        """
+        取消订阅一个事件。
+
+        如果事件或接收者不存在，将静默处理。
+
+        Args:
+            receiver: 要取消订阅的协程函数。
+            event: 事件名称。
+        """
+        if event in self._subscribers:
+            self._subscribers[event].discard(receiver)
+
+    async def notify(self, event: str, *args, **kwargs) -> List[Any]:
+        """
+        异步地、并发地通知所有订阅了该事件的接收者。
+
+        此方法会等待所有订阅者任务完成后再返回，并收集所有结果或异常。
+
+        Args:
+            event: 要触发的事件名称。
+            *args: 传递给接收者的位置参数。
+            **kwargs: 传递给接收者的关键字参数。
+
+        Returns:
+            一个列表，包含每个订阅者任务的返回结果或在执行期间捕获的异常。
+        """
+        receivers = self._subscribers.get(event, set())
+        if not receivers:
+            return []
+
+        tasks = [asyncio.create_task(receiver(*args, **kwargs)) for receiver in receivers]
+
+        # 并发执行所有任务并返回结果列表（包括异常）
+        return await asyncio.gather(*tasks, return_exceptions=True)
 
-    async def notify(self, event: str, *args, **kwargs) -> None:
-        for receiver in self._subscribers[event]:
-            # 不能 await
-            asyncio.create_task(receiver(*args, **kwargs))
+# #!/usr/bin/python
+# # -*- coding:UTF-8 -*-
+# import asyncio
+# from collections import defaultdict
+# from inspect import iscoroutinefunction
+# from typing import Dict, Set, Callable, Coroutine
+#
+# from crawlo.exceptions import ReceiverTypeError
+#
+#
+# class Subscriber:
+#
+#     def __init__(self):
+#         self._subscribers: Dict[str, Set[Callable[..., Coroutine]]] = defaultdict(set)
+#
+#     def subscribe(self, receiver: Callable[..., Coroutine], *, event: str) -> None:
+#         if not iscoroutinefunction(receiver):
+#             raise ReceiverTypeError(f"{receiver.__qualname__} must be a coroutine function")
+#         self._subscribers[event].add(receiver)
+#
+#     def unsubscribe(self, receiver: Callable[..., Coroutine], *, event: str) -> None:
+#         self._subscribers[event].discard(receiver)
+#
+#     async def notify(self, event: str, *args, **kwargs) -> None:
+#         for receiver in self._subscribers[event]:
+#             # 不能 await
+#             asyncio.create_task(receiver(*args, **kwargs))

crawlo/utils/concurrency_manager.py

@@ -0,0 +1,125 @@
+import os
+import platform
+import logging
+from typing import Optional
+
+try:
+    import psutil  # 用于获取系统资源信息的第三方库
+except ImportError:
+    psutil = None  # 如果psutil不可用则设为None
+
+logger = logging.getLogger(__name__)
+
+
+def calculate_optimal_concurrency(user_specified: Optional[int] = None, use_logical_cores: bool = True) -> int:
+    """
+    基于系统资源计算最优并发数，或使用用户指定值
+
+    参数:
+        user_specified: 用户指定的并发数（优先使用）
+        use_logical_cores: 是否使用逻辑CPU核心数（超线程），默认为True
+
+    返回:
+        计算得出的最优并发数
+
+    说明:
+        1. 优先使用用户指定的并发数
+        2. 根据操作系统类型采用不同的计算策略：
+           - Windows: 保守计算，避免内存压力
+           - macOS: 平衡资源使用
+           - Linux: 充分利用服务器资源
+           - 其他系统: 使用合理默认值
+        3. 使用可用内存和CPU核心数进行计算
+        4. 提供psutil不可用时的备用方案
+    """
+    # 优先使用用户指定的并发数
+    if user_specified is not None:
+        logger.info(f"使用用户指定的并发数: {user_specified}")
+        return user_specified
+
+    try:
+        current_os = platform.system()  # 获取当前操作系统类型
+        logger.debug(f"检测到操作系统: {current_os}")
+
+        # 获取CPU核心数（根据参数决定是否使用逻辑核心）
+        cpu_count = psutil.cpu_count(logical=use_logical_cores) or 1 if psutil else os.cpu_count() or 1
+
+        # 根据操作系统类型选择不同的计算方法
+        if current_os == "Windows":
+            concurrency = _get_concurrency_for_windows(cpu_count, use_logical_cores)
+        elif current_os == "Darwin":  # macOS系统
+            concurrency = _get_concurrency_for_macos(cpu_count, use_logical_cores)
+        elif current_os == "Linux":
+            concurrency = _get_concurrency_for_linux(cpu_count, use_logical_cores)
+        else:  # 其他操作系统
+            concurrency = _get_concurrency_default(cpu_count)
+
+        logger.info(f"计算得到最大并发数: {concurrency}")
+        return concurrency
+
+    except Exception as e:
+        logger.warning(f"动态计算并发数失败: {str(e)}，使用默认值50")
+        return 50  # 计算失败时的安全默认值
+
+
+def _get_concurrency_for_windows(cpu_count: int, use_logical_cores: bool) -> int:
+    """Windows系统专用的并发数计算逻辑"""
+    if psutil:
+        # 计算可用内存（GB）
+        available_memory = psutil.virtual_memory().available / (1024 ** 3)
+        # 内存计算：每4GB可用内存分配10个并发
+        mem_based = int((available_memory / 4) * 10)
+        # CPU计算：使用逻辑核心时乘数较大
+        cpu_based = cpu_count * (5 if use_logical_cores else 3)
+        # 取5-100之间的值，选择内存和CPU限制中较小的
+        return max(5, min(100, mem_based, cpu_based))
+    else:
+        # 无psutil时的备用方案
+        return min(50, cpu_count * 5)
+
+
+def _get_concurrency_for_macos(cpu_count: int, use_logical_cores: bool) -> int:
+    """macOS系统专用的并发数计算逻辑"""
+    if psutil:
+        available_memory = psutil.virtual_memory().available / (1024 ** 3)
+        # 内存计算：每3GB可用内存分配10个并发
+        mem_based = int((available_memory / 3) * 10)
+        # CPU计算：使用逻辑核心时乘数较大
+        cpu_based = cpu_count * (6 if use_logical_cores else 4)
+        # 取5-120之间的值
+        return max(5, min(120, mem_based, cpu_based))
+    else:
+        try:
+            # macOS备用方案：使用系统命令获取物理CPU核心数
+            import subprocess
+            output = subprocess.check_output(["sysctl", "hw.physicalcpu"])
+            cpu_count = int(output.split()[1])
+            return min(60, cpu_count * 5)
+        except:
+            return 40  # Mac电脑的合理默认值
+
+
+def _get_concurrency_for_linux(cpu_count: int, use_logical_cores: bool) -> int:
+    """Linux系统专用的并发数计算逻辑（更激进）"""
+    if psutil:
+        available_memory = psutil.virtual_memory().available / (1024 ** 3)
+        # 内存计算：每1.5GB可用内存分配10个并发
+        mem_based = int((available_memory / 1.5) * 10)
+        # CPU计算：服务器环境使用更大的乘数
+        cpu_based = cpu_count * (8 if use_logical_cores else 5)
+        # 取5-200之间的值
+        return max(5, min(200, mem_based, cpu_based))
+    else:
+        try:
+            # Linux备用方案：解析/proc/cpuinfo文件
+            with open("/proc/cpuinfo") as f:
+                cpu_count = f.read().count("processor\t:")
+                if cpu_count > 0:
+                    return min(200, cpu_count * 8)
+        except:
+            return 50  # Linux服务器的合理默认值
+
+
+def _get_concurrency_default(cpu_count: int) -> int:
+    """未知操作系统的默认计算逻辑"""
+    return min(50, cpu_count * 5)  # 保守的默认计算方式