crawlo 1.0.0__py3-none-any.whl → 1.0.2__py3-none-any.whl

crawlo/pipelines/mysql_batch_pipline.py

@@ -0,0 +1,134 @@
+import asyncio
+import logging
+from crawlo.exceptions import NotConfiguredError
+
+logger = logging.getLogger(__name__)
+
+
+class AsyncmyMySQLPipeline:
+    """
+    异步 MySQL 管道，用于批量将爬取的数据存储到 MySQL 数据库
+    """
+
+    def __init__(self, host, port, user, password, db, batch_size, table_name):
+        self.host = host
+        self.port = port
+        self.user = user
+        self.password = password
+        self.db = db
+        self.batch_size = batch_size
+        self.table_name = table_name
+        self.buffer = []
+        self.pool = None
+        self.flush_task = None
+        self.insert_stmt = None
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        """从爬虫配置中获取数据库连接参数"""
+        batch_size = crawler.settings.get_int('MYSQL_BATCH_SIZE', 100)
+
+        return cls(
+            host=crawler.settings.get('MYSQL_HOST'),
+            port=crawler.settings.get('MYSQL_PORT'),
+            user=crawler.settings.get('MYSQL_USER'),
+            password=crawler.settings.get('MYSQL_PASSWORD'),
+            db=crawler.settings.get('MYSQL_DB'),
+            batch_size=batch_size,
+            table_name=crawler.settings.get('MYSQL_TABLE'),
+        )
+
+    async def open_spider(self, spider):
+        """爬虫启动时初始化数据库连接池"""
+        try:
+            import asyncmy
+            self.pool = await asyncmy.create_pool(
+                host=self.host,
+                port=self.port,
+                user=self.user,
+                password=self.password,
+                db=self.db,
+                autocommit=True,
+                charset='utf8mb4',
+                cursorclass=asyncmy.cursors.DictCursor
+            )
+            logger.info(f"MySQL 连接池已创建，主机: {self.host}, 数据库: {self.db}")
+
+            # 创建自动刷新任务
+            self.flush_task = asyncio.create_task(self._auto_flush())
+
+        except Exception as e:
+            logger.error(f"无法创建 MySQL 连接池: {e}")
+            raise NotConfiguredError(f"MySQL 连接失败: {e}")
+
+    async def process_item(self, item, spider):
+        """处理爬取的每个项目，添加到缓冲区"""
+        self.buffer.append(dict(item))
+
+        # 当缓冲区达到批量大小时自动刷新
+        if len(self.buffer) >= self.batch_size:
+            await self._flush_buffer()
+
+        return item
+
+    async def _flush_buffer(self):
+        """将缓冲区中的数据批量写入数据库"""
+        if not self.buffer:
+            return
+
+        try:
+            async with self.pool.acquire() as conn:
+                async with conn.cursor() as cursor:
+                    # 动态生成插入语句
+                    if not self.insert_stmt:
+                        columns = ', '.join(self.buffer[0].keys())
+                        placeholders = ', '.join(['%s'] * len(self.buffer[0]))
+                        self.insert_stmt = f"INSERT INTO {self.table_name} ({columns}) VALUES ({placeholders})"
+
+                    # 准备批量插入的数据
+                    values = [tuple(item.values()) for item in self.buffer]
+
+                    # 执行批量插入
+                    await cursor.executemany(self.insert_stmt, values)
+                    logger.debug(f"已批量插入 {len(values)} 条记录到 {self.table_name}")
+
+                    # 清空缓冲区
+                    self.buffer.clear()
+
+        except Exception as e:
+            logger.error(f"批量插入失败: {e}")
+            # 发生错误时保留数据，避免数据丢失
+            # 实际生产环境中可能需要更复杂的错误处理策略
+
+    async def _auto_flush(self):
+        """定期自动刷新缓冲区，防止数据长时间停留在内存中"""
+        try:
+            while True:
+                await asyncio.sleep(10)  # 每10秒检查一次
+                if self.buffer:
+                    await self._flush_buffer()
+        except asyncio.CancelledError:
+            logger.info("自动刷新任务已取消")
+
+    async def spider_closed(self, spider):
+        """爬虫关闭时执行清理工作"""
+        try:
+            # 取消自动刷新任务
+            if self.flush_task:
+                self.flush_task.cancel()
+                await self.flush_task  # 等待任务完成取消
+
+            # 确保缓冲区中的剩余数据被写入数据库
+            if self.buffer:
+                await self._flush_buffer()
+
+        except asyncio.CancelledError:
+            logger.info("爬虫关闭过程中自动刷新任务被取消")
+        except Exception as e:
+            logger.error(f"爬虫关闭时发生错误: {e}")
+        finally:
+            # 关闭数据库连接池
+            if self.pool:
+                self.pool.close()
+                await self.pool.wait_closed()
+                logger.info("MySQL 连接池已关闭")

crawlo/pipelines/mysql_pipeline.py

@@ -1,5 +1,195 @@
 # -*- coding: utf-8 -*-
+import asyncio
+import aiomysql
+from typing import Optional
+from asyncmy import create_pool
+from crawlo.utils.log import get_logger
+from crawlo.exceptions import ItemDiscard
+from crawlo.utils.tools import make_insert_sql, logger
 
 
-class MySQLPipeline(object):
-    pass
+class AsyncmyMySQLPipeline:
+    def __init__(self, crawler):
+        self.crawler = crawler
+        self.settings = crawler.settings
+        self.logger = get_logger(self.__class__.__name__, self.settings.get('LOG_LEVEL'))
+
+        # 使用异步锁和初始化标志确保线程安全
+        self._pool_lock = asyncio.Lock()
+        self._pool_initialized = False
+        self.pool = None
+        self.table_name = (
+                self.settings.get('MYSQL_TABLE') or
+                getattr(crawler.spider, 'mysql_table', None) or
+                f"{crawler.spider.name}_items"
+        )
+
+        # 注册关闭事件
+        crawler.subscriber.subscribe(self.spider_closed, event='spider_closed')
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        return cls(crawler)
+
+    async def _ensure_pool(self):
+        """确保连接池已初始化（线程安全）"""
+        if self._pool_initialized:
+            return
+
+        async with self._pool_lock:
+            if not self._pool_initialized:  # 双重检查避免竞争条件
+                try:
+                    self.pool = await create_pool(
+                        host=self.settings.get('MYSQL_HOST', 'localhost'),
+                        port=self.settings.get_int('MYSQL_PORT', 3306),
+                        user=self.settings.get('MYSQL_USER', 'root'),
+                        password=self.settings.get('MYSQL_PASSWORD', ''),
+                        db=self.settings.get('MYSQL_DB', 'scrapy_db'),
+                        minsize=self.settings.get_int('MYSQL_POOL_MIN', 3),
+                        maxsize=self.settings.get_int('MYSQL_POOL_MAX', 10),
+                        echo=self.settings.get_bool('MYSQL_ECHO', False)
+                    )
+                    self._pool_initialized = True
+                    self.logger.debug(f"MySQL连接池初始化完成（表: {self.table_name}）")
+                except Exception as e:
+                    self.logger.error(f"MySQL连接池初始化失败: {e}")
+                    raise
+
+    async def process_item(self, item, spider, kwargs=None) -> Optional[dict]:
+        """处理item的核心方法"""
+        kwargs = kwargs or {}
+        spider_name = getattr(spider, 'name', 'unknown')  # 获取爬虫名称
+        try:
+            await self._ensure_pool()
+            item_dict = dict(item)
+            sql = make_insert_sql(table=self.table_name, data=item_dict, **kwargs)
+
+            rowcount = await self._execute_sql(sql=sql)
+            if rowcount > 1:
+                self.logger.info(
+                    f"爬虫 {spider_name} 成功插入 {rowcount} 条记录到表 {self.table_name}"
+                )
+            elif rowcount == 1:
+                self.logger.debug(
+                    f"爬虫 {spider_name} 成功插入单条记录到表 {self.table_name}"
+                )
+            else:
+                self.logger.warning(
+                    f"爬虫 {spider_name}: SQL执行成功但未插入新记录 - {sql[:100]}..."
+                )
+
+            return item
+
+        except Exception as e:
+            self.logger.error(f"处理item时发生错误: {e}")
+            raise ItemDiscard(f"处理失败: {e}")
+
+    async def _execute_sql(self, sql: str, values: list = None) -> int:
+        """执行SQL语句并处理结果"""
+        async with self.pool.acquire() as conn:
+            async with conn.cursor() as cursor:
+                try:
+                    # 根据是否有参数值选择不同的执行方法
+                    if values is not None:
+                        rowcount = await cursor.execute(sql, values)
+                    else:
+                        rowcount = await cursor.execute(sql)
+
+                    await conn.commit()
+                    self.crawler.stats.inc_value('mysql/insert_success')
+                    return rowcount
+                except Exception as e:
+                    await conn.rollback()
+                    self.crawler.stats.inc_value('mysql/insert_failed')
+                    raise ItemDiscard(f"MySQL插入失败: {e}")
+
+    async def spider_closed(self):
+        """关闭爬虫时清理资源"""
+        if self.pool:
+            self.pool.close()
+            await self.pool.wait_closed()
+            self.logger.info("MySQL连接池已关闭")
+
+
+class AiomysqlMySQLPipeline:
+    def __init__(self, crawler):
+        self.crawler = crawler
+        self.settings = crawler.settings
+        self.logger = get_logger(self.__class__.__name__, self.settings.get('LOG_LEVEL'))
+
+        # 使用异步锁和初始化标志
+        self._pool_lock = asyncio.Lock()
+        self._pool_initialized = False
+        self.pool = None
+        self.table_name = (
+                self.settings.get('MYSQL_TABLE') or
+                getattr(crawler.spider, 'mysql_table', None) or
+                f"{crawler.spider.name}_items"
+        )
+
+        crawler.subscriber.subscribe(self.spider_closed, event='spider_closed')
+
+    @classmethod
+    def create_instance(cls, crawler):
+        return cls(crawler)
+
+    async def _init_pool(self):
+        """延迟初始化连接池（线程安全）"""
+        if self._pool_initialized:
+            return
+
+        async with self._pool_lock:
+            if not self._pool_initialized:
+                try:
+                    self.pool = await aiomysql.create_pool(
+                        host=self.settings.get('MYSQL_HOST', 'localhost'),
+                        port=self.settings.getint('MYSQL_PORT', 3306),
+                        user=self.settings.get('MYSQL_USER', 'root'),
+                        password=self.settings.get('MYSQL_PASSWORD', ''),
+                        db=self.settings.get('MYSQL_DB', 'scrapy_db'),
+                        minsize=self.settings.getint('MYSQL_POOL_MIN', 2),
+                        maxsize=self.settings.getint('MYSQL_POOL_MAX', 5),
+                        cursorclass=aiomysql.DictCursor,
+                        autocommit=False
+                    )
+                    self._pool_initialized = True
+                    self.logger.debug(f"aiomysql连接池已初始化（表: {self.table_name}）")
+                except Exception as e:
+                    self.logger.error(f"aiomysql连接池初始化失败: {e}")
+                    raise
+
+    async def process_item(self, item, spider) -> Optional[dict]:
+        """处理item方法"""
+        try:
+            await self._init_pool()
+
+            item_dict = dict(item)
+            sql = f"""
+            INSERT INTO `{self.table_name}` 
+            ({', '.join([f'`{k}`' for k in item_dict.keys()])})
+            VALUES ({', '.join(['%s'] * len(item_dict))})
+            """
+
+            async with self.pool.acquire() as conn:
+                async with conn.cursor() as cursor:
+                    try:
+                        await cursor.execute(sql, list(item_dict.values()))
+                        await conn.commit()
+                        self.crawler.stats.inc_value('mysql/insert_success')
+                    except aiomysql.Error as e:
+                        await conn.rollback()
+                        self.crawler.stats.inc_value('mysql/insert_failed')
+                        raise ItemDiscard(f"MySQL错误: {e.args[1]}")
+
+            return item
+
+        except Exception as e:
+            self.logger.error(f"Pipeline处理异常: {e}")
+            raise ItemDiscard(f"处理失败: {e}")
+
+    async def spider_closed(self):
+        """资源清理"""
+        if self.pool:
+            self.pool.close()
+            await self.pool.wait_closed()
+            self.logger.info("aiomysql连接池已释放")

crawlo/pipelines/pipeline_manager.py

@@ -24,18 +24,18 @@ class PipelineManager:
         self._add_methods()
 
     @classmethod
-    def create_instance(cls, *args, **kwargs):
+    def from_crawler(cls, *args, **kwargs):
         o = cls(*args, **kwargs)
         return o
 
     def _add_pipelines(self, pipelines):
         for pipeline in pipelines:
             pipeline_cls = load_class(pipeline)
-            if not hasattr(pipeline_cls, 'create_instance'):
+            if not hasattr(pipeline_cls, 'from_crawler'):
                 raise PipelineInitError(
                     f"Pipeline init failed, must inherit from `BasePipeline` or have a `create_instance` method"
                 )
-            self.pipelines.append(pipeline_cls.create_instance(self.crawler))
+            self.pipelines.append(pipeline_cls.from_crawler(self.crawler))
         if pipelines:
             self.logger.info(f"enabled pipelines: \n {pformat(pipelines)}")
 

crawlo/settings/default_settings.py

@@ -1,8 +1,10 @@
 #!/usr/bin/python
 # -*- coding:UTF-8 -*-
+# 默认项目名称
+import os
 
+PROJECT_NAME = 'crawlo'
 VERSION = 1.0
-
 # 并发数
 CONCURRENCY = 8
 
@@ -25,6 +27,10 @@ IGNORE_HTTP_CODES = [403, 404]
 RETRY_HTTP_CODES = [408, 429, 500, 502, 503, 504, 522, 524]
 # 允许通过的状态码
 ALLOWED_CODES = []
+# 请求优先级设置
+RETRY_PRIORITY = -1
+#
+DEPTH_PRIORITY = 1
 
 STATS_DUMP = True
 # ssl 验证
@@ -37,3 +43,47 @@ LOG_LEVEL = 'DEBUG'
 DOWNLOADER = "crawlo.downloader.aiohttp_downloader.AioHttpDownloader"  # HttpXDownloader
 
 EXTENSIONS = []
+
+# --------------------------------------------------- 公共MySQL配置 -----------------------------------------------------
+MYSQL_HOST = '127.0.0.1'
+MYSQL_PORT = 3306
+MYSQL_USER = 'scrapy_user'
+MYSQL_PASSWORD = 'your_password'
+MYSQL_DB = 'scrapy_data'
+MYSQL_TABLE = 'crawled_data'
+
+# asyncmy专属配置
+MYSQL_POOL_MIN = 5  # 连接池最小连接数
+MYSQL_POOL_MAX = 20  # 连接池最大连接数
+MYSQL_ECHO = False
+
+# 批量插入大小
+MYSQL_BATCH_SIZE = 100
+
+# --------------------------------------------------- MongoDB 基础配置 -----------------------------------------------------
+MONGO_URI = 'mongodb://user:password@host:27017'
+MONGO_DATABASE = 'scrapy_data'
+MONGO_COLLECTION = 'crawled_items'  # 可选，默认使用spider名称
+
+# 连接池优化配置（仅方案二需要）
+MONGO_MAX_POOL_SIZE = 200  # 最大连接数
+MONGO_MIN_POOL_SIZE = 20  # 最小保持连接数
+
+# 启用管道
+PIPELINES = [
+    'crawlo.pipelines.console_pipeline.ConsolePipeline',
+]
+
+# filter
+REQUEST_DIR = '.'
+FILTER_DEBUG = True
+FILTER_CLASS = 'crawlo.filters.memory_filter.MemoryFilter'
+
+# redis filter
+CLEANUP_FP = False
+DECODE_RESPONSES = True
+REDIS_KEY = 'request_fingerprint'
+REDIS_HOST = os.getenv('REDIS_HOST', '127.0.0.1')
+REDIS_PASSWORD = os.getenv('REDIS_PASSWORD', 'oscar&0503')
+REDIS_PORT = os.getenv('REDIS_PORT', 6379)
+REDIS_URL = f'redis://:{REDIS_PASSWORD or ""}@{REDIS_HOST}:{REDIS_PORT}/0'

crawlo/spider/__init__.py

@@ -18,10 +18,10 @@ class Spider(object):
     def start_requests(self):
         if self.start_urls:
             for url in self.start_urls:
-                yield Request(url=url)
+                yield Request(url=url, dont_filter=True)
         else:
             if hasattr(self, 'start_url') and isinstance(getattr(self, 'start_url'), str):
-                yield Request(getattr(self, 'start_url'))
+                yield Request(getattr(self, 'start_url'), dont_filter=True)
 
     def parse(self, response):
         raise NotImplementedError

crawlo/subscriber.py

@@ -3,25 +3,104 @@
 import asyncio
 from collections import defaultdict
 from inspect import iscoroutinefunction
-from typing import Dict, Set, Callable, Coroutine
+from typing import Dict, Set, Callable, Coroutine, Any, TypeAlias, List
 
-from crawlo.exceptions import ReceiverTypeError
+
+class ReceiverTypeError(TypeError):
+    """当订阅的接收者不是一个协程函数时抛出。"""
+    pass
+
+
+ReceiverCoroutine: TypeAlias = Callable[..., Coroutine[Any, Any, Any]]
 
 
 class Subscriber:
+    """
+    一个支持异步协程的发布/订阅（Pub/Sub）模式实现。
+
+    这个类允许你注册（订阅）协程函数来监听特定事件，并在事件发生时
+    以并发的方式异步地通知所有订阅者。
+    """
 
     def __init__(self):
-        self._subscribers: Dict[str, Set[Callable[..., Coroutine]]] = defaultdict(set)
+        """初始化一个空的订阅者字典。"""
+        self._subscribers: Dict[str, Set[ReceiverCoroutine]] = defaultdict(set)
 
-    def subscribe(self, receiver: Callable[..., Coroutine], *, event: str) -> None:
+    def subscribe(self, receiver: ReceiverCoroutine, *, event: str) -> None:
+        """
+        订阅一个事件。
+
+        Args:
+            receiver: 一个协程函数 (例如 async def my_func(...))。
+            event: 要订阅的事件名称。
+
+        Raises:
+            ReceiverTypeError: 如果提供的 `receiver` 不是一个协程函数。
+        """
         if not iscoroutinefunction(receiver):
-            raise ReceiverTypeError(f"{receiver.__qualname__} must be a coroutine function")
+            raise ReceiverTypeError(f"接收者 '{receiver.__qualname__}' 必须是一个协程函数。")
         self._subscribers[event].add(receiver)
 
-    def unsubscribe(self, receiver: Callable[..., Coroutine], *, event: str) -> None:
-        self._subscribers[event].discard(receiver)
+    def unsubscribe(self, receiver: ReceiverCoroutine, *, event: str) -> None:
+        """
+        取消订阅一个事件。
+
+        如果事件或接收者不存在，将静默处理。
+
+        Args:
+            receiver: 要取消订阅的协程函数。
+            event: 事件名称。
+        """
+        if event in self._subscribers:
+            self._subscribers[event].discard(receiver)
+
+    async def notify(self, event: str, *args, **kwargs) -> List[Any]:
+        """
+        异步地、并发地通知所有订阅了该事件的接收者。
+
+        此方法会等待所有订阅者任务完成后再返回，并收集所有结果或异常。
+
+        Args:
+            event: 要触发的事件名称。
+            *args: 传递给接收者的位置参数。
+            **kwargs: 传递给接收者的关键字参数。
+
+        Returns:
+            一个列表，包含每个订阅者任务的返回结果或在执行期间捕获的异常。
+        """
+        receivers = self._subscribers.get(event, set())
+        if not receivers:
+            return []
+
+        tasks = [asyncio.create_task(receiver(*args, **kwargs)) for receiver in receivers]
+
+        # 并发执行所有任务并返回结果列表（包括异常）
+        return await asyncio.gather(*tasks, return_exceptions=True)
 
-    async def notify(self, event: str, *args, **kwargs) -> None:
-        for receiver in self._subscribers[event]:
-            # 不能 await
-            asyncio.create_task(receiver(*args, **kwargs))
+# #!/usr/bin/python
+# # -*- coding:UTF-8 -*-
+# import asyncio
+# from collections import defaultdict
+# from inspect import iscoroutinefunction
+# from typing import Dict, Set, Callable, Coroutine
+#
+# from crawlo.exceptions import ReceiverTypeError
+#
+#
+# class Subscriber:
+#
+#     def __init__(self):
+#         self._subscribers: Dict[str, Set[Callable[..., Coroutine]]] = defaultdict(set)
+#
+#     def subscribe(self, receiver: Callable[..., Coroutine], *, event: str) -> None:
+#         if not iscoroutinefunction(receiver):
+#             raise ReceiverTypeError(f"{receiver.__qualname__} must be a coroutine function")
+#         self._subscribers[event].add(receiver)
+#
+#     def unsubscribe(self, receiver: Callable[..., Coroutine], *, event: str) -> None:
+#         self._subscribers[event].discard(receiver)
+#
+#     async def notify(self, event: str, *args, **kwargs) -> None:
+#         for receiver in self._subscribers[event]:
+#             # 不能 await
+#             asyncio.create_task(receiver(*args, **kwargs))

crawlo/utils/concurrency_manager.py

@@ -0,0 +1,125 @@
+import os
+import platform
+import logging
+from typing import Optional
+
+try:
+    import psutil  # 用于获取系统资源信息的第三方库
+except ImportError:
+    psutil = None  # 如果psutil不可用则设为None
+
+logger = logging.getLogger(__name__)
+
+
+def calculate_optimal_concurrency(user_specified: Optional[int] = None, use_logical_cores: bool = True) -> int:
+    """
+    基于系统资源计算最优并发数，或使用用户指定值
+
+    参数:
+        user_specified: 用户指定的并发数（优先使用）
+        use_logical_cores: 是否使用逻辑CPU核心数（超线程），默认为True
+
+    返回:
+        计算得出的最优并发数
+
+    说明:
+        1. 优先使用用户指定的并发数
+        2. 根据操作系统类型采用不同的计算策略：
+           - Windows: 保守计算，避免内存压力
+           - macOS: 平衡资源使用
+           - Linux: 充分利用服务器资源
+           - 其他系统: 使用合理默认值
+        3. 使用可用内存和CPU核心数进行计算
+        4. 提供psutil不可用时的备用方案
+    """
+    # 优先使用用户指定的并发数
+    if user_specified is not None:
+        logger.info(f"使用用户指定的并发数: {user_specified}")
+        return user_specified
+
+    try:
+        current_os = platform.system()  # 获取当前操作系统类型
+        logger.debug(f"检测到操作系统: {current_os}")
+
+        # 获取CPU核心数（根据参数决定是否使用逻辑核心）
+        cpu_count = psutil.cpu_count(logical=use_logical_cores) or 1 if psutil else os.cpu_count() or 1
+
+        # 根据操作系统类型选择不同的计算方法
+        if current_os == "Windows":
+            concurrency = _get_concurrency_for_windows(cpu_count, use_logical_cores)
+        elif current_os == "Darwin":  # macOS系统
+            concurrency = _get_concurrency_for_macos(cpu_count, use_logical_cores)
+        elif current_os == "Linux":
+            concurrency = _get_concurrency_for_linux(cpu_count, use_logical_cores)
+        else:  # 其他操作系统
+            concurrency = _get_concurrency_default(cpu_count)
+
+        logger.info(f"计算得到最大并发数: {concurrency}")
+        return concurrency
+
+    except Exception as e:
+        logger.warning(f"动态计算并发数失败: {str(e)}，使用默认值50")
+        return 50  # 计算失败时的安全默认值
+
+
+def _get_concurrency_for_windows(cpu_count: int, use_logical_cores: bool) -> int:
+    """Windows系统专用的并发数计算逻辑"""
+    if psutil:
+        # 计算可用内存（GB）
+        available_memory = psutil.virtual_memory().available / (1024 ** 3)
+        # 内存计算：每4GB可用内存分配10个并发
+        mem_based = int((available_memory / 4) * 10)
+        # CPU计算：使用逻辑核心时乘数较大
+        cpu_based = cpu_count * (5 if use_logical_cores else 3)
+        # 取5-100之间的值，选择内存和CPU限制中较小的
+        return max(5, min(100, mem_based, cpu_based))
+    else:
+        # 无psutil时的备用方案
+        return min(50, cpu_count * 5)
+
+
+def _get_concurrency_for_macos(cpu_count: int, use_logical_cores: bool) -> int:
+    """macOS系统专用的并发数计算逻辑"""
+    if psutil:
+        available_memory = psutil.virtual_memory().available / (1024 ** 3)
+        # 内存计算：每3GB可用内存分配10个并发
+        mem_based = int((available_memory / 3) * 10)
+        # CPU计算：使用逻辑核心时乘数较大
+        cpu_based = cpu_count * (6 if use_logical_cores else 4)
+        # 取5-120之间的值
+        return max(5, min(120, mem_based, cpu_based))
+    else:
+        try:
+            # macOS备用方案：使用系统命令获取物理CPU核心数
+            import subprocess
+            output = subprocess.check_output(["sysctl", "hw.physicalcpu"])
+            cpu_count = int(output.split()[1])
+            return min(60, cpu_count * 5)
+        except:
+            return 40  # Mac电脑的合理默认值
+
+
+def _get_concurrency_for_linux(cpu_count: int, use_logical_cores: bool) -> int:
+    """Linux系统专用的并发数计算逻辑（更激进）"""
+    if psutil:
+        available_memory = psutil.virtual_memory().available / (1024 ** 3)
+        # 内存计算：每1.5GB可用内存分配10个并发
+        mem_based = int((available_memory / 1.5) * 10)
+        # CPU计算：服务器环境使用更大的乘数
+        cpu_based = cpu_count * (8 if use_logical_cores else 5)
+        # 取5-200之间的值
+        return max(5, min(200, mem_based, cpu_based))
+    else:
+        try:
+            # Linux备用方案：解析/proc/cpuinfo文件
+            with open("/proc/cpuinfo") as f:
+                cpu_count = f.read().count("processor\t:")
+                if cpu_count > 0:
+                    return min(200, cpu_count * 8)
+        except:
+            return 50  # Linux服务器的合理默认值
+
+
+def _get_concurrency_default(cpu_count: int) -> int:
+    """未知操作系统的默认计算逻辑"""
+    return min(50, cpu_count * 5)  # 保守的默认计算方式