llm-engine-kitty 0.1.0.dev0__py3-none-any.whl

llm_engine/engine.py

@@ -0,0 +1,771 @@
+# llm_engine/engine.py
+
+import asyncio
+import httpx
+import json
+import time
+
+from abc import ABC, abstractmethod
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from tqdm import tqdm
+from typing import Any, Dict, List, Optional
+
+import kitty_logger
+
+from .model_config import ModelConfig, ModelConfigRegistry
+from .schemas import (
+    InferenceParameters,
+    InferenceRequest,
+    InferenceRequestResult,
+    PreparedRequest,
+    Message,
+    MessageRole,
+    ModelOutput,
+    ChunkChoice,
+    ChunkDelta,
+    ChatCompletionChunk,
+    ChatCompletionChoice,
+    ChatCompletionResponse,
+)
+from .utils import gen_unique_id
+
+logger = kitty_logger.getLogger(__name__)
+
+
+class BaseEngine(ABC):
+
+    __slots__ = ()
+
+    pass
+
+
+class SyncEngine(BaseEngine):
+
+    __slots__ = ()
+
+    @abstractmethod
+    def inference(self, request: InferenceRequest) -> InferenceRequestResult:
+        pass
+
+    def batch_inference(self, requests: List[InferenceRequest], max_concurrency: int = 1) -> List[InferenceRequestResult]:
+
+        class_name = self.__class__.__name__
+
+        # fmt: off
+        logger.warning(
+            f"引擎 '{class_name}' 未实现高效的 'batch_inference' 方法，将回退至串行模式。"
+            f"指定的并发数 (max_concurrency={max_concurrency}) 不会生效，"
+            f"正在串行处理 {len(requests)} 个请求，这可能会影响执行效率。"
+        )
+        # fmt: on
+
+        return [self.inference(req) for req in requests]
+
+    def infer(self, query: str) -> str:
+
+        inference_result: InferenceRequestResult = self.inference(InferenceRequest(messages=[Message(role=MessageRole.USER, content=query)]))
+
+        if inference_result.success is True and inference_result.model_output is not None:
+
+            return inference_result.model_output.content
+
+        else:
+
+            return ""
+
+    def batch_infer(self, queries: List[str], max_concurrency: int = 8) -> List[str]:
+
+        inference_requests: List[InferenceRequest] = [InferenceRequest(messages=[Message(role=MessageRole.USER, content=query)]) for query in queries]
+
+        inference_results: List[InferenceRequestResult] = self.batch_inference(inference_requests, max_concurrency=max_concurrency)
+
+        return [result.model_output.content if result.success is True and result.model_output is not None else "" for result in inference_results]
+
+
+class AsyncEngine(BaseEngine):
+
+    __slots__ = ()
+
+    pass
+
+
+class CoroutineEngine(BaseEngine):
+
+    __slots__ = ()
+
+    @abstractmethod
+    async def inference(self, request: InferenceRequest) -> InferenceRequestResult:
+        pass
+
+    async def batch_inference(self, requests: List[InferenceRequest], max_concurrency: int = 1) -> List[InferenceRequestResult]:
+
+        class_name = self.__class__.__name__
+
+        # fmt: off
+        logger.warning(
+            f"引擎 '{class_name}' 未实现高效的 'batch_inference' 方法，将回退至串行模式。"
+            f"指定的并发数 (max_concurrency={max_concurrency}) 不会生效，"
+            f"正在串行处理 {len(requests)} 个请求，这可能会影响执行效率。"
+        )
+        # fmt: on
+
+        return [await self.inference(req) for req in requests]
+
+    async def infer(self, query: str) -> str:
+
+        try:
+            inference_result: InferenceRequestResult = await self.inference(InferenceRequest(messages=[Message(role=MessageRole.USER, content=query)]))
+        except Exception as e:
+            logger.error(f"infer 抛出未捕获异常: {e}")
+            return ""
+
+        if inference_result.success is True and inference_result.model_output is not None:
+            return inference_result.model_output.content
+        else:
+            return ""
+
+    async def batch_infer(self, queries: List[str], max_concurrency: int = 8) -> List[str]:
+
+        inference_requests: List[InferenceRequest] = [InferenceRequest(messages=[Message(role=MessageRole.USER, content=query)]) for query in queries]
+
+        inference_results: List[InferenceRequestResult] = await self.batch_inference(inference_requests, max_concurrency=max_concurrency)
+
+        return [result.model_output.content if result.success is True and result.model_output is not None else "" for result in inference_results]
+
+
+class SimpleEngine(SyncEngine):
+
+    __slots__ = (
+        "model_registry",
+        "default_model_name",
+        "default_model",
+        "default_api_key",
+        "default_inference_parameters",
+        "extra_headers",
+        "extra_payload",
+        "stream",
+    )
+
+    def __init__(
+        self,
+        model_registry: ModelConfigRegistry,
+        default_model: str,
+        default_api_key: Optional[str] = None,
+        default_inference_parameters: Optional[InferenceParameters] = None,
+        extra_headers: Optional[Dict[str, str]] = None,
+        extra_payload: Optional[Dict[str, Any]] = None,
+        stream: bool = False,
+    ) -> None:
+        self.model_registry: ModelConfigRegistry = model_registry
+        self.default_model_name: str = default_model
+        self.default_model: ModelConfig = self.model_registry.get(default_model)
+        self.default_api_key: Optional[str] = default_api_key
+        self.default_inference_parameters: Optional[InferenceParameters] = default_inference_parameters
+        self.extra_headers: Dict[str, str] = extra_headers if extra_headers is not None else {}
+        self.extra_payload: Dict[str, Any] = extra_payload if extra_payload is not None else {}
+        self.stream: bool = stream
+
+    def get_model_api_key(self, inference_request: InferenceRequest, model: ModelConfig) -> str:
+
+        if inference_request.api_key is not None:
+            return inference_request.api_key
+        elif model.api_key is not None:
+            return model.api_key
+        elif self.default_api_key is not None:
+            return self.default_api_key
+        else:
+            logger.warning(f"没有找到可用的 api_key, 将传递空 api_key")
+            return ""
+
+    def build_request_headers(self, inference_request: InferenceRequest, model: ModelConfig) -> Dict[str, str]:
+
+        headers: Dict[str, str] = {
+            "Authorization": "Bearer " + self.get_model_api_key(inference_request, model),
+            "Content-Type": "application/json",
+        }
+
+        headers.update(self.extra_headers)
+        headers.update(model.extra_headers)
+        headers.update(inference_request.extra_headers)
+
+        return headers
+
+    def build_request_payload(self, inference_request: InferenceRequest, model: ModelConfig) -> Dict[str, Any]:
+        payload: Dict[str, Any] = {
+            "model": model.model_id,
+            "stream": inference_request.stream if inference_request.stream is not None else self.stream,
+            "messages": [msg.to_dict() for msg in inference_request.messages],
+        }
+
+        if self.default_inference_parameters:
+            payload.update(self.default_inference_parameters.to_dict())
+        if model.default_inference_parameters:
+            payload.update(model.default_inference_parameters.to_dict())
+        if inference_request.inference_parameters:
+            payload.update(inference_request.inference_parameters.to_dict())
+
+        payload.update(self.extra_payload)
+        payload.update(model.extra_payload)
+        payload.update(inference_request.extra_payload)
+
+        return payload
+
+    def build_request(self, request: InferenceRequest) -> PreparedRequest:
+        """核心组装逻辑：将业务请求转换为底层 HTTP 请求参数"""
+
+        model_name = request.model_name or self.default_model_name
+        model = self.model_registry.get(model_name)
+
+        return PreparedRequest(
+            model_name=model_name,
+            url=model.get_url(),
+            headers=self.build_request_headers(request, model),
+            payload=self.build_request_payload(request, model),
+        )
+
+    def send_request(
+        self,
+        url: str,
+        headers: Dict[str, str],
+        payload: Dict[str, Any],
+        stream: bool,
+        max_retries: int = 5,
+        base_delay: int = 2,
+    ) -> ModelOutput:
+
+        for attempt in range(max_retries):
+            is_last_attempt: bool = attempt == (max_retries - 1)
+            try:
+                with httpx.Client(timeout=httpx.Timeout(timeout=10.0, read=3600.0)) as http_client:
+
+                    with http_client.stream("POST", url, json=payload, headers=headers) as http_response:
+
+                        if http_response.status_code != 200:
+
+                            if is_last_attempt:
+                                response_body = http_response.read().decode("utf-8", errors="replace")
+                                logger.error(f"HTTP {http_response.status_code}，已达最大重试次数，响应内容: {response_body[:500]}")
+                                raise httpx.HTTPStatusError(f"HTTP {http_response.status_code}", request=http_response.request, response=http_response)
+
+                            wait_time = self._get_wait_time(http_response.status_code, attempt, base_delay)
+                            response_body = http_response.read().decode("utf-8", errors="replace")
+                            logger.warning(f"HTTP {http_response.status_code}, 第 {attempt+1} 次重试，等待 {wait_time}s... 响应内容: {response_body[:500]}")
+                            time.sleep(wait_time)
+                            continue
+
+                        if not stream:
+                            full_body = http_response.read().decode("utf-8")
+                            resp: ChatCompletionResponse = ChatCompletionResponse.model_validate_json(full_body)
+                            choice: ChatCompletionChoice = resp.choices[0]
+                            if choice.finish_reason and choice.finish_reason != "stop":
+                                logger.warning(f"finish_reason='{choice.finish_reason}' (非 stop)，content_len={len(choice.message.content or '')}, usage={resp.usage.model_dump() if resp.usage else None}")
+                            return ModelOutput(
+                                role=choice.message.role,
+                                content=choice.message.content,
+                                reasoning=choice.message.reasoning_content,
+                                finish_reason=choice.finish_reason,
+                                usage=resp.usage.model_dump() if resp.usage else None,
+                            )
+
+                        else:
+                            model_reasoning: str = ""
+                            model_response: str = ""
+                            role: Optional[str] = None
+                            usage: Optional[Dict] = None
+                            finish_reason: Optional[str] = None
+
+                            for line in http_response.iter_lines():
+
+                                if not line or line.strip() == "":
+                                    continue
+
+                                logger.debug(f"line: '{line}'")
+
+                                if not line.startswith("data: "):
+                                    logger.error(f"数据行没有以'data: '开头，将跳过。line: {line}")
+                                    continue
+
+                                raw_data = line.removeprefix("data: ").strip()
+                                if raw_data == "[DONE]":
+                                    logger.debug("收到结束标志'[DONE]', 不再解析后续包")
+                                    break
+
+                                try:
+                                    chunk: ChatCompletionChunk = ChatCompletionChunk.model_validate_json(raw_data)
+                                except Exception as e:
+                                    logger.error(f"解析数据包失败: {e}, 原始数据: {raw_data}")
+                                    continue
+
+                                if chunk.usage:
+                                    if usage is not None:
+                                        logger.warning(f"收到多个含有usage的包，后收到的usage将会覆盖先前的token用量信息。")
+                                    usage = chunk.usage.model_dump()
+
+                                if not chunk.choices:
+                                    continue
+
+                                chunk_choice: ChunkChoice = chunk.choices[0]
+                                chunk_delta: ChunkDelta = chunk_choice.delta
+
+                                if chunk_delta.content:
+                                    model_response += chunk_delta.content
+                                if chunk_delta.reasoning_content:
+                                    model_reasoning += chunk_delta.reasoning_content
+
+                                if chunk_delta.role:
+                                    if role is not None:
+                                        logger.warning(f"收到多个含有role的包，后收到的role将会覆盖先前的role信息。")
+                                    role = chunk_delta.role
+
+                                if chunk_choice.finish_reason:
+                                    if finish_reason is not None:
+                                        logger.warning(f"收到多个含有finish_reason的包，后收到的finish_reason将会覆盖先前的finish_reason信息。")
+                                    finish_reason = chunk_choice.finish_reason
+
+                            if finish_reason and finish_reason != "stop":
+                                logger.warning(f"finish_reason='{finish_reason}' (非 stop)，content_len={len(model_response)}, usage={usage}")
+                            return ModelOutput(
+                                role=role,
+                                content=model_response,
+                                reasoning=model_reasoning,
+                                finish_reason=finish_reason,
+                                usage=usage,
+                            )
+
+            except (httpx.NetworkError, httpx.TimeoutException, httpx.HTTPStatusError) as e:
+                if is_last_attempt:
+                    logger.error(f"达到最大重试次数，最后一次错误: {e}")
+                    raise
+                if not isinstance(e, httpx.HTTPStatusError):
+                    wait_time = base_delay**attempt
+                    logger.warning(f"网络异常: {e}, 等待 {wait_time}s 后重试...")
+                    time.sleep(wait_time)
+
+        raise RuntimeError("Unexpected end of retry loop")
+
+    def _get_wait_time(self, status_code: int, attempt: int, base_delay: int) -> float:
+        """集中管理不同错误的等待时间算法"""
+        if status_code == 429:
+            # 频率限制通常需要更长的等待
+            return 10 * (attempt + 1)
+        elif status_code >= 500:
+            # 服务器内部错误使用指数退避
+            return float(base_delay**attempt)
+        return 5.0  # 其他错误默认值
+
+    def inference(self, request: InferenceRequest) -> InferenceRequestResult:
+        prepared_request: PreparedRequest = self.build_request(request)
+        model = self.model_registry.get(request.model_name or self.default_model_name)
+
+        # URL 在 build_request 时一次性选定，整个重试循环复用同一 URL。
+        # 如需重试时重新选 URL（例如多节点场景），需将 URL 选取移至 send_request 的 attempt 循环内。
+        start_time: float = time.time()
+        try:
+            model_output: ModelOutput = self.send_request(
+                url=prepared_request.url,
+                headers=prepared_request.headers,
+                payload=prepared_request.payload,
+                stream=prepared_request.payload["stream"],
+            )
+        finally:
+            model.release_url(prepared_request.url)
+
+        return InferenceRequestResult(
+            success=True,
+            task_id="",
+            request=request,
+            model_output=model_output,
+            duration=time.time() - start_time,
+        )
+
+    def batch_inference(
+        self,
+        requests: List[InferenceRequest],
+        max_concurrency: int = 8,
+        output_file: Optional[str] = None,
+        silent_mode: bool = False,
+    ) -> List[InferenceRequestResult]:
+
+        result_dict: Dict[int, InferenceRequestResult] = {}
+
+        with ThreadPoolExecutor(max_workers=max_concurrency) as executor:
+            future_to_idx = {executor.submit(self.inference, req): i for i, req in enumerate(requests)}
+
+            futures_iterator = as_completed(future_to_idx)
+            if not silent_mode:
+                futures_iterator = tqdm(
+                    futures_iterator,
+                    total=len(requests),
+                    desc=f"Batch: {self.default_model_name}",
+                )
+
+            f = open(output_file, "w", encoding="utf-8") if output_file else None
+
+            try:
+                for future in futures_iterator:
+                    idx = future_to_idx[future]
+                    try:
+                        result: InferenceRequestResult = future.result()
+                        result_dict[idx] = result
+
+                        if f:
+                            f.write(result.model_dump_json() + "\n")
+                            f.flush()
+
+                    except Exception as e:
+                        logger.error(f"Request #{idx} 抛出未捕获异常: {e}")
+                        error_res = self._get_error_result(requests[idx], e)
+                        result_dict[idx] = error_res
+                        if f:
+                            f.write(error_res.model_dump_json() + "\n")
+            finally:
+                if f:
+                    f.close()
+
+        return [result_dict[i] for i in range(len(requests))]
+
+    def _get_error_result(self, request: InferenceRequest, error: Exception) -> InferenceRequestResult:
+        """内部工具：当请求彻底失败时返回标准化错误结构"""
+        return InferenceRequestResult(
+            success=False,
+            task_id="error",
+            request=request,
+            error_message=f"{type(error).__name__}: {str(error)}",
+        )
+
+    def mock_request(self, query: Optional[str] = None, request: Optional[InferenceRequest] = None) -> PreparedRequest:
+        """
+        模拟构建请求，返回最终的 URL、Headers 和 Payload。
+        用于验证配置组装、API Key 传递和参数覆盖是否符合预期。
+        """
+
+        if request is None:
+            if query is None:
+                query = f"请求文本: [{gen_unique_id()}]"
+            request = InferenceRequest(messages=[Message(role=MessageRole.USER, content=query)])
+
+        prepared_request: PreparedRequest = self.build_request(request)
+
+        return prepared_request
+
+
+class SimpleCoroutineEngine(CoroutineEngine):
+
+    __slots__ = (
+        "model_registry",
+        "default_model_name",
+        "default_model",
+        "default_api_key",
+        "default_inference_parameters",
+        "extra_headers",
+        "extra_payload",
+        "stream",
+    )
+
+    def __init__(
+        self,
+        model_registry: ModelConfigRegistry,
+        default_model: str,
+        default_api_key: Optional[str] = None,
+        default_inference_parameters: Optional[InferenceParameters] = None,
+        extra_headers: Optional[Dict[str, str]] = None,
+        extra_payload: Optional[Dict[str, Any]] = None,
+        stream: bool = False,
+    ) -> None:
+        self.model_registry: ModelConfigRegistry = model_registry
+        self.default_model_name: str = default_model
+        self.default_model: ModelConfig = self.model_registry.get(default_model)
+        self.default_api_key: Optional[str] = default_api_key
+        self.default_inference_parameters: Optional[InferenceParameters] = default_inference_parameters
+        self.extra_headers: Dict[str, str] = extra_headers if extra_headers is not None else {}
+        self.extra_payload: Dict[str, Any] = extra_payload if extra_payload is not None else {}
+        self.stream: bool = stream
+
+    def get_model_api_key(self, inference_request: InferenceRequest, model: ModelConfig) -> str:
+
+        if inference_request.api_key is not None:
+            return inference_request.api_key
+        elif model.api_key is not None:
+            return model.api_key
+        elif self.default_api_key is not None:
+            return self.default_api_key
+        else:
+            logger.warning(f"没有找到可用的 api_key, 将传递空 api_key")
+            return ""
+
+    def build_request_headers(self, inference_request: InferenceRequest, model: ModelConfig) -> Dict[str, str]:
+
+        headers: Dict[str, str] = {
+            "Authorization": "Bearer " + self.get_model_api_key(inference_request, model),
+            "Content-Type": "application/json",
+        }
+
+        headers.update(self.extra_headers)
+        headers.update(model.extra_headers)
+        headers.update(inference_request.extra_headers)
+
+        return headers
+
+    def build_request_payload(self, inference_request: InferenceRequest, model: ModelConfig) -> Dict[str, Any]:
+        payload: Dict[str, Any] = {
+            "model": model.model_id,
+            "stream": inference_request.stream if inference_request.stream is not None else self.stream,
+            "messages": [msg.to_dict() for msg in inference_request.messages],
+        }
+
+        if self.default_inference_parameters:
+            payload.update(self.default_inference_parameters.to_dict())
+        if model.default_inference_parameters:
+            payload.update(model.default_inference_parameters.to_dict())
+        if inference_request.inference_parameters:
+            payload.update(inference_request.inference_parameters.to_dict())
+
+        payload.update(self.extra_payload)
+        payload.update(model.extra_payload)
+        payload.update(inference_request.extra_payload)
+
+        return payload
+
+    def build_request(self, request: InferenceRequest) -> PreparedRequest:
+        """核心组装逻辑：将业务请求转换为底层 HTTP 请求参数"""
+
+        model_name = request.model_name or self.default_model_name
+        model = self.model_registry.get(model_name)
+
+        return PreparedRequest(
+            model_name=model_name,
+            url=model.get_url(),
+            headers=self.build_request_headers(request, model),
+            payload=self.build_request_payload(request, model),
+        )
+
+    def _get_wait_time(self, status_code: int, attempt: int, base_delay: int) -> float:
+        """集中管理不同错误的等待时间算法"""
+        if status_code == 429:
+            # return 10 * (attempt + 1)
+            return 1.0
+        elif status_code >= 500:
+            # return float(base_delay**attempt)
+            return 1.0
+        return 1.0
+
+    def _get_error_result(self, request: InferenceRequest, error: Exception) -> InferenceRequestResult:
+        """内部工具：当请求彻底失败时返回标准化错误结构"""
+        return InferenceRequestResult(
+            success=False,
+            task_id="error",
+            request=request,
+            error_message=f"{type(error).__name__}: {str(error)}",
+        )
+
+    def mock_request(self, query: Optional[str] = None, request: Optional[InferenceRequest] = None) -> PreparedRequest:
+        """
+        模拟构建请求，返回最终的 URL、Headers 和 Payload。
+        用于验证配置组装、API Key 传递和参数覆盖是否符合预期。
+        """
+
+        if request is None:
+            if query is None:
+                query = f"请求文本: [{gen_unique_id()}]"
+            request = InferenceRequest(messages=[Message(role=MessageRole.USER, content=query)])
+
+        prepared_request: PreparedRequest = self.build_request(request)
+
+        return prepared_request
+
+    async def send_request(
+        self,
+        url: str,
+        headers: Dict[str, str],
+        payload: Dict[str, Any],
+        stream: bool,
+        max_retries: int = 256,
+        base_delay: int = 2,
+    ) -> ModelOutput:
+
+        async with httpx.AsyncClient(timeout=httpx.Timeout(timeout=10.0, read=1800.0)) as http_client:
+            for attempt in range(max_retries):
+                is_last_attempt: bool = attempt == (max_retries - 1)
+                try:
+                    async with http_client.stream("POST", url, json=payload, headers=headers) as http_response:
+
+                        if http_response.status_code != 200:
+
+                            if is_last_attempt:
+                                response_body = (await http_response.aread()).decode("utf-8", errors="replace")
+                                logger.error(f"HTTP {http_response.status_code}，已达最大重试次数，响应内容: {response_body[:500]}")
+                                raise httpx.HTTPStatusError(f"HTTP {http_response.status_code}", request=http_response.request, response=http_response)
+
+                            wait_time = self._get_wait_time(http_response.status_code, attempt, base_delay)
+                            response_body = (await http_response.aread()).decode("utf-8", errors="replace")
+                            logger.warning(f"HTTP {http_response.status_code}, 第 {attempt+1} 次重试，等待 {wait_time}s... 响应内容: {response_body[:500]}")
+                            await asyncio.sleep(wait_time)
+                            continue
+
+                        if not stream:
+                            full_body = (await http_response.aread()).decode("utf-8")
+                            resp: ChatCompletionResponse = ChatCompletionResponse.model_validate_json(full_body)
+                            if not resp.choices:
+                                if is_last_attempt:
+                                    logger.error(f"resp.choices 为空，已达最大重试次数，疑似被风控拦截。响应内容: {full_body[:500]}")
+                                    raise RuntimeError(f"empty choices after {max_retries} attempts, body: {full_body[:500]}")
+                                # wait_time = base_delay**attempt
+                                wait_time = base_delay
+                                logger.warning(f"resp.choices 为空，疑似被风控拦截，第 {attempt+1} 次重试，等待 {wait_time}s... 响应内容: {full_body[:500]}")
+                                await asyncio.sleep(wait_time)
+                                continue
+                            choice: ChatCompletionChoice = resp.choices[0]
+                            if choice.finish_reason and choice.finish_reason != "stop":
+                                logger.warning(f"finish_reason='{choice.finish_reason}' (非 stop)，content_len={len(choice.message.content or '')}, usage={resp.usage.model_dump() if resp.usage else None}")
+                            return ModelOutput(
+                                role=choice.message.role,
+                                content=choice.message.content,
+                                reasoning=choice.message.reasoning_content,
+                                finish_reason=choice.finish_reason,
+                                usage=resp.usage.model_dump() if resp.usage else None,
+                            )
+
+                        else:
+                            model_reasoning: str = ""
+                            model_response: str = ""
+                            role: Optional[str] = None
+                            usage: Optional[Dict] = None
+                            finish_reason: Optional[str] = None
+
+                            async for line in http_response.aiter_lines():
+
+                                if not line or line.strip() == "":
+                                    continue
+
+                                logger.debug(f"line: '{line}'")
+
+                                if not line.startswith("data: "):
+                                    logger.error(f"数据行没有以'data: '开头，将跳过。line: {line}")
+                                    continue
+
+                                raw_data = line.removeprefix("data: ").strip()
+                                if raw_data == "[DONE]":
+                                    logger.debug("收到结束标志'[DONE]', 不再解析后续包")
+                                    break
+
+                                try:
+                                    chunk: ChatCompletionChunk = ChatCompletionChunk.model_validate_json(raw_data)
+                                except Exception as e:
+                                    logger.error(f"解析数据包失败: {e}, 原始数据: {raw_data}")
+                                    continue
+
+                                if chunk.usage:
+                                    if usage is not None:
+                                        logger.warning(f"收到多个含有usage的包，后收到的usage将会覆盖先前的token用量信息。")
+                                    usage = chunk.usage.model_dump()
+
+                                if not chunk.choices:
+                                    continue
+
+                                chunk_choice: ChunkChoice = chunk.choices[0]
+                                chunk_delta: ChunkDelta = chunk_choice.delta
+
+                                if chunk_delta.content:
+                                    model_response += chunk_delta.content
+                                if chunk_delta.reasoning_content:
+                                    model_reasoning += chunk_delta.reasoning_content
+
+                                if chunk_delta.role:
+                                    if role is not None:
+                                        logger.warning(f"收到多个含有role的包，后收到的role将会覆盖先前的role信息。")
+                                    role = chunk_delta.role
+
+                                if chunk_choice.finish_reason:
+                                    if finish_reason is not None:
+                                        logger.warning(f"收到多个含有finish_reason的包，后收到的finish_reason将会覆盖先前的finish_reason信息。")
+                                    finish_reason = chunk_choice.finish_reason
+
+                            if finish_reason and finish_reason != "stop":
+                                logger.warning(f"finish_reason='{finish_reason}' (非 stop)，content_len={len(model_response)}, usage={usage}")
+                            return ModelOutput(
+                                role=role,
+                                content=model_response,
+                                reasoning=model_reasoning,
+                                finish_reason=finish_reason,
+                                usage=usage,
+                            )
+
+                except (httpx.NetworkError, httpx.TimeoutException, httpx.HTTPStatusError) as e:
+                    if is_last_attempt:
+                        logger.error(f"达到最大重试次数，最后一次错误: {e}")
+                        raise
+                    if not isinstance(e, httpx.HTTPStatusError):
+                        wait_time = base_delay**attempt
+                        logger.warning(f"网络异常: {type(e).__name__}: {e!r}, 等待 {wait_time}s 后重试...")
+                        await asyncio.sleep(wait_time)
+
+            raise RuntimeError("Unexpected end of retry loop")
+
+    async def inference(self, request: InferenceRequest) -> InferenceRequestResult:
+        prepared_request: PreparedRequest = self.build_request(request)
+        model = self.model_registry.get(request.model_name or self.default_model_name)
+
+        # URL 在 build_request 时一次性选定，整个重试循环复用同一 URL。
+        # 如需重试时重新选 URL（例如多节点场景），需将 URL 选取移至 send_request 的 attempt 循环内。
+        start_time: float = time.time()
+        try:
+            model_output: ModelOutput = await self.send_request(
+                url=prepared_request.url,
+                headers=prepared_request.headers,
+                payload=prepared_request.payload,
+                stream=prepared_request.payload["stream"],
+            )
+        finally:
+            model.release_url(prepared_request.url)
+
+        return InferenceRequestResult(
+            success=True,
+            task_id="",
+            request=request,
+            model_output=model_output,
+            duration=time.time() - start_time,
+        )
+
+    async def batch_inference(
+        self,
+        requests: List[InferenceRequest],
+        max_concurrency: int = 8,
+        output_file: Optional[str] = None,
+        silent_mode: bool = False,
+    ) -> List[InferenceRequestResult]:
+
+        semaphore = asyncio.Semaphore(max_concurrency)
+        result_dict: Dict[int, InferenceRequestResult] = {}
+
+        async def run_one(idx: int, req: InferenceRequest):
+            async with semaphore:
+                try:
+                    result = await self.inference(req)
+                    result_dict[idx] = result
+                    return idx, result
+                except Exception as e:
+                    logger.error(f"Request #{idx} 抛出未捕获异常: {e}")
+                    err = self._get_error_result(req, e)
+                    result_dict[idx] = err
+                    return idx, err
+
+        tasks = [asyncio.create_task(run_one(i, req)) for i, req in enumerate(requests)]
+
+        pbar = tqdm(total=len(requests), desc=f"Batch: {self.default_model_name}") if not silent_mode else None
+        f = open(output_file, "w", encoding="utf-8") if output_file else None
+
+        try:
+            for coro in asyncio.as_completed(tasks):
+                idx, result = await coro
+                if f:
+                    f.write(result.model_dump_json() + "\n")
+                    f.flush()
+                if pbar:
+                    pbar.update(1)
+        finally:
+            if pbar:
+                pbar.close()
+            if f:
+                f.close()
+
+        return [result_dict[i] for i in range(len(requests))]