auto-coder 0.1.278__py3-none-any.whl → 0.1.280__py3-none-any.whl

{auto_coder-0.1.278.dist-info → auto_coder-0.1.280.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: auto-coder
-Version: 0.1.278
+Version: 0.1.280
 Summary: AutoCoder: AutoCoder
 Author: allwefantasy
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
@@ -26,7 +26,7 @@ Requires-Dist: tabulate
 Requires-Dist: jupyter-client
 Requires-Dist: prompt-toolkit
 Requires-Dist: tokenizers
-Requires-Dist: byzerllm[saas] >=0.1.169
+Requires-Dist: byzerllm[saas] >=0.1.170
 Requires-Dist: patch
 Requires-Dist: diff-match-patch
 Requires-Dist: GitPython

{auto_coder-0.1.278.dist-info → auto_coder-0.1.280.dist-info}/RECORD

@@ -12,7 +12,7 @@ autocoder/chat_auto_coder_lang.py,sha256=ShOQVOnMA-WlT-fB9OrOer-xQkbcWxJGl-WMPuZ
 autocoder/command_args.py,sha256=9aYJ-AmPxP1sQh6ciw04FWHjSn31f2W9afXFwo8wgx4,30441
 autocoder/lang.py,sha256=U6AjVV8Rs1uLyjFCZ8sT6WWuNUxMBqkXXIOs4S120uk,14511
 autocoder/models.py,sha256=PlG1tKHSHwB57cKLOl5gTl5yTzFUDzCgeHPJU3N9F6Q,9106
-autocoder/version.py,sha256=Q3OPt1PiXvKx5xTUCPxUFZP2zsuajLbxdpiOEm_K2L4,23
+autocoder/version.py,sha256=mNnPow60dgdANkDcEoYTXr9_lpoMQZSEy1-LRu7QFHs,23
 autocoder/agent/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 autocoder/agent/auto_demand_organizer.py,sha256=NWSAEsEk94vT3lGjfo25kKLMwYdPcpy9e-i21txPasQ,6942
 autocoder/agent/auto_filegroup.py,sha256=CW7bqp0FW1GIEMnl-blyAc2UGT7O9Mom0q66ITz1ckM,6635
@@ -107,15 +107,16 @@ autocoder/privacy/__init__.py,sha256=LnIVvGu_K66zCE-yhN_-dPO8R80pQyedCsXJ7wRqQaI
 autocoder/privacy/model_filter.py,sha256=-N9ZvxxDKpxU7hkn-tKv-QHyXjvkCopUaKgvJwTOGQs,3369
 autocoder/pyproject/__init__.py,sha256=ms-A_pocgGv0oZPEW8JAdXi7G-VSVhkQ6CnWFe535Ec,14477
 autocoder/rag/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-autocoder/rag/api_server.py,sha256=dRbhAZVRAOlZ64Cnxf4_rKb4iJwHnrWS9Zr67IVORw0,7288
-autocoder/rag/doc_filter.py,sha256=yEXaBw1XJH57Gtvk4-RFQtd5eawA6SBjzxeRZrIsQew,11623
+autocoder/rag/api_server.py,sha256=xiypCkdbclY0Z3Cmq5FTvtKrfQUV7yKcDaFFUttA2n0,7242
+autocoder/rag/doc_filter.py,sha256=UduVO2mlrngwJICrefjDJTYfdmQ4GcRXrfWDQ7xXksk,14206
 autocoder/rag/document_retriever.py,sha256=5oThtxukGuRFF96o3pHKsk306a8diXbhgSrbqyU2BvM,8894
-autocoder/rag/llm_wrapper.py,sha256=wf56ofQNOaBkLhnoxK9VoVnHWD0gsj0pP8mUBfS92RI,2737
-autocoder/rag/long_context_rag.py,sha256=qFlNmbgQnstCSCb0SxfkMEYtZRr8p6YEc6u0jpve4Q0,32002
+autocoder/rag/lang.py,sha256=TVNx5m7OtBcdfahzI29tMj9m1yrEm32G1c1zc4ZNIPs,3130
+autocoder/rag/llm_wrapper.py,sha256=Ht5GF5yJtrztoliujsZzx_ooWZmHkd5xLZKcGEiicZw,4303
+autocoder/rag/long_context_rag.py,sha256=3CAlf7GM-LgewS5j9XGKvsKSO4MM6M8TTkKxAGzqVY0,39308
 autocoder/rag/rag_config.py,sha256=8LwFcTd8OJWWwi1_WY4IzjqgtT6RyE2j4PjxS5cCTDE,802
 autocoder/rag/rag_entry.py,sha256=6TKtErZ0Us9XSV6HgRKXA6yR3SiZGPHpynOKSaR1wgE,2463
 autocoder/rag/raw_rag.py,sha256=BOr0YGf3umjqXOIDVO1LXQ0bIHx8hzBdiubND2ezyxc,2946
-autocoder/rag/relevant_utils.py,sha256=tgTKGbojCrxuZ7dKbyPh2rCw9TIhwE6ltRxJosaA97U,1267
+autocoder/rag/relevant_utils.py,sha256=tnv_g25DDWYPGT-mpfubIyZv86_g2gPXjM4FPvdeIEE,1739
 autocoder/rag/simple_directory_reader.py,sha256=LkKreCkNdEOoL4fNhc3_hDoyyWTQUte4uqextISRz4U,24485
 autocoder/rag/simple_rag.py,sha256=I902EUqOK1WM0Y2WFd7RzDJYofElvTZNLVCBtX5A9rc,14885
 autocoder/rag/token_checker.py,sha256=jc76x6KWmvVxds6W8juZfQGaoErudc2HenG3sNQfSLs,2819
@@ -159,15 +160,16 @@ autocoder/utils/queue_communicate.py,sha256=buyEzdvab1QA4i2QKbq35rG5v_9x9PWVLWWM
 autocoder/utils/request_event_queue.py,sha256=r3lo5qGsB1dIjzVQ05dnr0z_9Z3zOkBdP1vmRciKdi4,2095
 autocoder/utils/request_queue.py,sha256=nwp6PMtgTCiuwJI24p8OLNZjUiprC-TsefQrhMI-yPE,3889
 autocoder/utils/rest.py,sha256=hLBhr78y-WVnV0oQf9Rxc22EwqF78KINkScvYa1MuYA,6435
+autocoder/utils/stream_thinking.py,sha256=vbDObflBFW53eWEjMTEHf3nyL167_cqpDLh9zRx7Yk8,7015
 autocoder/utils/tests.py,sha256=BqphrwyycGAvs-5mhH8pKtMZdObwhFtJ5MC_ZAOiLq8,1340
 autocoder/utils/thread_utils.py,sha256=tv9fhFZOjI18AxVUJbpe_xjBGMpkqgDcOlz9pnDtNik,8583
 autocoder/utils/types.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 autocoder/utils/auto_coder_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 autocoder/utils/auto_coder_utils/chat_stream_out.py,sha256=lkJ_A-sYU36JMzjFWkk3pR6uos8oZHYt9GPsPe_CPAo,11766
 autocoder/utils/chat_auto_coder_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-auto_coder-0.1.278.dist-info/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
-auto_coder-0.1.278.dist-info/METADATA,sha256=W8ANCnQ9mFraLjuXIUaggoO_jH5Yx-bmm4FNxHLfOvo,2643
-auto_coder-0.1.278.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
-auto_coder-0.1.278.dist-info/entry_points.txt,sha256=0nzHtHH4pNcM7xq4EBA2toS28Qelrvcbrr59GqD_0Ak,350
-auto_coder-0.1.278.dist-info/top_level.txt,sha256=Jqc0_uJSw2GwoFQAa9iJxYns-2mWla-9ok_Y3Gcznjk,10
-auto_coder-0.1.278.dist-info/RECORD,,
+auto_coder-0.1.280.dist-info/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
+auto_coder-0.1.280.dist-info/METADATA,sha256=SDBMvUk6v6YP7RSwlAWHFGfa3LTOUj3fky1Yz0hlFB0,2643
+auto_coder-0.1.280.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
+auto_coder-0.1.280.dist-info/entry_points.txt,sha256=0nzHtHH4pNcM7xq4EBA2toS28Qelrvcbrr59GqD_0Ak,350
+auto_coder-0.1.280.dist-info/top_level.txt,sha256=Jqc0_uJSw2GwoFQAa9iJxYns-2mWla-9ok_Y3Gcznjk,10
+auto_coder-0.1.280.dist-info/RECORD,,

autocoder/rag/api_server.py

@@ -103,9 +103,7 @@ async def create_chat_completion(
     NOTE: Currently we do not support the following features:
         - function_call (Users should implement this by themselves)
         - logit_bias (to be supported by vLLM engine)
-    """
-    # async with async_timeout.timeout(TIMEOUT):
-
+    """    
     generator = await openai_serving_chat.create_chat_completion(body, request)
     if isinstance(generator, ErrorResponse):
         return JSONResponse(

autocoder/rag/doc_filter.py

@@ -1,13 +1,15 @@
 import time
-from typing import List, Dict, Optional
+from typing import List, Dict, Optional, Generator, Tuple
 from loguru import logger
 from concurrent.futures import ThreadPoolExecutor, as_completed
+from autocoder.rag.lang import get_message_with_format_and_newline
 
 from autocoder.rag.relevant_utils import (
     parse_relevance,
     FilterDoc,
     TaskTiming,
-    DocFilterResult
+    DocFilterResult,
+    ProgressUpdate
 )
 
 from autocoder.common import SourceCode, AutoCoderArgs
@@ -49,6 +51,7 @@ def _check_relevance_with_conversation(
     其中， <relevant> 是你认为文档中和问题的相关度，0-10之间的数字，数字越大表示相关度越高。
     """
 
+
 class DocFilter:
     def __init__(
         self,
@@ -73,10 +76,10 @@ class DocFilter:
     ) -> DocFilterResult:
         return self.filter_docs_with_threads(conversations, documents)
 
-    def filter_docs_with_threads(
+    def filter_docs_with_progress(
         self, conversations: List[Dict[str, str]], documents: List[SourceCode]
-    ) -> DocFilterResult:
-
+    ) -> Generator[Tuple[ProgressUpdate, Optional[DocFilterResult]], None, DocFilterResult]:
+        """使用线程过滤文档，同时产生进度更新"""
         start_time = time.time()
         logger.info(f"=== DocFilter Starting ===")
         logger.info(
@@ -93,6 +96,16 @@ class DocFilter:
         relevant_count = 0
         model_name = self.recall_llm.default_model_name or "unknown"
 
+        doc_filter_result = DocFilterResult(
+            docs=[],
+            raw_docs=[],
+            input_tokens_counts=[],
+            generated_tokens_counts=[],
+            durations=[],
+            model_name=model_name
+        )
+        relevant_docs = doc_filter_result.docs
+
         with ThreadPoolExecutor(
             max_workers=self.args.index_filter_workers or 5
         ) as executor:
@@ -141,16 +154,19 @@ class DocFilter:
             logger.info(
                 f"Submitted {submitted_tasks} document filtering tasks to thread pool")
 
+            # 发送初始进度更新
+            yield (ProgressUpdate(
+                phase="doc_filter",
+                completed=0,
+                total=len(documents),
+                relevant_count=0,
+                message=get_message_with_format_and_newline(
+                    "doc_filter_start",
+                    total=len(documents)
+                )
+            ), None)
+
             # 处理完成的任务
-            doc_filter_result = DocFilterResult(
-                docs=[],
-                raw_docs=[],
-                input_tokens_counts=[],
-                generated_tokens_counts=[],
-                durations=[],
-                model_name=model_name
-            )            
-            relevant_docs = doc_filter_result.docs
             for future in as_completed(list(future_to_doc.keys())):
                 try:
                     doc, submit_time = future_to_doc[future]
@@ -194,32 +210,50 @@ class DocFilter:
                         f"\n  - Timing: Duration={task_timing.duration:.2f}s, Processing={task_timing.real_duration:.2f}s, Queue={queue_time:.2f}s"
                         f"\n  - Response: {v}"
                     )
-                    
+
                     if "rag" not in doc.metadata:
                         doc.metadata["rag"] = {}
                     doc.metadata["rag"]["recall"] = {
                         "input_tokens_count": input_tokens_count,
                         "generated_tokens_count": generated_tokens_count,
                         "recall_model": model_name,
-                        "duration": task_timing.real_duration                                                                                                            
+                        "duration": task_timing.real_duration
                     }
-                    
-                    doc_filter_result.input_tokens_counts.append(input_tokens_count)
-                    doc_filter_result.generated_tokens_counts.append(generated_tokens_count)
-                    doc_filter_result.durations.append(task_timing.real_duration)
-                    
+
+                    doc_filter_result.input_tokens_counts.append(
+                        input_tokens_count)
+                    doc_filter_result.generated_tokens_counts.append(
+                        generated_tokens_count)
+                    doc_filter_result.durations.append(
+                        task_timing.real_duration)
+
                     new_filter_doc = FilterDoc(
-                            source_code=doc,
-                            relevance=relevance,
-                            task_timing=task_timing,
-                        )
-                    
+                        source_code=doc,
+                        relevance=relevance,
+                        task_timing=task_timing,
+                    )
+
                     doc_filter_result.raw_docs.append(new_filter_doc)
 
                     if is_relevant:
                         relevant_docs.append(
                             new_filter_doc
                         )
+
+                    # 产生进度更新
+                    yield (ProgressUpdate(
+                        phase="doc_filter",
+                        completed=completed_tasks,
+                        total=len(documents),
+                        relevant_count=relevant_count,
+                        message=get_message_with_format_and_newline(
+                            "doc_filter_progress",
+                            progress_percent=progress_percent,
+                            relevant_count=relevant_count,
+                            total=len(documents)
+                        )
+                    ), None)
+
                 except Exception as exc:
                     try:
                         doc, submit_time = future_to_doc[future]
@@ -236,7 +270,7 @@ class DocFilter:
                             FilterDoc(
                                 source_code=doc,
                                 relevance=None,
-                                task_timing=TaskTiming(),                                
+                                task_timing=TaskTiming(),
                             )
                         )
                     except Exception as e:
@@ -244,6 +278,18 @@ class DocFilter:
                             f"Document filtering error in task tracking: {exc}"
                         )
 
+                    # 报告错误进度
+                    yield (ProgressUpdate(
+                        phase="doc_filter",
+                        completed=completed_tasks,
+                        total=len(documents),
+                        relevant_count=relevant_count,
+                        message=get_message_with_format_and_newline(
+                            "doc_filter_error",
+                            error=str(exc)
+                        )
+                    ), None)
+
         # Sort relevant_docs by relevance score in descending order
         relevant_docs.sort(
             key=lambda x: x.relevance.relevant_score, reverse=True)
@@ -254,7 +300,7 @@ class DocFilter:
             doc.task_timing.real_duration for doc in relevant_docs) / len(relevant_docs) if relevant_docs else 0
         avg_queue_time = sum(doc.task_timing.real_start_time -
                              doc.task_timing.submit_time for doc in relevant_docs) / len(relevant_docs) if relevant_docs else 0
-        
+
         total_input_tokens = sum(doc_filter_result.input_tokens_counts)
         total_generated_tokens = sum(doc_filter_result.generated_tokens_counts)
 
@@ -278,4 +324,33 @@ class DocFilter:
         else:
             logger.warning("No relevant documents found!")
 
-        return doc_filter_result
+        # 返回最终结果
+        yield (ProgressUpdate(
+            phase="doc_filter",
+            completed=len(documents),
+            total=len(documents),
+            relevant_count=relevant_count,
+            message=get_message_with_format_and_newline(
+                "doc_filter_complete",
+                total_time=total_time,
+                relevant_count=relevant_count
+            )
+        ), doc_filter_result)
+
+    def filter_docs_with_threads(
+        self, conversations: List[Dict[str, str]], documents: List[SourceCode]
+    ) -> DocFilterResult:
+        # 保持兼容性的接口
+        for _, result in self.filter_docs_with_progress(conversations, documents):
+            if result is not None:
+                return result
+
+        # 这是一个应急情况，不应该到达这里
+        return DocFilterResult(
+            docs=[],
+            raw_docs=[],
+            input_tokens_counts=[],
+            generated_tokens_counts=[],
+            durations=[],
+            model_name=self.recall_llm.default_model_name or "unknown"
+        )

autocoder/rag/lang.py

@@ -0,0 +1,50 @@
+import locale
+from byzerllm.utils import format_str_jinja2
+
+MESSAGES = {
+    "en": {
+        "rag_error_title": "RAG Error",
+        "rag_error_message": "Failed to generate response: {{error}}",
+        "rag_searching_docs": "Searching documents with {{model}}...",
+        "rag_docs_filter_result": "{{model}} processed {{docs_num}} documents, cost {{filter_time}} seconds, input tokens: {{input_tokens}}, output tokens: {{output_tokens}}",
+        "dynamic_chunking_start": "Dynamic chunking start with {{model}}",
+        "dynamic_chunking_result": "Dynamic chunking result with {{model}}, first round cost {{first_round_time}} seconds, second round cost {{sencond_round_time}} seconds, input tokens: {{input_tokens}}, output tokens: {{output_tokens}}, first round full docs: {{first_round_full_docs}}, second round extracted docs: {{second_round_extracted_docs}}",
+        "send_to_model": "Send to model {{model}} with {{tokens}} tokens",
+        "doc_filter_start": "Document filtering start, total {{total}} documents",
+        "doc_filter_progress": "Document filtering progress: {{progress_percent}}% processed {{relevant_count}}/{{total}} documents",
+        "doc_filter_error": "Document filtering error: {{error}}",
+        "doc_filter_complete": "Document filtering complete, cost {{total_time}} seconds, found {{relevant_count}} relevant documents"
+    },
+    "zh": {
+        "rag_error_title": "RAG 错误",
+        "rag_error_message": "生成响应失败: {{error}}",
+        "rag_searching_docs": "正在使用 {{model}} 搜索文档...",
+        "rag_docs_filter_result": "{{model}} 处理了 {{docs_num}} 个文档, 耗时 {{filter_time}} 秒, 输入 tokens: {{input_tokens}}, 输出 tokens: {{output_tokens}}",
+        "dynamic_chunking_start": "使用 {{model}} 进行动态分块",
+        "dynamic_chunking_result": "使用 {{model}} 进行动态分块, 第一轮耗时 {{first_round_time}} 秒, 第二轮耗时 {{sencond_round_time}} 秒, 输入 tokens: {{input_tokens}}, 输出 tokens: {{output_tokens}}, 第一轮全量文档: {{first_round_full_docs}}, 第二轮提取文档: {{second_round_extracted_docs}}",
+        "send_to_model": "发送给模型 {{model}} 的 tokens 数量预估为 {{tokens}}",
+        "doc_filter_start": "开始过滤文档，共 {{total}} 个文档",
+        "doc_filter_progress": "文档过滤进度：{{progress_percent}}%，处理了 {{relevant_count}}/{{total}} 个文档",
+        "doc_filter_error": "文档过滤错误：{{error}}",
+        "doc_filter_complete": "文档过滤完成，耗时 {{total_time}} 秒，找到 {{relevant_count}} 个相关文档"
+    }
+}
+
+
+def get_system_language():
+    try:
+        return locale.getdefaultlocale()[0][:2]
+    except:
+        return 'en'
+
+
+def get_message(key):
+    lang = get_system_language()
+    return MESSAGES.get(lang, MESSAGES['en']).get(key, MESSAGES['en'][key])
+
+
+def get_message_with_format(msg_key: str, **kwargs):
+    return format_str_jinja2(get_message(msg_key), **kwargs)
+
+def get_message_with_format_and_newline(msg_key: str, **kwargs):
+    return format_str_jinja2(get_message(msg_key), **kwargs) + "\n"

autocoder/rag/llm_wrapper.py

@@ -1,61 +1,91 @@
-from typing import Any, Dict, List, Optional, Union,Callable
-from byzerllm.utils.client.types import (    
+from typing import Any, Dict, List, Optional, Union, Callable
+from byzerllm.utils.client.types import (
     LLMFunctionCallResponse,
-    LLMClassResponse,LLMResponse    
+    LLMClassResponse, LLMResponse
 )
 import pydantic
 from byzerllm import ByzerLLM
 from byzerllm.utils.client import LLMResponse
 from byzerllm.utils.types import SingleOutputMeta
 from autocoder.rag.simple_rag import SimpleRAG
+from autocoder.rag.long_context_rag import LongContextRAG
 from loguru import logger
 from byzerllm.utils.langutil import asyncfy_with_semaphore
 
+
 class LLWrapper:
 
-    def __init__(self,llm:ByzerLLM,rag:SimpleRAG):
+    def __init__(self, llm: ByzerLLM, rag: Union[SimpleRAG, LongContextRAG]):
         self.llm = llm
         self.rag = rag
 
     def chat_oai(self,
                  conversations,
-                 tools:List[Union[Callable,str]]=[], 
-                 tool_choice:Optional[Union[Callable,str]]=None,
-                 execute_tool:bool=False,  
-                 impl_func:Optional[Callable]=None,
-                 execute_impl_func:bool=False,
-                 impl_func_params:Optional[Dict[str,Any]]=None,
-                 func_params:Optional[Dict[str,Any]]=None,
-                 response_class:Optional[Union[pydantic.BaseModel,str]] = None, 
-                 response_after_chat:Optional[Union[pydantic.BaseModel,str]] = False,
-                 enable_default_sys_message:bool=True,                 
-                 model:Optional[str] = None,
-                 role_mapping=None,llm_config:Dict[str,Any]={}
-                 )->Union[List[LLMResponse],List[LLMFunctionCallResponse],List[LLMClassResponse]]: 
-        res,contexts = self.rag.stream_chat_oai(conversations,llm_config=llm_config)
-        s = "".join(res)        
-        return [LLMResponse(output=s,metadata={},input="")]
-
-    def stream_chat_oai(self,conversations, 
-                        model:Optional[str]=None, 
-                        role_mapping=None,
-                        delta_mode=False,
-                        llm_config:Dict[str,Any]={}): 
-        res,contexts = self.rag.stream_chat_oai(conversations,llm_config=llm_config)        
-        for t in res:                        
-            yield (t,SingleOutputMeta(0,0))
-    
-
-    async def async_stream_chat_oai(self,conversations, 
-                        model:Optional[str]=None, 
+                 tools: List[Union[Callable, str]] = [],
+                 tool_choice: Optional[Union[Callable, str]] = None,
+                 execute_tool: bool = False,
+                 impl_func: Optional[Callable] = None,
+                 execute_impl_func: bool = False,
+                 impl_func_params: Optional[Dict[str, Any]] = None,
+                 func_params: Optional[Dict[str, Any]] = None,
+                 response_class: Optional[Union[pydantic.BaseModel, str]] = None,
+                 response_after_chat: Optional[Union[pydantic.BaseModel, str]] = False,
+                 enable_default_sys_message: bool = True,
+                 model: Optional[str] = None,
+                 role_mapping=None,
+                 llm_config: Dict[str, Any] = {},
+                 only_return_prompt: bool = False,
+                 extra_request_params: Dict[str, Any] = {}
+                 ) -> Union[List[LLMResponse], List[LLMFunctionCallResponse], List[LLMClassResponse]]:
+        res, contexts = self.rag.stream_chat_oai(
+            conversations, llm_config=llm_config, extra_request_params=extra_request_params)
+        metadata = {"request_id":""}
+        output = ""
+        for chunk in res:
+            output += chunk[0]
+            metadata["input_tokens_count"] = chunk[1].input_tokens_count
+            metadata["generated_tokens_count"] = chunk[1].generated_tokens_count
+            metadata["reasoning_content"] = chunk[1].reasoning_content
+            metadata["finish_reason"] = chunk[1].finish_reason
+            metadata["first_token_time"] = chunk[1].first_token_time
+            
+        return [LLMResponse(output=output, metadata=metadata, input="")]
+
+    def stream_chat_oai(self, conversations,
+                        model: Optional[str] = None,
                         role_mapping=None,
                         delta_mode=False,
-                        llm_config:Dict[str,Any]={}): 
-        res,contexts = await asyncfy_with_semaphore(lambda: self.rag.stream_chat_oai(conversations,llm_config=llm_config))()
+                        llm_config: Dict[str, Any] = {},
+                        extra_request_params: Dict[str, Any] = {}
+                        ):
+        res, contexts = self.rag.stream_chat_oai(
+                conversations, llm_config=llm_config, extra_request_params=extra_request_params)
+
+        if isinstance(res, tuple):
+            for (t, metadata) in res:
+                yield (t, SingleOutputMeta(
+                    input_tokens_count=metadata.get("input_tokens_count", 0),
+                    generated_tokens_count=metadata.get(
+                        "generated_tokens_count", 0),
+                    reasoning_content=metadata.get("reasoning_content", ""),
+                    finish_reason=metadata.get("finish_reason", "stop"),
+                    first_token_time=metadata.get("first_token_time", 0)
+                ))
+        else:
+            for t in res:
+                yield (t, SingleOutputMeta(0, 0))
+
+    async def async_stream_chat_oai(self, conversations,
+                                    model: Optional[str] = None,
+                                    role_mapping=None,
+                                    delta_mode=False,
+                                    llm_config: Dict[str, Any] = {},
+                                    extra_request_params: Dict[str, Any] = {}
+                                    ):
+        res, contexts = await asyncfy_with_semaphore(lambda: self.rag.stream_chat_oai(conversations, llm_config=llm_config, extra_request_params=extra_request_params))()
         # res,contexts = await self.llm.async_stream_chat_oai(conversations,llm_config=llm_config)
-        for t in res:                                    
-            yield (t,SingleOutputMeta(0,0))
-                         
+        for t in res:
+            yield t
 
-    def __getattr__(self, name):        
-        return getattr(self.llm, name)
+    def __getattr__(self, name):
+        return getattr(self.llm, name)