auto-coder 0.1.271__py3-none-any.whl → 0.1.272__py3-none-any.whl

{auto_coder-0.1.271.dist-info → auto_coder-0.1.272.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: auto-coder
-Version: 0.1.271
+Version: 0.1.272
 Summary: AutoCoder: AutoCoder
 Author: allwefantasy
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
@@ -26,7 +26,7 @@ Requires-Dist: tabulate
 Requires-Dist: jupyter-client
 Requires-Dist: prompt-toolkit
 Requires-Dist: tokenizers
-Requires-Dist: byzerllm[saas] >=0.1.168
+Requires-Dist: byzerllm[saas] >=0.1.169
 Requires-Dist: patch
 Requires-Dist: diff-match-patch
 Requires-Dist: GitPython

{auto_coder-0.1.271.dist-info → auto_coder-0.1.272.dist-info}/RECORD

@@ -4,7 +4,7 @@ autocoder/auto_coder_lang.py,sha256=Rtupq6N3_HT7JRhDKdgCBcwRaiAnyCOR_Gsp4jUomrI,
 autocoder/auto_coder_rag.py,sha256=nwgsXO2-scssWStjX3S910tDp-OZXZRddSYrpyC4Nq0,29021
 autocoder/auto_coder_rag_client_mcp.py,sha256=QRxUbjc6A8UmDMQ8lXgZkjgqtq3lgKYeatJbDY6rSo0,6270
 autocoder/auto_coder_rag_mcp.py,sha256=-RrjNwFaS2e5v8XDIrKR-zlUNUE8UBaeOtojffBrvJo,8521
-autocoder/auto_coder_runner.py,sha256=JGyLT231zu2cZgnf98Y-W5wDQ0xORZJef1GfL7SZQto,100725
+autocoder/auto_coder_runner.py,sha256=1SGmVP_X10vNJujdaDc3QuiTeHa4wDIpLp41poUJW7w,100773
 autocoder/auto_coder_server.py,sha256=6YQweNEKUrGAZ3yPvw8_qlNZJYLVSVUXGrn1K6udLts,20413
 autocoder/benchmark.py,sha256=Ypomkdzd1T3GE6dRICY3Hj547dZ6_inqJbBJIp5QMco,4423
 autocoder/chat_auto_coder.py,sha256=skujpqYqf4EvBLQARJELxj7Xwq9KQj2FGefUAiutF7c,16711
@@ -12,7 +12,7 @@ autocoder/chat_auto_coder_lang.py,sha256=ShOQVOnMA-WlT-fB9OrOer-xQkbcWxJGl-WMPuZ
 autocoder/command_args.py,sha256=9aYJ-AmPxP1sQh6ciw04FWHjSn31f2W9afXFwo8wgx4,30441
 autocoder/lang.py,sha256=U6AjVV8Rs1uLyjFCZ8sT6WWuNUxMBqkXXIOs4S120uk,14511
 autocoder/models.py,sha256=PlG1tKHSHwB57cKLOl5gTl5yTzFUDzCgeHPJU3N9F6Q,9106
-autocoder/version.py,sha256=ZgM5wgwos-e45Kw9MewMvDsa_W_7THMB-EhDfus_ybw,23
+autocoder/version.py,sha256=81VehHvNKUJ_hEbhkGJ2_z23uverZt6jaYYz6duFiJI,23
 autocoder/agent/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 autocoder/agent/auto_demand_organizer.py,sha256=NWSAEsEk94vT3lGjfo25kKLMwYdPcpy9e-i21txPasQ,6942
 autocoder/agent/auto_filegroup.py,sha256=CW7bqp0FW1GIEMnl-blyAc2UGT7O9Mom0q66ITz1ckM,6635
@@ -34,8 +34,8 @@ autocoder/common/__init__.py,sha256=d1AmAAYhm4b17dVhAJFwV3Vv2r1lUhMejpjr32Poyg8,
 autocoder/common/anything2images.py,sha256=0ILBbWzY02M-CiWB-vzuomb_J1hVdxRcenAfIrAXq9M,25283
 autocoder/common/anything2img.py,sha256=4TREa-sOA-iargieUy7MpyCYVUE-9Mmq0wJtwomPqnE,7662
 autocoder/common/audio.py,sha256=Kn9nWKQddWnUrAz0a_ZUgjcu4VUU_IcZBigT7n3N3qc,7439
-autocoder/common/auto_coder_lang.py,sha256=WZm5XQaYeECiTiFiHjXWHRZKeV0ux2v64hwZqY8ZDDU,31709
-autocoder/common/auto_configure.py,sha256=L0wjvR-6wFNpP3c9ZxwDOunTtGrzwzjUwsS6BUnJ3W8,12470
+autocoder/common/auto_coder_lang.py,sha256=ONjY2R5BPc3wuSQOKkxvfMP1nID2dtKJ5s6XCZ0Jzjo,31733
+autocoder/common/auto_configure.py,sha256=c1k6ttDyvc9_5amiCTt4PUq9mRu1IeG560Q3kIRy-Uk,12474
 autocoder/common/buildin_tokenizer.py,sha256=L7d5t39ZFvUd6EoMPXUhYK1toD0FHlRH1jtjKRGokWU,1236
 autocoder/common/chunk_validation.py,sha256=BrR_ZWavW8IANuueEE7hS8NFAwEvm8TX34WnPx_1hs8,3030
 autocoder/common/cleaner.py,sha256=NU72i8C6o9m0vXExab7nao5bstBUsfJFcj11cXa9l4U,1089
@@ -102,25 +102,26 @@ autocoder/index/symbols_utils.py,sha256=_EP7E_qWXxluAxq3FGZLlLfdrfwx3FmxCdulI8VG
 autocoder/index/types.py,sha256=a2s_KV5FJlq7jqA2ELSo9E1sjuLwDB-JJYMhSpzBAhU,596
 autocoder/index/filter/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 autocoder/index/filter/normal_filter.py,sha256=MI-8xdXCrniaxYCHVTLkq5tafvcUiauD3LN0b3ymRXI,8361
-autocoder/index/filter/quick_filter.py,sha256=D6B8zz9TGAFPOp-OBm7UwuAEtg1DYQ8jEq4tHgZocbo,22572
+autocoder/index/filter/quick_filter.py,sha256=5_P-HKK9Wim3ysHUECsegCS-z2PAKYOtfq8FrUY8K24,22777
 autocoder/privacy/__init__.py,sha256=LnIVvGu_K66zCE-yhN_-dPO8R80pQyedCsXJ7wRqQaI,72
 autocoder/privacy/model_filter.py,sha256=-N9ZvxxDKpxU7hkn-tKv-QHyXjvkCopUaKgvJwTOGQs,3369
 autocoder/pyproject/__init__.py,sha256=ms-A_pocgGv0oZPEW8JAdXi7G-VSVhkQ6CnWFe535Ec,14477
 autocoder/rag/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 autocoder/rag/api_server.py,sha256=dRbhAZVRAOlZ64Cnxf4_rKb4iJwHnrWS9Zr67IVORw0,7288
-autocoder/rag/doc_filter.py,sha256=ZCixxUXNBbz6UiGbgXvbDWdn5moLac3HnZEphpasTDc,6579
+autocoder/rag/doc_filter.py,sha256=yEXaBw1XJH57Gtvk4-RFQtd5eawA6SBjzxeRZrIsQew,11623
 autocoder/rag/document_retriever.py,sha256=5oThtxukGuRFF96o3pHKsk306a8diXbhgSrbqyU2BvM,8894
-autocoder/rag/llm_wrapper.py,sha256=sbDxCANiZyWb_ocqNgqu2oy3c2t8orPNRGleEs-Uwl8,2649
-autocoder/rag/long_context_rag.py,sha256=SwU6UhTVjdKQu1VOaj07VGHFMG-mRTwRuOH1GwwFygo,24680
+autocoder/rag/llm_wrapper.py,sha256=wf56ofQNOaBkLhnoxK9VoVnHWD0gsj0pP8mUBfS92RI,2737
+autocoder/rag/long_context_rag.py,sha256=CzPC-ct6PVIKBkHsKon4s92YXmi8jZOlGgcquOwWQlI,31802
 autocoder/rag/rag_config.py,sha256=8LwFcTd8OJWWwi1_WY4IzjqgtT6RyE2j4PjxS5cCTDE,802
 autocoder/rag/rag_entry.py,sha256=6TKtErZ0Us9XSV6HgRKXA6yR3SiZGPHpynOKSaR1wgE,2463
 autocoder/rag/raw_rag.py,sha256=BOr0YGf3umjqXOIDVO1LXQ0bIHx8hzBdiubND2ezyxc,2946
-autocoder/rag/relevant_utils.py,sha256=OGfp98OXG4jr3jNmtHIeXGPF8mOlIbTnolPIVTZzYZU,929
+autocoder/rag/relevant_utils.py,sha256=tgTKGbojCrxuZ7dKbyPh2rCw9TIhwE6ltRxJosaA97U,1267
 autocoder/rag/simple_directory_reader.py,sha256=LkKreCkNdEOoL4fNhc3_hDoyyWTQUte4uqextISRz4U,24485
 autocoder/rag/simple_rag.py,sha256=I902EUqOK1WM0Y2WFd7RzDJYofElvTZNLVCBtX5A9rc,14885
 autocoder/rag/token_checker.py,sha256=jc76x6KWmvVxds6W8juZfQGaoErudc2HenG3sNQfSLs,2819
 autocoder/rag/token_counter.py,sha256=C-Lwc4oIjJpZDEqp9WLHGOe6hb4yhrdJpMtkrtp_1qc,2125
-autocoder/rag/token_limiter.py,sha256=5JznJ-h1bG5x2Plx5H0MGkaAHo2GiD7CDIJmeZWWk_8,11015
+autocoder/rag/token_limiter.py,sha256=3VgJF4may3ESyATmBIiOe05oc3VsidJcJTJ5EhoSvH8,18854
+autocoder/rag/token_limiter_utils.py,sha256=FATNEXBnFJy8IK3PWNt1pspIv8wuTgy3F_ACNvqoc4I,404
 autocoder/rag/types.py,sha256=WPgLpUTwbk0BAikyDOc0NOEwV5k73myF38zWdOuYdC4,2499
 autocoder/rag/utils.py,sha256=x5L8gskxUUGkVEP5K-0C-iYntE84asuDXOCqdzdNUoA,4956
 autocoder/rag/variable_holder.py,sha256=PFvBjFcR7-fNDD4Vcsc8CpH2Te057vcpwJMxtrfUgKI,75
@@ -164,9 +165,9 @@ autocoder/utils/types.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 autocoder/utils/auto_coder_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 autocoder/utils/auto_coder_utils/chat_stream_out.py,sha256=lkJ_A-sYU36JMzjFWkk3pR6uos8oZHYt9GPsPe_CPAo,11766
 autocoder/utils/chat_auto_coder_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-auto_coder-0.1.271.dist-info/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
-auto_coder-0.1.271.dist-info/METADATA,sha256=eqiFY5uN-v89P1pH5YtnxnghXFABqjgmm4pPOqRUOpE,2643
-auto_coder-0.1.271.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
-auto_coder-0.1.271.dist-info/entry_points.txt,sha256=0nzHtHH4pNcM7xq4EBA2toS28Qelrvcbrr59GqD_0Ak,350
-auto_coder-0.1.271.dist-info/top_level.txt,sha256=Jqc0_uJSw2GwoFQAa9iJxYns-2mWla-9ok_Y3Gcznjk,10
-auto_coder-0.1.271.dist-info/RECORD,,
+auto_coder-0.1.272.dist-info/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
+auto_coder-0.1.272.dist-info/METADATA,sha256=_71Qvh-jvuaAUr81tcWZyyjmsY8-_bt50WO-jSimtTQ,2643
+auto_coder-0.1.272.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
+auto_coder-0.1.272.dist-info/entry_points.txt,sha256=0nzHtHH4pNcM7xq4EBA2toS28Qelrvcbrr59GqD_0Ak,350
+auto_coder-0.1.272.dist-info/top_level.txt,sha256=Jqc0_uJSw2GwoFQAa9iJxYns-2mWla-9ok_Y3Gcznjk,10
+auto_coder-0.1.272.dist-info/RECORD,,

autocoder/auto_coder_runner.py

@@ -2133,7 +2133,7 @@ def manage_models(query: str):
         # Check duplication
         if any(m["name"] == data_dict["name"] for m in models_data):
             printer.print_in_terminal("models_add_model_exists", style="yellow", name=data_dict["name"])
-            result_manager.add_result(content=printer.get_message_from_key("models_add_model_exists",name=data_dict["name"]),meta={
+            result_manager.add_result(content=printer.get_message_from_key_with_format("models_add_model_exists",name=data_dict["name"]),meta={
                 "action": "models",
                 "input": {
                     "query": query
@@ -2177,7 +2177,7 @@ def manage_models(query: str):
         filtered_models = [m for m in models_data if m["name"] != name]
         if len(filtered_models) == len(models_data):
             printer.print_in_terminal("models_add_model_remove", style="yellow", name=name)
-            result_manager.add_result(content=printer.get_message_from_key("models_add_model_remove",name=name),meta={
+            result_manager.add_result(content=printer.get_message_from_key_with_format("models_add_model_remove",name=name),meta={
                 "action": "models",
                 "input": {
                     "query": query
@@ -2186,7 +2186,7 @@ def manage_models(query: str):
             return
         models_module.save_models(filtered_models)
         printer.print_in_terminal("models_add_model_removed", style="green", name=name)
-        result_manager.add_result(content=printer.get_message_from_key("models_add_model_removed",name=name),meta={ 
+        result_manager.add_result(content=printer.get_message_from_key_with_format("models_add_model_removed",name=name),meta={ 
             "action": "models",
             "input": {
                 "query": query
@@ -2194,7 +2194,7 @@ def manage_models(query: str):
         })
     else:
         printer.print_in_terminal("models_unknown_subcmd", style="yellow", subcmd=subcmd)
-        result_manager.add_result(content=printer.get_message_from_key("models_unknown_subcmd",subcmd=subcmd),meta={ 
+        result_manager.add_result(content=printer.get_message_from_key_with_format("models_unknown_subcmd",subcmd=subcmd),meta={ 
             "action": "models",
             "input": {
                 "query": query

autocoder/common/auto_coder_lang.py

@@ -130,7 +130,7 @@ MESSAGES = {
         "quick_filter_too_long": "⚠️ index file is too large ({{ tokens_len }}/{{ max_tokens }}). The query will be split into {{ split_size }} chunks.",
         "quick_filter_tokens_len": "📊 Current index size: {{ tokens_len }} tokens",
         "estimated_chat_input_tokens": "Estimated chat input tokens: {{ estimated_input_tokens }}",
-        "estimated_input_tokens_in_generate": "Estimated input tokens in generate ({{ generate_mode }}): {{ estimated_input_tokens }}",        
+        "estimated_input_tokens_in_generate": "Estimated input tokens in generate ({{ generate_mode }}): {{ estimated_input_tokens_in_generate }}",        
         "model_has_access_restrictions": "{{model_name}} has access restrictions, cannot use the current function",
         "auto_command_not_found": "Auto command not found: {{command}}. Please check your input and try again.",
         "auto_command_failed": "Auto command failed: {{error}}. Please check your input and try again.",
@@ -319,7 +319,7 @@ MESSAGES = {
         "quick_filter_title": "{{ model_name }} 正在分析如何筛选上下文...",
         "quick_filter_failed": "❌ 快速过滤器失败: {{ error }}. ",
         "estimated_chat_input_tokens": "对话输入token预估为: {{ estimated_input_tokens }}",
-        "estimated_input_tokens_in_generate": "生成代码({{ generate_mode }})预计输入token数: {{ estimated_input_tokens }}",        
+        "estimated_input_tokens_in_generate": "生成代码({{ generate_mode }})预计输入token数: {{ estimated_input_tokens_in_generate }}",        
         "model_has_access_restrictions": "{{model_name}} 有访问限制，无法使用当前功能",
         "auto_command_not_found": "未找到自动命令: {{command}}。请检查您的输入并重试。",
         "auto_command_failed": "自动命令执行失败: {{error}}。请检查您的输入并重试。",

autocoder/common/auto_configure.py

@@ -154,7 +154,7 @@ def config_readme() -> str:
     项目类型通常为如下三种选择：
     1. ts
     2. py
-    3. 代码文件后缀名列表（比如.java,.py,.go,.js,.ts），多个按逗号分割
+    3. 代码文件后缀名列表（比如.java,.py,.go,.js,.ts），多个按逗号分割    
 
     推荐使用 3 选项，因为项目类型通常为多种后缀名混合。
     """

autocoder/index/filter/quick_filter.py

@@ -287,6 +287,7 @@ class QuickFilter():
         当用户提一个需求的时候，我们要找到两种类型的源码文件：
         1. 根据需求需要被修改的文件，我们叫 edited_files
         2. 为了能够完成修改这些文件，还需要的一些额外参考文件, 我们叫 reference_files
+        3. 因为修改了 edited_files 文件，可能有一些依赖 edited_files 的文件也需要被修改，我们叫 dependent_files
 
         现在，给定下面的索引文件：
 
@@ -315,9 +316,9 @@ class QuickFilter():
         ```
 
         特别注意:    
-        1. 如果用户的query里有 @文件 或者 @@符号，那么被@的文件或者@@的符号必须要返回。
-        2. 根据需求以及根据 @文件 或者 @@符号 找到的文件，猜测需要被修改的edited_files文件，然后尝试通过索引文件诸如导入语句等信息找到这些文件依赖的其他文件得到 reference_files。
-        3. file_list 里的文件序号，按被 @ 或者 @@ 文件，edited_files文件，reference_files文件的顺序排列。注意，reference_files 你要根据需求来猜测是否需要，过滤掉不相关的，避免返回文件数过多。
+        1. 如果用户的query里有 @文件 或者 @@符号，并且他们在索引文件中，那么被@的文件或者@@的符号必须要返回。
+        2. 根据需求以及根据 @文件 或者 @@符号 找到的文件，猜测需要被修改的edited_files文件，然后尝试通过索引文件诸如导入语句等信息找到这些文件依赖的其他文件得到 reference_files,dependent_files。
+        3. file_list 里的文件序号，按被 @ 或者 @@ 文件，edited_files文件，reference_files,dependent_files文件的顺序排列。注意，reference_files 你要根据需求来猜测是否需要，过滤掉不相关的，避免返回文件数过多。
         4. 如果 query 里是一段历史对话，那么对话里的内容提及的文件路径必须要返回。        
         5. 如果用户需求为空，则直接返回空列表即可。        
         6. 返回的 json格式数据不允许有注释

autocoder/rag/doc_filter.py

@@ -5,8 +5,9 @@ from concurrent.futures import ThreadPoolExecutor, as_completed
 
 from autocoder.rag.relevant_utils import (
     parse_relevance,
-    FilterDoc,    
+    FilterDoc,
     TaskTiming,
+    DocFilterResult
 )
 
 from autocoder.common import SourceCode, AutoCoderArgs
@@ -48,7 +49,6 @@ def _check_relevance_with_conversation(
     其中， <relevant> 是你认为文档中和问题的相关度，0-10之间的数字，数字越大表示相关度越高。
     """
 
-
 class DocFilter:
     def __init__(
         self,
@@ -62,40 +62,57 @@ class DocFilter:
             self.recall_llm = self.llm.get_sub_client("recall_model")
         else:
             self.recall_llm = self.llm
-        
+
         self.args = args
         self.relevant_score = self.args.rag_doc_filter_relevance
         self.on_ray = on_ray
-        self.path = path        
+        self.path = path
 
     def filter_docs(
         self, conversations: List[Dict[str, str]], documents: List[SourceCode]
-    ) -> List[FilterDoc]:
-        return self.filter_docs_with_threads(conversations, documents)            
+    ) -> DocFilterResult:
+        return self.filter_docs_with_threads(conversations, documents)
 
     def filter_docs_with_threads(
         self, conversations: List[Dict[str, str]], documents: List[SourceCode]
-    ) -> List[FilterDoc]:
-        
+    ) -> DocFilterResult:
+
+        start_time = time.time()
+        logger.info(f"=== DocFilter Starting ===")
+        logger.info(
+            f"Configuration: relevance_threshold={self.relevant_score}, thread_workers={self.args.index_filter_workers or 5}")
+
         rag_manager = RagConfigManager(path=self.path)
         rag_config = rag_manager.load_config()
-        documents = list(documents)   
-        logger.info(f"Filtering {len(documents)} documents....")
+
+        documents = list(documents)
+        logger.info(f"Filtering {len(documents)} documents...")
+
+        submitted_tasks = 0
+        completed_tasks = 0
+        relevant_count = 0
+        model_name = self.recall_llm.default_model_name or "unknown"
+
         with ThreadPoolExecutor(
             max_workers=self.args.index_filter_workers or 5
         ) as executor:
             future_to_doc = {}
+
+            # 提交所有任务
             for doc in documents:
                 submit_time = time.time()
+                submitted_tasks += 1
 
                 def _run(conversations, docs):
                     submit_time_1 = time.time()
+                    meta = None
                     try:
                         llm = self.recall_llm
+                        meta_holder = byzerllm.MetaHolder()
 
                         v = (
                             _check_relevance_with_conversation.with_llm(
-                                llm)
+                                llm).with_meta(meta_holder)
                             .options({"llm_config": {"max_length": 10}})
                             .run(
                                 conversations=conversations,
@@ -103,14 +120,16 @@ class DocFilter:
                                 filter_config=rag_config.filter_config,
                             )
                         )
+
+                        meta = meta_holder.get_meta_model()
                     except Exception as e:
                         logger.error(
                             f"Error in _check_relevance_with_conversation: {str(e)}"
                         )
-                        return (None, submit_time_1, time.time())
+                        return (None, submit_time_1, time.time(), meta)
 
                     end_time_2 = time.time()
-                    return (v, submit_time_1, end_time_2)
+                    return (v, submit_time_1, end_time_2, meta)
 
                 m = executor.submit(
                     _run,
@@ -119,57 +138,144 @@ class DocFilter:
                 )
                 future_to_doc[m] = (doc, submit_time)
 
-        relevant_docs = []
-        for future in as_completed(list(future_to_doc.keys())):
-            try:
-                doc, submit_time = future_to_doc[future]
-                end_time = time.time()
-                v, submit_time_1, end_time_2 = future.result()
-                task_timing = TaskTiming(
-                    submit_time=submit_time,
-                    end_time=end_time,
-                    duration=end_time - submit_time,
-                    real_start_time=submit_time_1,
-                    real_end_time=end_time_2,
-                    real_duration=end_time_2 - submit_time_1,
-                )                
-
-                relevance = parse_relevance(v)
-                logger.info(
-                    f"Document filtering progress:\n"
-                    f"  - File: {doc.module_name}\n"
-                    f"  - Relevance: {'Relevant' if relevance and relevance.is_relevant else 'Not Relevant'}\n"
-                    f"  - Score: {relevance.relevant_score if relevance else 'N/A'}\n"
-                    f"  - Score Threshold: {self.relevant_score}\n"
-                    f"  - Raw Response: {v}\n"
-                    f"  - Timing:\n"
-                    f"    * Total Duration: {task_timing.duration:.2f}s\n"
-                    f"    * Real Duration: {task_timing.real_duration:.2f}s\n"
-                    f"    * Queue Time: {(task_timing.real_start_time - task_timing.submit_time):.2f}s"
-                )
-                if (
-                    relevance
-                    # and relevance.is_relevant
-                    and relevance.relevant_score >= self.relevant_score
-                ):
-                    relevant_docs.append(
-                        FilterDoc(
+            logger.info(
+                f"Submitted {submitted_tasks} document filtering tasks to thread pool")
+
+            # 处理完成的任务
+            doc_filter_result = DocFilterResult(
+                docs=[],
+                raw_docs=[],
+                input_tokens_counts=[],
+                generated_tokens_counts=[],
+                durations=[],
+                model_name=model_name
+            )            
+            relevant_docs = doc_filter_result.docs
+            for future in as_completed(list(future_to_doc.keys())):
+                try:
+                    doc, submit_time = future_to_doc[future]
+                    end_time = time.time()
+                    completed_tasks += 1
+                    progress_percent = (completed_tasks / len(documents)) * 100
+
+                    v, submit_time_1, end_time_2, meta = future.result()
+                    task_timing = TaskTiming(
+                        submit_time=submit_time,
+                        end_time=end_time,
+                        duration=end_time - submit_time,
+                        real_start_time=submit_time_1,
+                        real_end_time=end_time_2,
+                        real_duration=end_time_2 - submit_time_1,
+                    )
+
+                    relevance = parse_relevance(v)
+                    is_relevant = relevance and relevance.relevant_score >= self.relevant_score
+
+                    if is_relevant:
+                        relevant_count += 1
+                        status_text = f"RELEVANT (Score: {relevance.relevant_score:.1f})"
+                    else:
+                        score_text = f"{relevance.relevant_score:.1f}" if relevance else "N/A"
+                        status_text = f"NOT RELEVANT (Score: {score_text})"
+
+                    queue_time = task_timing.real_start_time - task_timing.submit_time
+
+                    input_tokens_count = meta.input_tokens_count if meta else 0
+                    generated_tokens_count = meta.generated_tokens_count if meta else 0
+
+                    logger.info(
+                        f"Document filtering [{progress_percent:.1f}%] - {completed_tasks}/{len(documents)}:"
+                        f"\n  - File: {doc.module_name}"
+                        f"\n  - Status: {status_text}"
+                        f"\n  - Model: {model_name}"
+                        f"\n  - Threshold: {self.relevant_score}"
+                        f"\n  - Input tokens: {input_tokens_count}"
+                        f"\n  - Generated tokens: {generated_tokens_count}"
+                        f"\n  - Timing: Duration={task_timing.duration:.2f}s, Processing={task_timing.real_duration:.2f}s, Queue={queue_time:.2f}s"
+                        f"\n  - Response: {v}"
+                    )
+                    
+                    if "rag" not in doc.metadata:
+                        doc.metadata["rag"] = {}
+                    doc.metadata["rag"]["recall"] = {
+                        "input_tokens_count": input_tokens_count,
+                        "generated_tokens_count": generated_tokens_count,
+                        "recall_model": model_name,
+                        "duration": task_timing.real_duration                                                                                                            
+                    }
+                    
+                    doc_filter_result.input_tokens_counts.append(input_tokens_count)
+                    doc_filter_result.generated_tokens_counts.append(generated_tokens_count)
+                    doc_filter_result.durations.append(task_timing.real_duration)
+                    
+                    new_filter_doc = FilterDoc(
                             source_code=doc,
                             relevance=relevance,
                             task_timing=task_timing,
                         )
-                    )
-            except Exception as exc:
-                try:
-                    doc, submit_time = future_to_doc[future]
-                    logger.error(
-                        f"Filtering document generated an exception (doc: {doc.module_name}): {exc}")
-                except Exception as e:
-                    logger.error(
-                        f"Filtering document generated an exception: {exc}")
+                    
+                    doc_filter_result.raw_docs.append(new_filter_doc)
+
+                    if is_relevant:
+                        relevant_docs.append(
+                            new_filter_doc
+                        )
+                except Exception as exc:
+                    try:
+                        doc, submit_time = future_to_doc[future]
+                        completed_tasks += 1
+                        progress_percent = (
+                            completed_tasks / len(documents)) * 100
+                        logger.error(
+                            f"Document filtering [{progress_percent:.1f}%] - {completed_tasks}/{len(documents)}:"
+                            f"\n  - File: {doc.module_name}"
+                            f"\n  - Error: {exc}"
+                            f"\n  - Duration: {time.time() - submit_time:.2f}s"
+                        )
+                        doc_filter_result.raw_docs.append(
+                            FilterDoc(
+                                source_code=doc,
+                                relevance=None,
+                                task_timing=TaskTiming(),                                
+                            )
+                        )
+                    except Exception as e:
+                        logger.error(
+                            f"Document filtering error in task tracking: {exc}"
+                        )
 
         # Sort relevant_docs by relevance score in descending order
         relevant_docs.sort(
             key=lambda x: x.relevance.relevant_score, reverse=True)
-        return relevant_docs
-    
+
+        total_time = time.time() - start_time
+
+        avg_processing_time = sum(
+            doc.task_timing.real_duration for doc in relevant_docs) / len(relevant_docs) if relevant_docs else 0
+        avg_queue_time = sum(doc.task_timing.real_start_time -
+                             doc.task_timing.submit_time for doc in relevant_docs) / len(relevant_docs) if relevant_docs else 0
+        
+        total_input_tokens = sum(doc_filter_result.input_tokens_counts)
+        total_generated_tokens = sum(doc_filter_result.generated_tokens_counts)
+
+        logger.info(
+            f"=== DocFilter Complete ==="
+            f"\n  * Total time: {total_time:.2f}s"
+            f"\n  * Documents processed: {completed_tasks}/{len(documents)}"
+            f"\n  * Relevant documents: {relevant_count} (threshold: {self.relevant_score})"
+            f"\n  * Average processing time: {avg_processing_time:.2f}s"
+            f"\n  * Average queue time: {avg_queue_time:.2f}s"
+            f"\n  * Total input tokens: {total_input_tokens}"
+            f"\n  * Total generated tokens: {total_generated_tokens}"
+        )
+
+        if relevant_docs:
+            logger.info(
+                f"Top 5 relevant documents:"
+                + "".join([f"\n  * {doc.source_code.module_name} (Score: {doc.relevance.relevant_score:.1f})"
+                          for doc in relevant_docs[:5]])
+            )
+        else:
+            logger.warning("No relevant documents found!")
+
+        return doc_filter_result

autocoder/rag/llm_wrapper.py

@@ -44,13 +44,15 @@ class LLWrapper:
         res,contexts = self.rag.stream_chat_oai(conversations,llm_config=llm_config)        
         for t in res:                        
             yield (t,SingleOutputMeta(0,0))
+    
 
     async def async_stream_chat_oai(self,conversations, 
                         model:Optional[str]=None, 
                         role_mapping=None,
                         delta_mode=False,
                         llm_config:Dict[str,Any]={}): 
-        res,contexts = await asyncfy_with_semaphore(lambda: self.rag.stream_chat_oai(conversations,llm_config=llm_config))()                
+        res,contexts = await asyncfy_with_semaphore(lambda: self.rag.stream_chat_oai(conversations,llm_config=llm_config))()
+        # res,contexts = await self.llm.async_stream_chat_oai(conversations,llm_config=llm_config)
         for t in res:                                    
             yield (t,SingleOutputMeta(0,0))
                          

autocoder/rag/long_context_rag.py

@@ -31,6 +31,8 @@ from tokenizers import Tokenizer
 from autocoder.rag.variable_holder import VariableHolder
 from importlib.metadata import version
 from autocoder.rag.stream_event import event_writer
+from autocoder.rag.relevant_utils import DocFilterResult
+from pydantic import BaseModel
 
 try:        
     from autocoder_pro.rag.llm_compute import LLMComputeEngine
@@ -42,6 +44,24 @@ except ImportError:
     LLMComputeEngine = None
 
 
+class RecallStat(BaseModel):
+    total_input_tokens: int
+    total_generated_tokens: int
+    model_name: str = "unknown"
+class ChunkStat(BaseModel):
+    total_input_tokens: int
+    total_generated_tokens: int            
+    model_name: str = "unknown"
+class AnswerStat(BaseModel):
+    total_input_tokens: int
+    total_generated_tokens: int
+    model_name: str = "unknown"
+
+class RAGStat(BaseModel):
+    recall_stat: RecallStat
+    chunk_stat: ChunkStat
+    answer_stat: AnswerStat
+
 class LongContextRAG:
     def __init__(
         self,
@@ -305,7 +325,7 @@ class LongContextRAG:
                 url = ",".join(contexts)
                 return [SourceCode(module_name=f"RAG:{url}", source_code="".join(v))]
 
-    def _filter_docs(self, conversations: List[Dict[str, str]]) -> List[FilterDoc]:
+    def _filter_docs(self, conversations: List[Dict[str, str]]) -> DocFilterResult:
         query = conversations[-1]["content"]
         documents = self._retrieve_documents(options={"query":query})
         return self.doc_filter.filter_docs(
@@ -439,7 +459,32 @@ class LongContextRAG:
 
             logger.info(f"Query: {query} only_contexts: {only_contexts}")
             start_time = time.time()
-            relevant_docs: List[FilterDoc] = self._filter_docs(conversations)            
+
+            rag_stat = RAGStat(
+                recall_stat=RecallStat(
+                    total_input_tokens=0,
+                    total_generated_tokens=0,
+                    model_name=self.llm.default_model_name,
+                ),
+                chunk_stat=ChunkStat(
+                    total_input_tokens=0,
+                    total_generated_tokens=0,
+                    model_name=self.llm.default_model_name,
+                ),
+                answer_stat=AnswerStat(
+                    total_input_tokens=0,
+                    total_generated_tokens=0,
+                    model_name=self.llm.default_model_name,
+                ),
+            )
+
+            doc_filter_result = self._filter_docs(conversations)            
+
+            rag_stat.recall_stat.total_input_tokens += sum(doc_filter_result.input_tokens_counts)
+            rag_stat.recall_stat.total_generated_tokens += sum(doc_filter_result.generated_tokens_counts)
+            rag_stat.recall_stat.model_name = doc_filter_result.model_name
+
+            relevant_docs: List[FilterDoc] = doc_filter_result.docs
             filter_time = time.time() - start_time
 
             # Filter relevant_docs to only include those with is_relevant=True
@@ -469,17 +514,15 @@ class LongContextRAG:
             # 将 FilterDoc 转化为 SourceCode 方便后续的逻辑继续做处理
             relevant_docs = [doc.source_code for doc in relevant_docs]
 
-            console = Console()
+            logger.info(f"=== RAG Search Results ===")
+            logger.info(f"Query: {query}")
+            logger.info(f"Found relevant docs: {len(relevant_docs)}")
 
-            # Create a table for the query information
-            query_table = Table(title="Query Information", show_header=False)
-            query_table.add_row("Query", query)
-            query_table.add_row("Relevant docs", str(len(relevant_docs)))
-
-            # Add relevant docs information
+            # 记录相关文档信息
             relevant_docs_info = []
-            for doc in relevant_docs:
-                info = f"- {doc.module_name.replace(self.path,'',1)}"
+            for i, doc in enumerate(relevant_docs):
+                doc_path = doc.module_name.replace(self.path, '', 1)
+                info = f"{i+1}. {doc_path}"
                 if "original_docs" in doc.metadata:
                     original_docs = ", ".join(
                         [
@@ -490,8 +533,11 @@ class LongContextRAG:
                     info += f" (Original docs: {original_docs})"
                 relevant_docs_info.append(info)
 
-            relevant_docs_info = "\n".join(relevant_docs_info)
-            query_table.add_row("Relevant docs list", relevant_docs_info)
+            if relevant_docs_info:
+                logger.info(
+                    f"Relevant documents list:"
+                    + "".join([f"\n  * {info}" for info in relevant_docs_info])
+                )
 
             first_round_full_docs = []
             second_round_extracted_docs = []
@@ -507,11 +553,18 @@ class LongContextRAG:
                     llm=self.llm,
                     disable_segment_reorder=self.args.disable_segment_reorder,
                 )
-                final_relevant_docs = token_limiter.limit_tokens(
+                
+                token_limiter_result = token_limiter.limit_tokens(
                     relevant_docs=relevant_docs,
                     conversations=conversations,
                     index_filter_workers=self.args.index_filter_workers or 5,
                 )
+
+                rag_stat.chunk_stat.total_input_tokens += sum(token_limiter_result.input_tokens_counts)
+                rag_stat.chunk_stat.total_generated_tokens += sum(token_limiter_result.generated_tokens_counts)
+                rag_stat.chunk_stat.model_name = token_limiter_result.model_name
+                
+                final_relevant_docs = token_limiter_result.docs
                 first_round_full_docs = token_limiter.first_round_full_docs
                 second_round_extracted_docs = token_limiter.second_round_extracted_docs
                 sencond_round_time = token_limiter.sencond_round_time
@@ -522,57 +575,64 @@ class LongContextRAG:
 
             logger.info(f"Finally send to model: {len(relevant_docs)}")
 
-            query_table.add_row("Only contexts", str(only_contexts))
-            query_table.add_row("Filter time", f"{filter_time:.2f} seconds")
-            query_table.add_row("Final relevant docs", str(len(relevant_docs)))
-            query_table.add_row(
-                "first_round_full_docs", str(len(first_round_full_docs))
-            )
-            query_table.add_row(
-                "second_round_extracted_docs", str(len(second_round_extracted_docs))
-            )
-            query_table.add_row(
-                "Second round time", f"{sencond_round_time:.2f} seconds"
+            # 记录分段处理的统计信息
+            logger.info(
+                f"=== Token Management ===\n"
+                f"  * Only contexts: {only_contexts}\n"
+                f"  * Filter time: {filter_time:.2f} seconds\n" 
+                f"  * Final relevant docs: {len(relevant_docs)}\n"
+                f"  * First round full docs: {len(first_round_full_docs)}\n"
+                f"  * Second round extracted docs: {len(second_round_extracted_docs)}\n"
+                f"  * Second round time: {sencond_round_time:.2f} seconds"
             )
 
-            # Add relevant docs information
+            # 记录最终选择的文档详情
             final_relevant_docs_info = []
-            for doc in relevant_docs:
-                info = f"- {doc.module_name.replace(self.path,'',1)}"
+            for i, doc in enumerate(relevant_docs):
+                doc_path = doc.module_name.replace(self.path, '', 1)
+                info = f"{i+1}. {doc_path}"
+                
+                metadata_info = []
                 if "original_docs" in doc.metadata:
                     original_docs = ", ".join(
                         [
-                            doc.replace(self.path, "", 1)
-                            for doc in doc.metadata["original_docs"]
+                            od.replace(self.path, "", 1)
+                            for od in doc.metadata["original_docs"]
                         ]
                     )
-                    info += f" (Original docs: {original_docs})"
+                    metadata_info.append(f"Original docs: {original_docs}")
+                    
                 if "chunk_ranges" in doc.metadata:
                     chunk_ranges = json.dumps(
                         doc.metadata["chunk_ranges"], ensure_ascii=False
                     )
-                    info += f" (Chunk ranges: {chunk_ranges})"
+                    metadata_info.append(f"Chunk ranges: {chunk_ranges}")
+                    
+                if "processing_time" in doc.metadata:
+                    metadata_info.append(f"Processing time: {doc.metadata['processing_time']:.2f}s")
+                    
+                if metadata_info:
+                    info += f" ({'; '.join(metadata_info)})"
+                    
                 final_relevant_docs_info.append(info)
 
-            final_relevant_docs_info = "\n".join(final_relevant_docs_info)
-            query_table.add_row("Final Relevant docs list", final_relevant_docs_info)
-
-            # Create a panel to contain the table
-            panel = Panel(
-                query_table,
-                title="RAG Search Results",
-                expand=False,
+            if final_relevant_docs_info:
+                logger.info(
+                    f"Final documents to be sent to model:"
+                    + "".join([f"\n  * {info}" for info in final_relevant_docs_info])
             )
 
-            # Log the panel using rich
-            console.print(panel)
-
+            # 记录令牌统计
             request_tokens = sum([doc.tokens for doc in relevant_docs])
             target_model = model or self.llm.default_model_name
             logger.info(
-                f"Start to send to model {target_model} with {request_tokens} tokens"
+                f"=== LLM Request ===\n"
+                f"  * Target model: {target_model}\n"
+                f"  * Total tokens: {request_tokens}"
             )
 
+            logger.info(f"Start to send to model {target_model} with {request_tokens} tokens")
+
             if LLMComputeEngine is not None and not self.args.disable_inference_enhance:
                 llm_compute_engine = LLMComputeEngine(
                     llm=target_llm,
@@ -585,17 +645,22 @@ class LongContextRAG:
                 new_conversations = llm_compute_engine.process_conversation(
                     conversations, query, [doc.source_code for doc in relevant_docs]
                 )
-
-                return (
-                    llm_compute_engine.stream_chat_oai(
+                chunks = llm_compute_engine.stream_chat_oai(
                         conversations=new_conversations,
                         model=model,
                         role_mapping=role_mapping,
                         llm_config=llm_config,
                         delta_mode=True,
-                    ),
-                    context,
-                )
+                    )
+                
+                def generate_chunks():
+                    for chunk in chunks:
+                        yield chunk[0]
+                        if chunk[1] is not None:
+                            rag_stat.answer_stat.total_input_tokens += chunk[1].input_tokens_count
+                            rag_stat.answer_stat.total_generated_tokens += chunk[1].generated_tokens_count                      
+                    self._print_rag_stats(rag_stat)        
+                return generate_chunks(), context
 
             new_conversations = conversations[:-1] + [
                 {
@@ -614,5 +679,85 @@ class LongContextRAG:
                 llm_config=llm_config,
                 delta_mode=True,
             )
+            
+            def generate_chunks():
+                for chunk in chunks:
+                    yield chunk[0]
+                    if chunk[1] is not None:
+                        rag_stat.answer_stat.total_input_tokens += chunk[1].input_tokens_count
+                        rag_stat.answer_stat.total_generated_tokens += chunk[1].generated_tokens_count                      
+                self._print_rag_stats(rag_stat)        
+            return generate_chunks(), context
+            
+            
 
-            return (chunk[0] for chunk in chunks), context
+    def _print_rag_stats(self, rag_stat: RAGStat) -> None:
+        """打印RAG执行的详细统计信息"""
+        total_input_tokens = (
+            rag_stat.recall_stat.total_input_tokens +
+            rag_stat.chunk_stat.total_input_tokens +
+            rag_stat.answer_stat.total_input_tokens
+        )
+        total_generated_tokens = (
+            rag_stat.recall_stat.total_generated_tokens +
+            rag_stat.chunk_stat.total_generated_tokens + 
+            rag_stat.answer_stat.total_generated_tokens
+        )
+        total_tokens = total_input_tokens + total_generated_tokens
+        
+        # 避免除以零错误
+        if total_tokens == 0:
+            recall_percent = chunk_percent = answer_percent = 0
+        else:
+            recall_percent = (rag_stat.recall_stat.total_input_tokens + rag_stat.recall_stat.total_generated_tokens) / total_tokens * 100
+            chunk_percent = (rag_stat.chunk_stat.total_input_tokens + rag_stat.chunk_stat.total_generated_tokens) / total_tokens * 100
+            answer_percent = (rag_stat.answer_stat.total_input_tokens + rag_stat.answer_stat.total_generated_tokens) / total_tokens * 100
+        
+        logger.info(
+            f"=== RAG 执行统计信息 ===\n"
+            f"总令牌使用: {total_tokens} 令牌\n"
+            f"  * 输入令牌总数: {total_input_tokens}\n"
+            f"  * 生成令牌总数: {total_generated_tokens}\n"
+            f"\n"
+            f"阶段统计:\n"
+            f"  1. 文档检索阶段:\n"
+            f"     - 模型: {rag_stat.recall_stat.model_name}\n"
+            f"     - 输入令牌: {rag_stat.recall_stat.total_input_tokens}\n"
+            f"     - 生成令牌: {rag_stat.recall_stat.total_generated_tokens}\n"
+            f"     - 阶段总计: {rag_stat.recall_stat.total_input_tokens + rag_stat.recall_stat.total_generated_tokens}\n"
+            f"\n"
+            f"  2. 文档分块阶段:\n"
+            f"     - 模型: {rag_stat.chunk_stat.model_name}\n"
+            f"     - 输入令牌: {rag_stat.chunk_stat.total_input_tokens}\n"
+            f"     - 生成令牌: {rag_stat.chunk_stat.total_generated_tokens}\n"
+            f"     - 阶段总计: {rag_stat.chunk_stat.total_input_tokens + rag_stat.chunk_stat.total_generated_tokens}\n"
+            f"\n"
+            f"  3. 答案生成阶段:\n"
+            f"     - 模型: {rag_stat.answer_stat.model_name}\n"
+            f"     - 输入令牌: {rag_stat.answer_stat.total_input_tokens}\n"
+            f"     - 生成令牌: {rag_stat.answer_stat.total_generated_tokens}\n"
+            f"     - 阶段总计: {rag_stat.answer_stat.total_input_tokens + rag_stat.answer_stat.total_generated_tokens}\n"
+            f"\n"
+            f"令牌分布百分比:\n"
+            f"  - 文档检索: {recall_percent:.1f}%\n"
+            f"  - 文档分块: {chunk_percent:.1f}%\n"
+            f"  - 答案生成: {answer_percent:.1f}%\n"
+        )
+        
+        # 记录原始统计数据，以便调试
+        logger.debug(f"RAG Stat 原始数据: {rag_stat}")
+        
+        # 返回成本估算
+        estimated_cost = self._estimate_token_cost(total_input_tokens, total_generated_tokens)
+        if estimated_cost > 0:
+            logger.info(f"估计成本: 约 ${estimated_cost:.4f} 人民币")
+
+    def _estimate_token_cost(self, input_tokens: int, output_tokens: int) -> float:
+        """估算当前请求的令牌成本（人民币）"""        
+        # 实际应用中，可以根据不同模型设置不同价格
+        input_cost_per_1m = 2.0/1000000   # 每百万输入令牌的成本
+        output_cost_per_1m = 8.0/100000   # 每百万输出令牌的成本
+        
+        cost = (input_tokens * input_cost_per_1m / 1000000) + (output_tokens* output_cost_per_1m/1000000)
+        return cost
+            

autocoder/rag/relevant_utils.py

@@ -1,7 +1,7 @@
 from autocoder.common import AutoCoderArgs, SourceCode
 from pydantic import BaseModel
 import re
-from typing import Optional
+from typing import Optional, List
 
 
 class DocRelevance(BaseModel):
@@ -23,6 +23,17 @@ class FilterDoc(BaseModel):
     task_timing: TaskTiming
 
 
+class DocFilterResult(BaseModel):
+    # 注意， docs 只保留最后成功过滤的文档
+    docs: List[FilterDoc]
+    # 注意， raw_docs 保留所有文档
+    raw_docs: List[FilterDoc]
+    input_tokens_counts: List[int]
+    generated_tokens_counts: List[int]
+    durations: List[float] 
+    model_name: str = "unknown"
+    
+
 def parse_relevance(text: Optional[str]) -> Optional[DocRelevance]:    
     if text is None:
         return None

autocoder/rag/token_limiter.py

@@ -7,7 +7,9 @@ from autocoder.common import SourceCode
 from byzerllm.utils.client.code_utils import extract_code
 import byzerllm
 from byzerllm import ByzerLLM
-
+from autocoder.rag.relevant_utils import TaskTiming
+from byzerllm import MetaHolder
+from autocoder.rag.token_limiter_utils import TokenLimiterResult
 
 class TokenLimiter:
     def __init__(
@@ -94,10 +96,24 @@ class TokenLimiter:
         relevant_docs: List[SourceCode],
         conversations: List[Dict[str, str]],
         index_filter_workers: int,
-    ) -> List[SourceCode]:
+    ) -> TokenLimiterResult:
+        logger.info(f"=== TokenLimiter Starting ===")
+        logger.info(f"Configuration: full_text_limit={self.full_text_limit}, segment_limit={self.segment_limit}, buff_limit={self.buff_limit}")
+        logger.info(f"Processing {len(relevant_docs)} source code documents")
+        
+        start_time = time.time()
         final_relevant_docs = []
         token_count = 0
         doc_num_count = 0
+        model_name = self.chunk_llm.default_model_name or "unknown"
+        token_limiter_result = TokenLimiterResult(
+                docs=[],
+                raw_docs=[],
+                input_tokens_counts=[],
+                generated_tokens_counts=[],
+                durations=[],
+                model_name=model_name
+            )
 
         reorder_relevant_docs = []
 
@@ -112,6 +128,7 @@ class TokenLimiter:
         ## TODO:
         ##     1. 未来根据参数决定是否开启重排以及重排的策略
         if not self.disable_segment_reorder:
+            logger.info("Document reordering enabled - organizing segments by original document order")
             num_count = 0
             for doc in relevant_docs:
                 num_count += 1
@@ -135,8 +152,11 @@ class TokenLimiter:
                     temp_docs.sort(key=lambda x: x.metadata["chunk_index"])
                     reorder_relevant_docs.extend(temp_docs)
         else:
+            logger.info("Document reordering disabled - using original retrieval order")
             reorder_relevant_docs = relevant_docs
 
+        logger.info(f"After reordering: {len(reorder_relevant_docs)} documents to process")
+
         ## 非窗口分区实现
         for doc in reorder_relevant_docs:
             doc_tokens = self.count_tokens(doc.source_code)
@@ -149,10 +169,15 @@ class TokenLimiter:
 
         ## 如果窗口无法放下所有的相关文档，则需要分区
         if len(final_relevant_docs) < len(reorder_relevant_docs):
+            logger.info(f"Token limit exceeded: {len(final_relevant_docs)}/{len(reorder_relevant_docs)} docs fit in window")
+            logger.info(f"=== Starting First Round: Full Text Loading ===")
+            
             ## 先填充full_text分区
             token_count = 0
             new_token_limit = self.full_text_limit
             doc_num_count = 0
+            first_round_start_time = time.time()
+            
             for doc in reorder_relevant_docs:
                 doc_tokens = self.count_tokens(doc.source_code)
                 doc_num_count += 1
@@ -161,11 +186,18 @@ class TokenLimiter:
                     token_count += doc_tokens
                 else:
                     break
+            
+            first_round_duration = time.time() - first_round_start_time
+            logger.info(
+                f"First round complete: loaded {len(self.first_round_full_docs)} documents"
+                f" ({token_count} tokens) in {first_round_duration:.2f}s"
+            )
 
             if len(self.first_round_full_docs) > 0:
                 remaining_tokens = (
                     self.full_text_limit + self.segment_limit - token_count
                 )
+                logger.info(f"Remaining token budget: {remaining_tokens}")
             else:
                 logger.warning(
                     "Full text area is empty, this is may caused by the single doc is too long"
@@ -175,49 +207,117 @@ class TokenLimiter:
             ## 继续填充segment分区
             sencond_round_start_time = time.time()
             remaining_docs = reorder_relevant_docs[len(self.first_round_full_docs) :]
+            
             logger.info(
-                f"first round docs: {len(self.first_round_full_docs)} remaining docs: {len(remaining_docs)} index_filter_workers: {index_filter_workers}"
+                f"=== Starting Second Round: Chunk Extraction ==="
+                f"\n  * Documents to process: {len(remaining_docs)}"
+                f"\n  * Remaining token budget: {remaining_tokens}"
+                f"\n  * Thread pool size: {index_filter_workers or 5}"
             )
 
+            total_processed = 0
+            successful_extractions = 0
+            
+            
             with ThreadPoolExecutor(max_workers=index_filter_workers or 5) as executor:
-                future_to_doc = {
-                    executor.submit(self.process_range_doc, doc, conversations): doc
-                    for doc in remaining_docs
-                }
+                future_to_doc = {}
+                for doc in remaining_docs:
+                    submit_time = time.time()
+                    future = executor.submit(self.process_range_doc, doc, conversations)
+                    future_to_doc[future] = (doc, submit_time)
 
                 for future in as_completed(future_to_doc):
-                    doc = future_to_doc[future]
+                    doc, submit_time = future_to_doc[future]
+                    end_time = time.time()
+                    total_processed += 1
+                    progress_percent = (total_processed / len(remaining_docs)) * 100
+                    
                     try:
                         result = future.result()
+                        task_duration = end_time - submit_time
+                        
                         if result and remaining_tokens > 0:
                             self.second_round_extracted_docs.append(result)
+                            token_limiter_result.raw_docs.append(result)
+                            
+                            if "rag" in result.metadata and "chunk" in result.metadata["rag"]:
+                                chunk_meta = result.metadata["rag"]["chunk"]
+                                token_limiter_result.input_tokens_counts.append(chunk_meta.get("input_tokens_count", 0))
+                                token_limiter_result.generated_tokens_counts.append(chunk_meta.get("generated_tokens_count", 0))
+                                token_limiter_result.durations.append(chunk_meta.get("duration", 0))
+                            
                             tokens = result.tokens
+                            successful_extractions += 1
+                            
+                            logger.info(
+                                f"Document extraction [{progress_percent:.1f}%] - {total_processed}/{len(remaining_docs)}:"
+                                f"\n  - File: {doc.module_name}"
+                                f"\n  - Chunks: {len(result.metadata.get('chunk_ranges', []))}"
+                                f"\n  - Extracted tokens: {tokens}"
+                                f"\n  - Remaining tokens: {remaining_tokens - tokens if tokens > 0 else remaining_tokens}"
+                                f"\n  - Processing time: {task_duration:.2f}s"
+                            )
+                            
                             if tokens > 0:
                                 remaining_tokens -= tokens
                             else:
                                 logger.warning(
                                     f"Token count for doc {doc.module_name} is 0 or negative"
                                 )
+                        elif result:
+                            logger.info(
+                                f"Document extraction [{progress_percent:.1f}%] - {total_processed}/{len(remaining_docs)}:"
+                                f"\n  - File: {doc.module_name}"
+                                f"\n  - Skipped: Token budget exhausted ({remaining_tokens} remaining)"
+                                f"\n  - Processing time: {task_duration:.2f}s"
+                            )
+                        else:
+                            logger.warning(
+                                f"Document extraction [{progress_percent:.1f}%] - {total_processed}/{len(remaining_docs)}:"
+                                f"\n  - File: {doc.module_name}"
+                                f"\n  - Result: No content extracted"
+                                f"\n  - Processing time: {task_duration:.2f}s"
+                            )
                     except Exception as exc:
                         logger.error(
-                            f"Processing doc {doc.module_name} generated an exception: {exc}"
+                            f"Document extraction [{progress_percent:.1f}%] - {total_processed}/{len(remaining_docs)}:"
+                            f"\n  - File: {doc.module_name}"
+                            f"\n  - Error: {exc}"
+                            f"\n  - Processing time: {end_time - submit_time:.2f}s"
                         )
 
             final_relevant_docs = (
                 self.first_round_full_docs + self.second_round_extracted_docs
             )
             self.sencond_round_time = time.time() - sencond_round_start_time
+            total_time = time.time() - start_time
+            
             logger.info(
-                f"Second round processing time: {self.sencond_round_time:.2f} seconds"
+                f"=== Second round complete ==="
+                f"\n  * Time: {self.sencond_round_time:.2f}s"
+                f"\n  * Documents processed: {total_processed}/{len(remaining_docs)}"
+                f"\n  * Successful extractions: {successful_extractions}"
+                f"\n  * Extracted tokens: {sum(doc.tokens for doc in self.second_round_extracted_docs)}"
             )
-
-        return final_relevant_docs
+        else:
+            logger.info(f"All {len(reorder_relevant_docs)} documents fit within token limits")
+            total_time = time.time() - start_time
+        
+        logger.info(
+            f"=== TokenLimiter Complete ==="
+            f"\n  * Total time: {total_time:.2f}s"
+            f"\n  * Documents selected: {len(final_relevant_docs)}/{len(relevant_docs)}"
+            f"\n  * Total tokens: {sum(doc.tokens for doc in final_relevant_docs)}"
+        )
+        token_limiter_result.docs = final_relevant_docs
+        return token_limiter_result
 
     def process_range_doc(
         self, doc: SourceCode, conversations: List[Dict[str, str]], max_retries=3
     ) -> SourceCode:
         for attempt in range(max_retries):
             content = ""
+            start_time = time.time()
             try:
                 source_code_with_line_number = ""
                 source_code_lines = doc.source_code.split("\n")
@@ -225,14 +325,19 @@ class TokenLimiter:
                     source_code_with_line_number += f"{idx+1} {line}\n"
                 
                 llm = self.chunk_llm
+                model_name = llm.default_model_name or "unknown"
+                meta_holder = MetaHolder()
 
+                extraction_start_time = time.time()
                 extracted_info = (
                     self.extract_relevance_range_from_docs_with_conversation.options(
                         {"llm_config": {"max_length": 100}}
                     )
-                    .with_llm(llm)
+                    .with_llm(llm).with_meta(meta_holder)
                     .run(conversations, [source_code_with_line_number])
                 )
+                extraction_duration = time.time() - extraction_start_time
+                
                 json_str = extract_code(extracted_info)[0][1]
                 json_objs = json.loads(json_str)
 
@@ -242,23 +347,59 @@ class TokenLimiter:
                     chunk = "\n".join(source_code_lines[start_line:end_line])
                     content += chunk + "\n"
 
+                total_duration = time.time() - start_time
+                
+
+                meta = meta_holder.get_meta_model()
+                
+                input_tokens_count = 0
+                generated_tokens_count = 0                                
+
+                if meta:
+                    input_tokens_count = meta.input_tokens_count
+                    generated_tokens_count = meta.generated_tokens_count                    
+                
+                logger.debug(
+                    f"Document {doc.module_name} chunk extraction details:"
+                    f"\n  - Chunks found: {len(json_objs)}"
+                    f"\n  - Input tokens: {input_tokens_count}"
+                    f"\n  - Generated tokens: {generated_tokens_count}" 
+                    f"\n  - LLM time: {extraction_duration:.2f}s"                    
+                    f"\n  - Total processing time: {total_duration:.2f}s"
+                )
+
+                if "rag" not in doc.metadata:
+                    doc.metadata["rag"] = {}
+                
+                doc.metadata["rag"]["chunk"] = {
+                    "original_doc": doc.module_name,
+                    "chunk_ranges": json_objs,
+                    "processing_time": total_duration,
+                    "llm_time": extraction_duration,  
+
+                    "input_tokens_count": input_tokens_count,
+                    "generated_tokens_count": generated_tokens_count,
+                    "duration": extraction_duration,  
+                    "chunk_model":model_name                                                                             
+                }
+                                
                 return SourceCode(
                     module_name=doc.module_name,
                     source_code=content.strip(),
-                    tokens=self.count_tokens(content),
+                    tokens=input_tokens_count + generated_tokens_count,
                     metadata={
-                        "original_doc": doc.module_name,
-                        "chunk_ranges": json_objs,
+                        **doc.metadata                        
                     },
                 )
             except Exception as e:
+                err_duration = time.time() - start_time
                 if attempt < max_retries - 1:
                     logger.warning(
-                        f"Error processing doc {doc.module_name}, retrying... (Attempt {attempt + 1}) Error: {str(e)}"
+                        f"Error processing doc {doc.module_name}, retrying... (Attempt {attempt + 1}) Error: {str(e)}, duration: {err_duration:.2f}s"
                     )
                 else:
                     logger.error(
-                        f"Failed to process doc {doc.module_name} after {max_retries} attempts: {str(e)}"
+                        f"Failed to process doc {doc.module_name} after {max_retries} attempts: {str(e)}, total duration: {err_duration:.2f}s"
                     )
                     return SourceCode(
                         module_name=doc.module_name, source_code="", tokens=0

autocoder/rag/token_limiter_utils.py

@@ -0,0 +1,13 @@
+import pydantic
+from typing import List
+from autocoder.common import SourceCode
+
+class TokenLimiterResult(pydantic.BaseModel):
+    # 注意， docs 只保留结果文档
+    docs: List[SourceCode]
+    # 注意， raw_docs 保留所有文档
+    raw_docs: List[SourceCode]
+    input_tokens_counts: List[int]
+    generated_tokens_counts: List[int]
+    durations: List[float]
+    model_name: str = "unknown"

autocoder/version.py

@@ -1 +1 @@
-__version__ = "0.1.271"
+__version__ = "0.1.272"