auto-coder 1.0.0__py3-none-any.whl → 2.0.1__py3-none-any.whl

autocoder/rag/agentic_rag.py

@@ -14,47 +14,67 @@ from importlib.metadata import version
 from pydantic import BaseModel
 from autocoder.common import openai_content as OpenAIContentProcessor
 from autocoder.rag.long_context_rag import LongContextRAG
-import json, os
+import json
+import os
 from autocoder.agent.base_agentic.base_agent import BaseAgent
-from autocoder.agent.base_agentic.types import AgentRequest
+from autocoder.agent.base_agentic.types import AgentRequest, AgenticEditConversationConfig
 from autocoder.common import SourceCodeList
-from autocoder.rag.tools import register_search_tool, register_recall_tool
+from autocoder.rag.tools import register_search_tool, register_recall_tool, register_todo_read_tool, register_todo_write_tool, register_web_search_tool, register_web_crawl_tool
 from byzerllm.utils.types import SingleOutputMeta
-from autocoder.utils.llms import get_single_llm
-try:
-    from autocoder_pro.rag.llm_compute import LLMComputeEngine
-    pro_version = version("auto-coder-pro")
-    autocoder_version = version("auto-coder")
-    logger.warning(
-        f"auto-coder-pro({pro_version}) plugin is enabled in auto-coder.rag({autocoder_version})")
-except ImportError:
-    logger.warning(
-        "Please install auto-coder-pro to enhance llm compute ability")
-    LLMComputeEngine = None
-
-
-class RAGAgent(BaseAgent):    
-    def __init__(self, name: str, 
-        llm: Union[byzerllm.ByzerLLM, byzerllm.SimpleByzerLLM], 
-        files: SourceCodeList, 
-        args: AutoCoderArgs, 
-        rag: LongContextRAG,
-        conversation_history: Optional[List[Dict[str, Any]]] = None):
-        self.llm = llm 
+import uuid
+
+class RAGAgent(BaseAgent):
+    def __init__(self, name: str,
+                 llm: Union[byzerllm.ByzerLLM, byzerllm.SimpleByzerLLM],
+                 files: SourceCodeList,
+                 args: AutoCoderArgs,
+                 rag: LongContextRAG,
+                 conversation_history: Optional[List[Dict[str, Any]]] = None,
+                 custom_system_prompt: Optional[str] = None,
+                 conversation_config: Optional[Any] = None,
+                 cancel_token: Optional[str] = None):
+
+        if conversation_config is None:
+            conversation_config = AgenticEditConversationConfig(
+                conversation_name=f"rag_session_{uuid.uuid4().hex[:8]}",
+                conversation_id=str(uuid.uuid4())
+            )
+
+        self.llm = llm
         self.default_llm = self.llm
         self.context_prune_llm = self.default_llm
         if self.default_llm.get_sub_client("context_prune_model"):
-            self.context_prune_llm = self.default_llm.get_sub_client("context_prune_model")
+            self.context_prune_llm = self.default_llm.get_sub_client(
+                "context_prune_model")
 
         self.llm = self.default_llm
         if self.default_llm.get_sub_client("agentic_model"):
             self.llm = self.default_llm.get_sub_client("agentic_model")
-        
-        self.rag = rag        
-        super().__init__(name, self.llm, files, args, conversation_history, default_tools_list=["read_file"])        
+
+        self.rag = rag
+        super().__init__(
+            name=name,
+            llm=self.llm,
+            files=files,
+            args=args,
+            conversation_history=conversation_history,
+            default_tools_list=["read_file"],
+            custom_system_prompt=custom_system_prompt,
+            conversation_config=conversation_config,
+            cancel_token=cancel_token
+        )
         # 注册RAG工具
         # register_search_tool()
         register_recall_tool()
+        register_todo_read_tool()
+        register_todo_write_tool()
+
+        # 如果配置了firecrawl_api_key，则注册web_search_tool和web_crawl_tool
+        if self.args.firecrawl_api_key or self.args.metaso_api_key or self.args.bochaai_api_key:            
+            logger.info(f"firecrawl_api_key or metaso_api_key or bochaai_api_key is configured, register web_search_tool and web_crawl_tool")
+            register_web_search_tool()
+            register_web_crawl_tool()
+
 
 class AgenticRAG:
     def __init__(
@@ -64,19 +84,18 @@ class AgenticRAG:
         path: str,
         tokenizer_path: Optional[str] = None,
     ) -> None:
-       self.llm = llm
-       self.args = args
-       self.path = path
-       self.tokenizer_path = tokenizer_path
-       self.rag = LongContextRAG(llm=self.llm, args=self.args, path=self.path, tokenizer_path=self.tokenizer_path) 
-
+        self.llm = llm
+        self.args = args
+        self.path = path
+        self.tokenizer_path = tokenizer_path
+        self.rag = LongContextRAG(
+            llm=self.llm, args=self.args, path=self.path, tokenizer_path=self.tokenizer_path)
 
     def build(self):
         pass
 
     def search(self, query: str) -> List[SourceCode]:
         return []
-    
 
     def stream_chat_oai(
         self,
@@ -85,7 +104,7 @@ class AgenticRAG:
         role_mapping=None,
         llm_config: Dict[str, Any] = {},
         extra_request_params: Dict[str, Any] = {}
-    ):        
+    ):
         try:
             return self._stream_chat_oai(
                 conversations,
@@ -100,7 +119,7 @@ class AgenticRAG:
             return ["出现错误，请稍后再试。"], []
 
     @byzerllm.prompt()
-    def conversation_to_query(self,messages: List[Dict[str, Any]]):
+    def conversation_to_query(self, messages: List[Dict[str, Any]]):
         '''        
         【历史对话】按时间顺序排列，从旧到新：
         {% for message in messages %}
@@ -111,45 +130,139 @@ class AgenticRAG:
         </content>
         </message>
         {% endfor %}
-        
+
         【当前问题】用户的最新需求如下:
         <current_query>
         {{ query }}
         </current_query>            
-        ''' 
+        '''
         temp_messages = messages[0:-1]
         message = messages[-1]
 
         return {
             "messages": temp_messages,
-            "query":message["content"]
+            "query": message["content"]
         }
 
     @byzerllm.prompt()
     def system_prompt(self):
         '''
-        你是一个基于知识库的智能助手，我的核心能力是通过检索增强生成（RAG）技术来回答用户问题。
-
-        你的工作流程如下：
-        1. 当用户提出问题时，你首先理解问题的核心意图和关键信息需求
-        2. 你会从多个角度分析问题，确定最佳的检索策略和关键词，然后召回工具 recall 获取与问题最相关的详细内容，只有在特别有必要的情况下，你才回使用 read_file 来获得相关文件更详细的信息。
-        5. 如果获得的信息足够回答用户问题，你会直接生成回答。
-        6. 如果获得的信息不足以回答用户问题，你会继续使用 recall 工具，直到你确信已经获取了足够的信息来回答用户问题。
-        7. 有的问题可能需要拆解成多个问题，分别进行recall,然后最终得到的结果才是完整信息，最后才能进行回答。                
-        8. 当你遇到图片的时候，请根据图片前后文本内容推测改图片与问题的相关性，有相关性则在回答中使用 ![]()格式输出该Markdown图片路径，否则不输出。        
+        You are an intelligent assistant based on a knowledge base. My core capability is to answer user questions through Retrieval-Augmented Generation (RAG) technology.
+
+        Your workflow is as follows:
+        1. When users ask questions, you first understand the core intent and key information needs of the question
+        2. For complex questions, you MUST use TODO tools to manage the analysis and retrieval process
+        3. You analyze the question from multiple angles, determine the best retrieval strategy and keywords, then use the recall tool to get the most relevant detailed content
+        4. If the obtained information is sufficient to answer the user's question, you will generate an answer directly
+        5. If the obtained information is insufficient, you will continue using the recall tool until you have obtained enough information
+
+        ## IMPORTANT: All Complex Questions MUST Use TODO Files
+
+        For ANY complex question that requires multiple steps, comparisons, or process tracing, you MUST use TODO tools to manage the work:
+
+        ### Example 1: Comparison Questions
+        User question: "What are the differences between the implementation methods of the user authentication module and the permission management module in the system?"
+
+        **MANDATORY Processing with TODO:**
+
+        1. **First create TODO list**:
+        ```
+        <todo_write>
+        <action>create</action>
+        <content>
+        <task>Retrieve and analyze user authentication module implementation details</task>
+        <task>Retrieve and analyze permission management module implementation details</task>
+        <task>Compare architecture design between the two modules</task>
+        <task>Compare technology selection and patterns used</task>
+        <task>Synthesize findings into comprehensive comparison</task>
+        </content>
+        <priority>high</priority>
+        </todo_write>
+        ```
+
+        2. **Execute with TODO tracking**:
+        - Check todo_read before starting
+        - Mark first todo as in_progress
+        - Use recall("user authentication login token") for authentication info
+        - Mark first todo as completed with notes
+        - Mark second todo as in_progress
+        - Use recall("permission management authorization role RBAC") for permission info
+        - Mark second todo as completed
+        - Continue through all todos systematically
+
+        3. **Complete analysis**: After all todos are marked completed, provide comprehensive comparison
+
+        ### Example 2: Process Tracing Questions
+        User question: "What is the entire process from user login to order completion? Which modules and interfaces are involved?"
+
+        **MANDATORY Processing with TODO:**
+
+        1. **First create TODO list**:
+        ```
+        <todo_write>
+        <action>create</action>
+        <content>
+        <task>Trace and document user login process and entry points</task>
+        <task>Analyze session management and token verification flow</task>
+        <task>Map order creation process and required data</task>
+        <task>Track order processing workflow and state transitions</task>
+        <task>Identify payment integration and completion mechanisms</task>
+        <task>Document all involved modules and interfaces</task>
+        <task>Create end-to-end process summary</task>
+        </content>
+        <priority>high</priority>
+        </todo_write>
+        ```
+
+        2. **Execute with TODO tracking**:
+        - Use todo_read to check current status
+        - For each todo: mark as in_progress → execute recalls → mark as completed
+        - Use recall for each process step (login, session, order, payment)
+        - Add new todos if additional steps are discovered
+        - Document findings in todo notes
+
+        3. **Synthesize results**: Once all todos completed, organize the complete business flow
+
+        ### When TODO Files are MANDATORY:
+
+        **You MUST use TODO files for:**
+        - Any question requiring comparison between multiple items
+        - Process tracing or workflow analysis questions
+        - Questions with "step by step", "entire process", or "all involved" keywords
+        - Multi-part questions that need systematic investigation
+        - Questions about relationships between different components
+        - Any analysis requiring more than 3 recall operations
+
+        ### TODO Tool Usage Rules:
+
+        1. **Always start complex questions with todo_write**: Create a comprehensive plan BEFORE any recall
+        2. **Use todo_read before each major step**: Maintain awareness of progress
+        3. **Mark todos systematically**: in_progress → completed with notes
+        4. **Add discovered todos**: Use action="add_task" when finding new aspects to investigate
+        5. **Document in notes**: Record key findings, blockers, or important information
+
+        ### Tool Usage Principles:
+        - **todo_write/todo_read**: MANDATORY for all complex questions - use FIRST before any recalls
+        - **recall tool**: Primary tool for information retrieval after TODO planning
+        - **read_file tool**: Only when specific file details are needed beyond recall results
+        - **Iterative approach**: Adjust keywords and add todos based on findings
+
+        ### Remember:
+        This is a RAG system focused on information retrieval and synthesis. We do not modify code or files, we only retrieve and analyze information to answer questions. TODO tools help organize complex retrieval tasks systematically.
+
+        8. When you encounter images, please infer the relevance of the image to the question based on the surrounding text content. If relevant, output the Markdown image path using ![]() format in your answer; otherwise, do not output it.
         {% if local_image_host %}
-        9. 图片路径处理
-        - 图片地址需返回绝对路径, 
-        - 对于Windows风格的路径，需要转换为Linux风格， 例如：![image](C:\\Users\\user\\Desktop\\image.png) 转换为 ![image](C:/Users/user/Desktop/image.png)
-        - 为请求图片资源 需增加 http://{{ local_image_host }}/static/ 作为前缀
-        举个例子：![image](/path/to/images/image.png)， 返回 ![image](http://{{ local_image_host }}/static/path/to/images/image.png)
+        9. Image path processing
+        - Image addresses need to return absolute paths
+        - For Windows-style paths, convert to Linux-style, for example: ![image](C:\\Users\\user\\Desktop\\image.png) converts to ![image](C:/Users/user/Desktop/image.png)
+        - To request image resources, add http://{{ local_image_host }}/static/ as a prefix
+        For example: ![image](/path/to/images/image.png), return ![image](http://{{ local_image_host }}/static/path/to/images/image.png)
         {% endif %} 
-        '''    
+        '''
         return {
             "local_image_host": self.args.local_image_host
         }
 
-    
     def _stream_chat_oai(
         self,
         conversations,
@@ -160,45 +273,72 @@ class AgenticRAG:
     ):
         if not llm_config:
             llm_config = {}
-        
+
         if extra_request_params:
             llm_config.update(extra_request_params)
-        
-        conversations = OpenAIContentProcessor.process_conversations(conversations)                
+
+        conversations = OpenAIContentProcessor.process_conversations(
+            conversations)
 
         context = []
 
         def _generate_sream():
-                           
-            recall_request = AgentRequest(user_input=self.conversation_to_query.prompt(conversations))              
+            # 提取最后一个 user 消息和对话历史
+            last_user_message = None
+            conversation_history = []
+
+            # 从后往前查找最后一个 role="user" 的消息
+            for i in range(len(conversations) - 1, -1, -1):
+                if conversations[i].get("role") == "user":
+                    last_user_message = conversations[i]["content"]
+                    # 将之前的所有消息作为历史记录（不包括最后一个user消息）
+                    conversation_history = conversations[:i]
+                    break
+
+            # 如果没有找到 user 消息，使用默认处理
+            if last_user_message is None:
+                last_user_message = self.conversation_to_query.prompt(
+                    conversations)
+                conversation_history = []
+
+            recall_request = AgentRequest(user_input=last_user_message)
+
+            # 为每个RAGAgent实例创建新的conversation_config
+            conversation_config = AgenticEditConversationConfig(
+                conversation_name=f"rag_session_{uuid.uuid4().hex[:8]}",
+                conversation_id=str(uuid.uuid4())
+            )
+
             rag_agent = RAGAgent(
                 name="RAGAgent",
                 llm=self.llm,
                 files=SourceCodeList(sources=[]),
                 args=self.args,
                 rag=self.rag,
-                conversation_history=[]
-            ) 
+                conversation_history=conversation_history,
+                custom_system_prompt=None,
+                conversation_config=conversation_config,
+                cancel_token=None
+            )
 
             rag_agent.who_am_i(self.system_prompt.prompt())
-                                     
-            events =rag_agent.run_with_generator(recall_request)
-            for (t,content) in events:    
-                if t == "thinking":                          
+
+            events = rag_agent.run_with_generator(recall_request)
+            for (t, content) in events:
+                if t == "thinking":
                     yield ("", SingleOutputMeta(
-                    generated_tokens_count=0,
-                    input_tokens_count=0,
-                    reasoning_content=content,                    
-                ))
+                        generated_tokens_count=0,
+                        input_tokens_count=0,
+                        reasoning_content=content,
+                    ))
                 else:
                     yield (content, SingleOutputMeta(
                         generated_tokens_count=0,
                         input_tokens_count=0,
-                        reasoning_content="",                    
+                        reasoning_content="",
                     ))
 
         return _generate_sream(), context
-
-        
-
     
+    def close(self):
+        self.rag.close()

autocoder/rag/cache/simple_cache.py

@@ -157,8 +157,7 @@ class AutoCoderRAGAsyncUpdateQueue(BaseCacheManager):
             # 检查是否在忽略规则中
             if self.ignore_spec and self.ignore_spec.match_file(os.path.relpath(file_path, self.path)):
                 return
-                
-            logger.info(f"File change detected: {change_type} - {file_path}")
+                            
             self.trigger_update()
         except Exception as e:
             logger.error(f"Error in file change handler: {e}")

autocoder/rag/loaders/image_loader.py

@@ -18,7 +18,7 @@ from byzerllm.utils.client import code_utils
 from autocoder.utils.llms import get_single_llm
 from loguru import logger
 from typing import List, Tuple, Optional
-from autocoder.common.text import TextSimilarity
+from autocoder.common.text_similarity import TextSimilarity
 from pydantic import BaseModel
 
 

autocoder/rag/long_context_rag.py

@@ -32,6 +32,7 @@ from autocoder.common.save_formatted_log import save_formatted_log
 from autocoder.rag.types import (
     RecallStat,ChunkStat,AnswerStat,OtherStat,RAGStat
 )
+from autocoder.common.llms import LLMManager
 import json, os
 try:
     from autocoder_pro.rag.llm_compute import LLMComputeEngine
@@ -1103,19 +1104,21 @@ class LongContextRAG:
 
     def _estimate_token_cost(self, rag_stat: RAGStat) -> float:
         """估算当前请求的令牌成本（人民币）"""
-        from autocoder.models import get_model_by_name
-        
+        llm_manager = LLMManager()
         total_cost = 0.0
         
         # 计算召回阶段成本
         if rag_stat.recall_stat.model_name != "unknown":
             try:
-                recall_model = get_model_by_name(rag_stat.recall_stat.model_name)
-                input_cost = recall_model.get("input_price", 0.0) / 1000000
-                output_cost = recall_model.get("output_price", 0.0) / 1000000
-                recall_cost = (rag_stat.recall_stat.total_input_tokens * input_cost) + \
-                             (rag_stat.recall_stat.total_generated_tokens * output_cost)
-                total_cost += recall_cost
+                recall_model = llm_manager.get_model(rag_stat.recall_stat.model_name)
+                if recall_model:
+                    input_cost = recall_model.input_price / 1000000
+                    output_cost = recall_model.output_price / 1000000
+                    recall_cost = (rag_stat.recall_stat.total_input_tokens * input_cost) + \
+                                 (rag_stat.recall_stat.total_generated_tokens * output_cost)
+                    total_cost += recall_cost
+                else:
+                    recall_cost = 0.0
             except Exception as e:
                 logger.warning(f"计算召回阶段成本时出错: {str(e)}")                
                 recall_cost = 0.0
@@ -1125,12 +1128,15 @@ class LongContextRAG:
         # 计算分块阶段成本
         if rag_stat.chunk_stat.model_name != "unknown":
             try:
-                chunk_model = get_model_by_name(rag_stat.chunk_stat.model_name)
-                input_cost = chunk_model.get("input_price", 0.0) / 1000000
-                output_cost = chunk_model.get("output_price", 0.0) / 1000000
-                chunk_cost = (rag_stat.chunk_stat.total_input_tokens * input_cost) + \
-                            (rag_stat.chunk_stat.total_generated_tokens * output_cost)
-                total_cost += chunk_cost
+                chunk_model = llm_manager.get_model(rag_stat.chunk_stat.model_name)
+                if chunk_model:
+                    input_cost = chunk_model.input_price / 1000000
+                    output_cost = chunk_model.output_price / 1000000
+                    chunk_cost = (rag_stat.chunk_stat.total_input_tokens * input_cost) + \
+                                (rag_stat.chunk_stat.total_generated_tokens * output_cost)
+                    total_cost += chunk_cost
+                else:
+                    chunk_cost = 0.0
             except Exception as e:
                 logger.warning(f"计算分块阶段成本时出错: {str(e)}")
                 # 使用默认值
@@ -1141,12 +1147,15 @@ class LongContextRAG:
         # 计算答案生成阶段成本
         if rag_stat.answer_stat.model_name != "unknown":
             try:
-                answer_model = get_model_by_name(rag_stat.answer_stat.model_name)
-                input_cost = answer_model.get("input_price", 0.0) / 1000000
-                output_cost = answer_model.get("output_price", 0.0) / 1000000
-                answer_cost = (rag_stat.answer_stat.total_input_tokens * input_cost) + \
-                             (rag_stat.answer_stat.total_generated_tokens * output_cost)
-                total_cost += answer_cost
+                answer_model = llm_manager.get_model(rag_stat.answer_stat.model_name)
+                if answer_model:
+                    input_cost = answer_model.input_price / 1000000
+                    output_cost = answer_model.output_price / 1000000
+                    answer_cost = (rag_stat.answer_stat.total_input_tokens * input_cost) + \
+                                 (rag_stat.answer_stat.total_generated_tokens * output_cost)
+                    total_cost += answer_cost
+                else:
+                    answer_cost = 0.0
             except Exception as e:
                 logger.warning(f"计算答案生成阶段成本时出错: {str(e)}")
                 # 使用默认值
@@ -1158,12 +1167,15 @@ class LongContextRAG:
         for i, other_stat in enumerate(rag_stat.other_stats):
             if other_stat.model_name != "unknown":
                 try:
-                    other_model = get_model_by_name(other_stat.model_name)
-                    input_cost = other_model.get("input_price", 0.0) / 1000000
-                    output_cost = other_model.get("output_price", 0.0) / 1000000
-                    other_cost = (other_stat.total_input_tokens * input_cost) + \
-                                (other_stat.total_generated_tokens * output_cost)
-                    total_cost += other_cost
+                    other_model = llm_manager.get_model(other_stat.model_name)
+                    if other_model:
+                        input_cost = other_model.input_price / 1000000
+                        output_cost = other_model.output_price / 1000000
+                        other_cost = (other_stat.total_input_tokens * input_cost) + \
+                                    (other_stat.total_generated_tokens * output_cost)
+                        total_cost += other_cost
+                    else:
+                        other_cost = 0.0
                 except Exception as e:
                     logger.warning(f"计算其他阶段 {i+1} 成本时出错: {str(e)}")
                     # 使用默认值
@@ -1174,3 +1186,7 @@ class LongContextRAG:
         # 将总成本保存到 rag_stat
         rag_stat.cost = total_cost
         return total_cost
+    
+    def close(self):
+        if self.tokenizer and isinstance(self.tokenizer, TokenCounter):
+            self.tokenizer.close()

autocoder/rag/qa_conversation_strategy.py

@@ -2,7 +2,7 @@ from abc import ABC, abstractmethod
 from typing import List, Dict, Any, Generator
 import byzerllm
 from autocoder.common import AutoCoderArgs
-from autocoder.common.rulefiles.autocoderrules_utils import get_rules
+from autocoder.common.rulefiles import get_rules
 
 class QAConversationStrategy(ABC):
     """

autocoder/rag/terminal/__init__.py

@@ -0,0 +1,17 @@
+"""
+RAG Terminal - RAG 系统的终端交互入口模块
+
+这个模块提供了 RAG 系统的终端交互功能，代码结构清晰，职责分明。
+"""
+
+# 延迟导入以避免循环依赖
+__all__ = ["run_cli"]
+
+
+def __getattr__(name):
+    """延迟导入以避免循环依赖"""
+    if name == "run_cli":
+        from autocoder.rag.terminal.bootstrap import run_cli
+
+        return run_cli
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")