auto-coder 0.1.271__tar.gz → 0.1.273__tar.gz

{auto_coder-0.1.271 → auto_coder-0.1.273}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: auto-coder
-Version: 0.1.271
+Version: 0.1.273
 Summary: AutoCoder: AutoCoder
 Author: allwefantasy
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
@@ -26,7 +26,7 @@ Requires-Dist: tabulate
 Requires-Dist: jupyter_client
 Requires-Dist: prompt-toolkit
 Requires-Dist: tokenizers
-Requires-Dist: byzerllm[saas]>=0.1.168
+Requires-Dist: byzerllm[saas]>=0.1.169
 Requires-Dist: patch
 Requires-Dist: diff_match_patch
 Requires-Dist: GitPython

{auto_coder-0.1.271 → auto_coder-0.1.273}/src/auto_coder.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: auto-coder
-Version: 0.1.271
+Version: 0.1.273
 Summary: AutoCoder: AutoCoder
 Author: allwefantasy
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
@@ -26,7 +26,7 @@ Requires-Dist: tabulate
 Requires-Dist: jupyter_client
 Requires-Dist: prompt-toolkit
 Requires-Dist: tokenizers
-Requires-Dist: byzerllm[saas]>=0.1.168
+Requires-Dist: byzerllm[saas]>=0.1.169
 Requires-Dist: patch
 Requires-Dist: diff_match_patch
 Requires-Dist: GitPython

{auto_coder-0.1.271 → auto_coder-0.1.273}/src/auto_coder.egg-info/SOURCES.txt

@@ -129,6 +129,7 @@ src/autocoder/rag/simple_rag.py
 src/autocoder/rag/token_checker.py
 src/autocoder/rag/token_counter.py
 src/autocoder/rag/token_limiter.py
+src/autocoder/rag/token_limiter_utils.py
 src/autocoder/rag/types.py
 src/autocoder/rag/utils.py
 src/autocoder/rag/variable_holder.py

{auto_coder-0.1.271 → auto_coder-0.1.273}/src/auto_coder.egg-info/requires.txt

@@ -16,7 +16,7 @@ tabulate
 jupyter_client
 prompt-toolkit
 tokenizers
-byzerllm[saas]>=0.1.168
+byzerllm[saas]>=0.1.169
 patch
 diff_match_patch
 GitPython

{auto_coder-0.1.271 → auto_coder-0.1.273}/src/autocoder/auto_coder_runner.py

@@ -2133,7 +2133,7 @@ def manage_models(query: str):
         # Check duplication
         if any(m["name"] == data_dict["name"] for m in models_data):
             printer.print_in_terminal("models_add_model_exists", style="yellow", name=data_dict["name"])
-            result_manager.add_result(content=printer.get_message_from_key("models_add_model_exists",name=data_dict["name"]),meta={
+            result_manager.add_result(content=printer.get_message_from_key_with_format("models_add_model_exists",name=data_dict["name"]),meta={
                 "action": "models",
                 "input": {
                     "query": query
@@ -2177,7 +2177,7 @@ def manage_models(query: str):
         filtered_models = [m for m in models_data if m["name"] != name]
         if len(filtered_models) == len(models_data):
             printer.print_in_terminal("models_add_model_remove", style="yellow", name=name)
-            result_manager.add_result(content=printer.get_message_from_key("models_add_model_remove",name=name),meta={
+            result_manager.add_result(content=printer.get_message_from_key_with_format("models_add_model_remove",name=name),meta={
                 "action": "models",
                 "input": {
                     "query": query
@@ -2186,7 +2186,7 @@ def manage_models(query: str):
             return
         models_module.save_models(filtered_models)
         printer.print_in_terminal("models_add_model_removed", style="green", name=name)
-        result_manager.add_result(content=printer.get_message_from_key("models_add_model_removed",name=name),meta={ 
+        result_manager.add_result(content=printer.get_message_from_key_with_format("models_add_model_removed",name=name),meta={ 
             "action": "models",
             "input": {
                 "query": query
@@ -2194,7 +2194,7 @@ def manage_models(query: str):
         })
     else:
         printer.print_in_terminal("models_unknown_subcmd", style="yellow", subcmd=subcmd)
-        result_manager.add_result(content=printer.get_message_from_key("models_unknown_subcmd",subcmd=subcmd),meta={ 
+        result_manager.add_result(content=printer.get_message_from_key_with_format("models_unknown_subcmd",subcmd=subcmd),meta={ 
             "action": "models",
             "input": {
                 "query": query

{auto_coder-0.1.271 → auto_coder-0.1.273}/src/autocoder/common/auto_coder_lang.py

@@ -130,7 +130,7 @@ MESSAGES = {
         "quick_filter_too_long": "⚠️ index file is too large ({{ tokens_len }}/{{ max_tokens }}). The query will be split into {{ split_size }} chunks.",
         "quick_filter_tokens_len": "📊 Current index size: {{ tokens_len }} tokens",
         "estimated_chat_input_tokens": "Estimated chat input tokens: {{ estimated_input_tokens }}",
-        "estimated_input_tokens_in_generate": "Estimated input tokens in generate ({{ generate_mode }}): {{ estimated_input_tokens }}",        
+        "estimated_input_tokens_in_generate": "Estimated input tokens in generate ({{ generate_mode }}): {{ estimated_input_tokens_in_generate }}",        
         "model_has_access_restrictions": "{{model_name}} has access restrictions, cannot use the current function",
         "auto_command_not_found": "Auto command not found: {{command}}. Please check your input and try again.",
         "auto_command_failed": "Auto command failed: {{error}}. Please check your input and try again.",
@@ -319,7 +319,7 @@ MESSAGES = {
         "quick_filter_title": "{{ model_name }} 正在分析如何筛选上下文...",
         "quick_filter_failed": "❌ 快速过滤器失败: {{ error }}. ",
         "estimated_chat_input_tokens": "对话输入token预估为: {{ estimated_input_tokens }}",
-        "estimated_input_tokens_in_generate": "生成代码({{ generate_mode }})预计输入token数: {{ estimated_input_tokens }}",        
+        "estimated_input_tokens_in_generate": "生成代码({{ generate_mode }})预计输入token数: {{ estimated_input_tokens_in_generate }}",        
         "model_has_access_restrictions": "{{model_name}} 有访问限制，无法使用当前功能",
         "auto_command_not_found": "未找到自动命令: {{command}}。请检查您的输入并重试。",
         "auto_command_failed": "自动命令执行失败: {{error}}。请检查您的输入并重试。",

{auto_coder-0.1.271 → auto_coder-0.1.273}/src/autocoder/common/auto_configure.py

@@ -154,7 +154,7 @@ def config_readme() -> str:
     项目类型通常为如下三种选择：
     1. ts
     2. py
-    3. 代码文件后缀名列表（比如.java,.py,.go,.js,.ts），多个按逗号分割
+    3. 代码文件后缀名列表（比如.java,.py,.go,.js,.ts），多个按逗号分割    
 
     推荐使用 3 选项，因为项目类型通常为多种后缀名混合。
     """

{auto_coder-0.1.271 → auto_coder-0.1.273}/src/autocoder/index/filter/quick_filter.py

@@ -287,6 +287,7 @@ class QuickFilter():
         当用户提一个需求的时候，我们要找到两种类型的源码文件：
         1. 根据需求需要被修改的文件，我们叫 edited_files
         2. 为了能够完成修改这些文件，还需要的一些额外参考文件, 我们叫 reference_files
+        3. 因为修改了 edited_files 文件，可能有一些依赖 edited_files 的文件也需要被修改，我们叫 dependent_files
 
         现在，给定下面的索引文件：
 
@@ -315,9 +316,9 @@ class QuickFilter():
         ```
 
         特别注意:    
-        1. 如果用户的query里有 @文件 或者 @@符号，那么被@的文件或者@@的符号必须要返回。
-        2. 根据需求以及根据 @文件 或者 @@符号 找到的文件，猜测需要被修改的edited_files文件，然后尝试通过索引文件诸如导入语句等信息找到这些文件依赖的其他文件得到 reference_files。
-        3. file_list 里的文件序号，按被 @ 或者 @@ 文件，edited_files文件，reference_files文件的顺序排列。注意，reference_files 你要根据需求来猜测是否需要，过滤掉不相关的，避免返回文件数过多。
+        1. 如果用户的query里有 @文件 或者 @@符号，并且他们在索引文件中，那么被@的文件或者@@的符号必须要返回。
+        2. 根据需求以及根据 @文件 或者 @@符号 找到的文件，猜测需要被修改的edited_files文件，然后尝试通过索引文件诸如导入语句等信息找到这些文件依赖的其他文件得到 reference_files,dependent_files。
+        3. file_list 里的文件序号，按被 @ 或者 @@ 文件，edited_files文件，reference_files,dependent_files文件的顺序排列。注意，reference_files 你要根据需求来猜测是否需要，过滤掉不相关的，避免返回文件数过多。
         4. 如果 query 里是一段历史对话，那么对话里的内容提及的文件路径必须要返回。        
         5. 如果用户需求为空，则直接返回空列表即可。        
         6. 返回的 json格式数据不允许有注释

auto_coder-0.1.273/src/autocoder/rag/doc_filter.py

@@ -0,0 +1,281 @@
+import time
+from typing import List, Dict, Optional
+from loguru import logger
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+from autocoder.rag.relevant_utils import (
+    parse_relevance,
+    FilterDoc,
+    TaskTiming,
+    DocFilterResult
+)
+
+from autocoder.common import SourceCode, AutoCoderArgs
+from autocoder.rag.rag_config import RagConfigManager
+from byzerllm import ByzerLLM
+import byzerllm
+
+
+@byzerllm.prompt()
+def _check_relevance_with_conversation(
+    conversations: List[Dict[str, str]],
+    documents: List[str],
+    filter_config: Optional[str] = None,
+) -> str:
+    """
+    使用以下文档和对话历史来回答问题。如果文档中没有相关信息，请说"我没有足够的信息来回答这个问题"。
+
+    文档：
+    <documents>
+    {% for doc in documents %}
+    {{ doc }}
+    {% endfor %}
+    </documents>
+
+    对话历史：
+    <conversations>
+    {% for msg in conversations %}
+    <{{ msg.role }}>: {{ msg.content }}
+    {% endfor %}
+    </conversations>
+
+    {% if filter_config %}
+    一些提示：
+    {{ filter_config }}
+    {% endif %}
+
+    请结合提供的文档以及用户对话历史，判断提供的文档是不是能和用户的最后一个问题相关。
+    如果该文档提供的知识能够和用户的问题相关，那么请回复"yes/<relevant>" 否则回复"no/<relevant>"。
+    其中， <relevant> 是你认为文档中和问题的相关度，0-10之间的数字，数字越大表示相关度越高。
+    """
+
+class DocFilter:
+    def __init__(
+        self,
+        llm: ByzerLLM,
+        args: AutoCoderArgs,
+        on_ray: bool = False,
+        path: Optional[str] = None,
+    ):
+        self.llm = llm
+        if self.llm.get_sub_client("recall_model"):
+            self.recall_llm = self.llm.get_sub_client("recall_model")
+        else:
+            self.recall_llm = self.llm
+
+        self.args = args
+        self.relevant_score = self.args.rag_doc_filter_relevance
+        self.on_ray = on_ray
+        self.path = path
+
+    def filter_docs(
+        self, conversations: List[Dict[str, str]], documents: List[SourceCode]
+    ) -> DocFilterResult:
+        return self.filter_docs_with_threads(conversations, documents)
+
+    def filter_docs_with_threads(
+        self, conversations: List[Dict[str, str]], documents: List[SourceCode]
+    ) -> DocFilterResult:
+
+        start_time = time.time()
+        logger.info(f"=== DocFilter Starting ===")
+        logger.info(
+            f"Configuration: relevance_threshold={self.relevant_score}, thread_workers={self.args.index_filter_workers or 5}")
+
+        rag_manager = RagConfigManager(path=self.path)
+        rag_config = rag_manager.load_config()
+
+        documents = list(documents)
+        logger.info(f"Filtering {len(documents)} documents...")
+
+        submitted_tasks = 0
+        completed_tasks = 0
+        relevant_count = 0
+        model_name = self.recall_llm.default_model_name or "unknown"
+
+        with ThreadPoolExecutor(
+            max_workers=self.args.index_filter_workers or 5
+        ) as executor:
+            future_to_doc = {}
+
+            # 提交所有任务
+            for doc in documents:
+                submit_time = time.time()
+                submitted_tasks += 1
+
+                def _run(conversations, docs):
+                    submit_time_1 = time.time()
+                    meta = None
+                    try:
+                        llm = self.recall_llm
+                        meta_holder = byzerllm.MetaHolder()
+
+                        v = (
+                            _check_relevance_with_conversation.with_llm(
+                                llm).with_meta(meta_holder)
+                            .options({"llm_config": {"max_length": 10}})
+                            .run(
+                                conversations=conversations,
+                                documents=docs,
+                                filter_config=rag_config.filter_config,
+                            )
+                        )
+
+                        meta = meta_holder.get_meta_model()
+                    except Exception as e:
+                        logger.error(
+                            f"Error in _check_relevance_with_conversation: {str(e)}"
+                        )
+                        return (None, submit_time_1, time.time(), meta)
+
+                    end_time_2 = time.time()
+                    return (v, submit_time_1, end_time_2, meta)
+
+                m = executor.submit(
+                    _run,
+                    conversations,
+                    [f"##File: {doc.module_name}\n{doc.source_code}"],
+                )
+                future_to_doc[m] = (doc, submit_time)
+
+            logger.info(
+                f"Submitted {submitted_tasks} document filtering tasks to thread pool")
+
+            # 处理完成的任务
+            doc_filter_result = DocFilterResult(
+                docs=[],
+                raw_docs=[],
+                input_tokens_counts=[],
+                generated_tokens_counts=[],
+                durations=[],
+                model_name=model_name
+            )            
+            relevant_docs = doc_filter_result.docs
+            for future in as_completed(list(future_to_doc.keys())):
+                try:
+                    doc, submit_time = future_to_doc[future]
+                    end_time = time.time()
+                    completed_tasks += 1
+                    progress_percent = (completed_tasks / len(documents)) * 100
+
+                    v, submit_time_1, end_time_2, meta = future.result()
+                    task_timing = TaskTiming(
+                        submit_time=submit_time,
+                        end_time=end_time,
+                        duration=end_time - submit_time,
+                        real_start_time=submit_time_1,
+                        real_end_time=end_time_2,
+                        real_duration=end_time_2 - submit_time_1,
+                    )
+
+                    relevance = parse_relevance(v)
+                    is_relevant = relevance and relevance.relevant_score >= self.relevant_score
+
+                    if is_relevant:
+                        relevant_count += 1
+                        status_text = f"RELEVANT (Score: {relevance.relevant_score:.1f})"
+                    else:
+                        score_text = f"{relevance.relevant_score:.1f}" if relevance else "N/A"
+                        status_text = f"NOT RELEVANT (Score: {score_text})"
+
+                    queue_time = task_timing.real_start_time - task_timing.submit_time
+
+                    input_tokens_count = meta.input_tokens_count if meta else 0
+                    generated_tokens_count = meta.generated_tokens_count if meta else 0
+
+                    logger.info(
+                        f"Document filtering [{progress_percent:.1f}%] - {completed_tasks}/{len(documents)}:"
+                        f"\n  - File: {doc.module_name}"
+                        f"\n  - Status: {status_text}"
+                        f"\n  - Model: {model_name}"
+                        f"\n  - Threshold: {self.relevant_score}"
+                        f"\n  - Input tokens: {input_tokens_count}"
+                        f"\n  - Generated tokens: {generated_tokens_count}"
+                        f"\n  - Timing: Duration={task_timing.duration:.2f}s, Processing={task_timing.real_duration:.2f}s, Queue={queue_time:.2f}s"
+                        f"\n  - Response: {v}"
+                    )
+                    
+                    if "rag" not in doc.metadata:
+                        doc.metadata["rag"] = {}
+                    doc.metadata["rag"]["recall"] = {
+                        "input_tokens_count": input_tokens_count,
+                        "generated_tokens_count": generated_tokens_count,
+                        "recall_model": model_name,
+                        "duration": task_timing.real_duration                                                                                                            
+                    }
+                    
+                    doc_filter_result.input_tokens_counts.append(input_tokens_count)
+                    doc_filter_result.generated_tokens_counts.append(generated_tokens_count)
+                    doc_filter_result.durations.append(task_timing.real_duration)
+                    
+                    new_filter_doc = FilterDoc(
+                            source_code=doc,
+                            relevance=relevance,
+                            task_timing=task_timing,
+                        )
+                    
+                    doc_filter_result.raw_docs.append(new_filter_doc)
+
+                    if is_relevant:
+                        relevant_docs.append(
+                            new_filter_doc
+                        )
+                except Exception as exc:
+                    try:
+                        doc, submit_time = future_to_doc[future]
+                        completed_tasks += 1
+                        progress_percent = (
+                            completed_tasks / len(documents)) * 100
+                        logger.error(
+                            f"Document filtering [{progress_percent:.1f}%] - {completed_tasks}/{len(documents)}:"
+                            f"\n  - File: {doc.module_name}"
+                            f"\n  - Error: {exc}"
+                            f"\n  - Duration: {time.time() - submit_time:.2f}s"
+                        )
+                        doc_filter_result.raw_docs.append(
+                            FilterDoc(
+                                source_code=doc,
+                                relevance=None,
+                                task_timing=TaskTiming(),                                
+                            )
+                        )
+                    except Exception as e:
+                        logger.error(
+                            f"Document filtering error in task tracking: {exc}"
+                        )
+
+        # Sort relevant_docs by relevance score in descending order
+        relevant_docs.sort(
+            key=lambda x: x.relevance.relevant_score, reverse=True)
+
+        total_time = time.time() - start_time
+
+        avg_processing_time = sum(
+            doc.task_timing.real_duration for doc in relevant_docs) / len(relevant_docs) if relevant_docs else 0
+        avg_queue_time = sum(doc.task_timing.real_start_time -
+                             doc.task_timing.submit_time for doc in relevant_docs) / len(relevant_docs) if relevant_docs else 0
+        
+        total_input_tokens = sum(doc_filter_result.input_tokens_counts)
+        total_generated_tokens = sum(doc_filter_result.generated_tokens_counts)
+
+        logger.info(
+            f"=== DocFilter Complete ==="
+            f"\n  * Total time: {total_time:.2f}s"
+            f"\n  * Documents processed: {completed_tasks}/{len(documents)}"
+            f"\n  * Relevant documents: {relevant_count} (threshold: {self.relevant_score})"
+            f"\n  * Average processing time: {avg_processing_time:.2f}s"
+            f"\n  * Average queue time: {avg_queue_time:.2f}s"
+            f"\n  * Total input tokens: {total_input_tokens}"
+            f"\n  * Total generated tokens: {total_generated_tokens}"
+        )
+
+        if relevant_docs:
+            logger.info(
+                f"Top 5 relevant documents:"
+                + "".join([f"\n  * {doc.source_code.module_name} (Score: {doc.relevance.relevant_score:.1f})"
+                          for doc in relevant_docs[:5]])
+            )
+        else:
+            logger.warning("No relevant documents found!")
+
+        return doc_filter_result

{auto_coder-0.1.271 → auto_coder-0.1.273}/src/autocoder/rag/llm_wrapper.py

@@ -44,13 +44,15 @@ class LLWrapper:
         res,contexts = self.rag.stream_chat_oai(conversations,llm_config=llm_config)        
         for t in res:                        
             yield (t,SingleOutputMeta(0,0))
+    
 
     async def async_stream_chat_oai(self,conversations, 
                         model:Optional[str]=None, 
                         role_mapping=None,
                         delta_mode=False,
                         llm_config:Dict[str,Any]={}): 
-        res,contexts = await asyncfy_with_semaphore(lambda: self.rag.stream_chat_oai(conversations,llm_config=llm_config))()                
+        res,contexts = await asyncfy_with_semaphore(lambda: self.rag.stream_chat_oai(conversations,llm_config=llm_config))()
+        # res,contexts = await self.llm.async_stream_chat_oai(conversations,llm_config=llm_config)
         for t in res:                                    
             yield (t,SingleOutputMeta(0,0))
                          

{auto_coder-0.1.271 → auto_coder-0.1.273}/src/autocoder/rag/loaders/docx_loader.py

@@ -1,9 +1,9 @@
 from io import BytesIO
-import docx2txt
 from autocoder.utils._markitdown import MarkItDown
 import traceback
 
 def extract_text_from_docx_old(docx_path):
+    import docx2txt
     with open(docx_path, "rb") as f:
         docx_content = f.read()
     docx_file = BytesIO(docx_content)