auto-coder 0.1.175__tar.gz → 0.1.176__tar.gz

{auto-coder-0.1.175 → auto-coder-0.1.176}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: auto-coder
-Version: 0.1.175
+Version: 0.1.176
 Summary: AutoCoder: AutoCoder
 Author: allwefantasy
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence

{auto-coder-0.1.175 → auto-coder-0.1.176}/src/auto_coder.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: auto-coder
-Version: 0.1.175
+Version: 0.1.176
 Summary: AutoCoder: AutoCoder
 Author: allwefantasy
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence

{auto-coder-0.1.175 → auto-coder-0.1.176}/src/autocoder/auto_coder_rag.py

@@ -225,6 +225,11 @@ def main(input_args: Optional[List[str]] = None):
         action="store_true",
         help="Disable automatic window adaptation for documents",
     )
+    serve_parser.add_argument(
+        "--disable_segment_reorder",
+        action="store_true",
+        help="Disable reordering of document segments after retrieval",
+    )
 
     # Tools command
     tools_parser = subparsers.add_parser("tools", help="Various tools")

{auto-coder-0.1.175 → auto-coder-0.1.176}/src/autocoder/common/__init__.py

@@ -285,8 +285,9 @@ class AutoCoderArgs(pydantic.BaseModel):
 
     monitor_mode: bool = False
     disable_auto_window: bool = False
-    
-    description: Optional[str] = ""
+    disable_segment_reorder: bool = False
+    rag_doc_filter_relevance: int = 5
+    tokenizer_path: Optional[str] = None
     skip_confirm: Optional[bool] = False
     silence: Optional[bool] = False
     exclude_files: Optional[Union[str, List[str]]] = ""

{auto-coder-0.1.175 → auto-coder-0.1.176}/src/autocoder/rag/long_context_rag.py

@@ -411,6 +411,7 @@ class LongContextRAG:
                     segment_limit=self.segment_limit,
                     buff_limit=self.buff_limit,
                     llm=self.llm,
+                    disable_segment_reorder = self.args.disable_segment_reorder
                 )
                 final_relevant_docs = token_limiter.limit_tokens(
                     relevant_docs=relevant_docs,

{auto-coder-0.1.175 → auto-coder-0.1.176}/src/autocoder/rag/token_limiter.py

@@ -17,6 +17,7 @@ class TokenLimiter:
         segment_limit: int,
         buff_limit: int,
         llm,
+        disable_segment_reorder:bool
     ):
         self.count_tokens = count_tokens
         self.full_text_limit = full_text_limit
@@ -26,6 +27,7 @@ class TokenLimiter:
         self.first_round_full_docs = []
         self.second_round_extracted_docs = []
         self.sencond_round_time = 0
+        self.disable_segment_reorder = disable_segment_reorder
 
     @byzerllm.prompt()
     def extract_relevance_range_from_docs_with_conversation(
@@ -93,8 +95,34 @@ class TokenLimiter:
         token_count = 0
         doc_num_count = 0
 
+        reorder_relevant_docs = []
+        added_docs = set()
+        
+        ## 文档分段（单个文档过大）和重排序逻辑
+        ## 1. 背景：在检索过程中，许多文档被切割成多个段落（segments）
+        ## 2. 问题：这些segments在召回时因为是按相关分做了排序可能是乱序的，不符合原文顺序，会强化大模型的幻觉。
+        ## 3. 目标：重新排序这些segments，确保来自同一文档的segments保持连续且按正确顺序排列。
+        ## 4. 实现方案：
+        ##    a) 方案一（保留位置）：统一文档的不同segments 根据chunk_index 来置换位置
+        ##    b) 方案二（当前实现）：遍历文档，发现某文档的segment A，立即查找该文档的所有其他segments，
+        ##       对它们进行排序，并将排序后多个segments插入到当前的segment A 位置中。
+        ## TODO: 
+        ##     1. 未来根据参数决定是否开启重排以及重排的策略   
+        if not self.disable_segment_reorder:    
+            for doc in relevant_docs:
+                if doc.metadata.get('original_doc') and doc.metadata.get('chunk_index'):
+                    if doc.metadata['original_doc'] not in added_docs:
+                        original_doc = doc.metadata['original_doc']
+                        chunks = [d for d in relevant_docs if d.metadata.get('original_doc') == original_doc]
+                        chunks.sort(key=lambda x: x.metadata['chunk_index'])
+                        reorder_relevant_docs.extend(chunks)
+                        added_docs.add(original_doc)
+                elif doc not in added_docs:
+                    reorder_relevant_docs.append(doc)
+                    added_docs.add(doc.module_name)
+
         ## 非窗口分区实现
-        for doc in relevant_docs:
+        for doc in reorder_relevant_docs:
             doc_tokens = self.count_tokens(doc.source_code)
             doc_num_count += 1
             if token_count + doc_tokens <= self.full_text_limit + self.segment_limit:
@@ -104,12 +132,12 @@ class TokenLimiter:
                 break
 
         ## 如果窗口无法放下所有的相关文档，则需要分区
-        if len(final_relevant_docs) < len(relevant_docs):
+        if len(final_relevant_docs) < len(reorder_relevant_docs):
             ## 先填充full_text分区
             token_count = 0
             new_token_limit = self.full_text_limit
             doc_num_count = 0
-            for doc in relevant_docs:
+            for doc in reorder_relevant_docs:
                 doc_tokens = self.count_tokens(doc.source_code)
                 doc_num_count += 1
                 if token_count + doc_tokens <= new_token_limit:
@@ -130,7 +158,7 @@ class TokenLimiter:
 
             ## 继续填充segment分区
             sencond_round_start_time = time.time()
-            remaining_docs = relevant_docs[len(self.first_round_full_docs) :]
+            remaining_docs = reorder_relevant_docs[len(self.first_round_full_docs) :]
             logger.info(
                 f"first round docs: {len(self.first_round_full_docs)} remaining docs: {len(remaining_docs)} index_filter_workers: {index_filter_workers}"
             )

auto-coder-0.1.176/src/autocoder/version.py

@@ -0,0 +1 @@
+__version__ = "0.1.176"

auto-coder-0.1.175/src/autocoder/version.py

@@ -1 +0,0 @@
-__version__ = "0.1.175"