auto-coder 0.1.175__py3-none-any.whl → 0.1.177__py3-none-any.whl

{auto_coder-0.1.175.dist-info → auto_coder-0.1.177.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: auto-coder
-Version: 0.1.175
+Version: 0.1.177
 Summary: AutoCoder: AutoCoder
 Author: allwefantasy
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence

{auto_coder-0.1.175.dist-info → auto_coder-0.1.177.dist-info}/RECORD

@@ -1,13 +1,13 @@
 autocoder/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 autocoder/auto_coder.py,sha256=HmgKa_ZApFlCsqo6BvuVeCPuncBT_Dh29ayZxxGR6lo,32216
 autocoder/auto_coder_lang.py,sha256=4qIS1tbEI8mpbtt6ThppTwKOM6MLuJTWJdgs5jIDGE0,2301
-autocoder/auto_coder_rag.py,sha256=y-iZkb7Zw2JORbxGYaOoIhVQXBohGyUBFXN675qsgXg,11280
+autocoder/auto_coder_rag.py,sha256=V82EyeslAO2Z8qkMrwkyC11f1Cz6Ccjo9c867f0J_x8,11455
 autocoder/auto_coder_server.py,sha256=qRY88mkBnqSGFDcwYE5gwpe2WPhIw1nEH6LdbjCQhQk,20306
 autocoder/chat_auto_coder.py,sha256=i5xIuWlTqF0pJz8kXoa-_bW3Ic3SfCFvU2WJIMxrUHU,81798
 autocoder/chat_auto_coder_lang.py,sha256=QYtu5gWEQmWKVovR_qUZ8plySZarNFX_Onk-1vN9IiA,8524
 autocoder/command_args.py,sha256=ftWw6HnFUZPiQPt1oV-SfpHQe69XN3knaFy1lpROBcU,26854
 autocoder/lang.py,sha256=e-07rYTgimpxS8sm-AxKSmH4kKQX4N05YFHJBg9trVs,12598
-autocoder/version.py,sha256=7VuhzksZrpUCv5FBnLSYtIkaqKki-s7Gt9mfU67bB6A,23
+autocoder/version.py,sha256=jOsxKVrfpzRExZ84ji63BWpk4HGAHjIouu90Hi2jqGI,23
 autocoder/agent/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 autocoder/agent/auto_tool.py,sha256=DBzip-P_T6ZtT2eHexPcusmKYD0h7ufzp7TLwXAY10E,11554
 autocoder/agent/coder.py,sha256=dnITYHqkcOip8zV4lywbkYNH9w7Q3qyYaUArJ4WPrTs,866
@@ -17,7 +17,7 @@ autocoder/agent/project_reader.py,sha256=-MWRqsr7O4mvU0PIpAhOUBb29htZAvA37pa_GeE
 autocoder/chat/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 autocoder/common/JupyterClient.py,sha256=O-wi6pXeAEYhAY24kDa0BINrLYvKS6rKyWe98pDClS0,2816
 autocoder/common/ShellClient.py,sha256=fM1q8t_XMSbLBl2zkCNC2J9xuyKN3eXzGm6hHhqL2WY,2286
-autocoder/common/__init__.py,sha256=FB0MdcGtI60-jPWXurNvOS9dZUZCg6HmgzHNVuHiW6c,10076
+autocoder/common/__init__.py,sha256=wKrFLZk9BMl755nL1gvPjXU-3uWKEnYBP8xsObIjM4g,10156
 autocoder/common/anything2images.py,sha256=0ILBbWzY02M-CiWB-vzuomb_J1hVdxRcenAfIrAXq9M,25283
 autocoder/common/audio.py,sha256=Kn9nWKQddWnUrAz0a_ZUgjcu4VUU_IcZBigT7n3N3qc,7439
 autocoder/common/cleaner.py,sha256=NU72i8C6o9m0vXExab7nao5bstBUsfJFcj11cXa9l4U,1089
@@ -60,9 +60,9 @@ autocoder/pyproject/__init__.py,sha256=-2-ImQVw6e3NQZQOyDlHEP5b4xVs5ur2G5izB-JCa
 autocoder/rag/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 autocoder/rag/api_server.py,sha256=zokIlDJlk7ucRorSLQm80uICO1mecfmn4J2zVqEBskE,6786
 autocoder/rag/doc_filter.py,sha256=LqU8Wi6klwpY9WTHVtkioSHpmo9IWhRz39dzV1gvp6E,9315
-autocoder/rag/document_retriever.py,sha256=plwm8BpC55VJTUWCZyG4HsXYm-niqUsXaBMDLrLgYj0,23348
+autocoder/rag/document_retriever.py,sha256=itypkUdY9vUITMAGhxHvWe-IZpxLi5h0A1mJuIVW6QA,23406
 autocoder/rag/llm_wrapper.py,sha256=xRbTBpLUH43Ah5jplL8WWWU-kjKfNgEJoUntLGBq5F4,2484
-autocoder/rag/long_context_rag.py,sha256=qKuXBuwuI_eoBh7XNApbDfVYtG4sPqBbh3s7yscIUuI,19391
+autocoder/rag/long_context_rag.py,sha256=n3HkjIYZqjrHHLcGDb1u0SVEae7qChu6py30x-Jp-KQ,20160
 autocoder/rag/rag_config.py,sha256=8LwFcTd8OJWWwi1_WY4IzjqgtT6RyE2j4PjxS5cCTDE,802
 autocoder/rag/rag_entry.py,sha256=V1RJ8RGqM30DNPmzymv64rZjNRGWn6kfc8sRy_LECg0,2451
 autocoder/rag/raw_rag.py,sha256=yS2Ur6kG0IRjhCj2_VonwxjY_xls_E62jO5Gz5j2nqE,2952
@@ -71,7 +71,7 @@ autocoder/rag/simple_directory_reader.py,sha256=LkKreCkNdEOoL4fNhc3_hDoyyWTQUte4
 autocoder/rag/simple_rag.py,sha256=I902EUqOK1WM0Y2WFd7RzDJYofElvTZNLVCBtX5A9rc,14885
 autocoder/rag/token_checker.py,sha256=jc76x6KWmvVxds6W8juZfQGaoErudc2HenG3sNQfSLs,2819
 autocoder/rag/token_counter.py,sha256=9ujfI5xQvwzKpN9XFWQGnXpm0h1sL7kgIJxgposcxNo,2096
-autocoder/rag/token_limiter.py,sha256=dGSjKWwP_3rMwr8Yq06xqK2BrHpxW8Trn1gQLfnDOA8,8749
+autocoder/rag/token_limiter.py,sha256=4cGy2kFCvbsM5CtONfuvLmXgpK_1HZTHehLTob08eks,10959
 autocoder/rag/types.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 autocoder/rag/variable_holder.py,sha256=pDayuCnlKj7-bkn4iUHX5gea9UObddbi3ZnXotmxCs4,45
 autocoder/rag/loaders/__init__.py,sha256=EQHEZ5Cmz-mGP2SllUTvcIbYCnF7W149dNpNItfs0yE,304
@@ -95,9 +95,9 @@ autocoder/utils/request_event_queue.py,sha256=r3lo5qGsB1dIjzVQ05dnr0z_9Z3zOkBdP1
 autocoder/utils/request_queue.py,sha256=nwp6PMtgTCiuwJI24p8OLNZjUiprC-TsefQrhMI-yPE,3889
 autocoder/utils/rest.py,sha256=3tXA8KZG6jKz_tddHNLGx77Icee88WcUeesfNsgPno4,8790
 autocoder/utils/tests.py,sha256=BqphrwyycGAvs-5mhH8pKtMZdObwhFtJ5MC_ZAOiLq8,1340
-auto_coder-0.1.175.dist-info/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
-auto_coder-0.1.175.dist-info/METADATA,sha256=JrgeRETDy_kU_7g-1UNJfKkxWYlSgamCpiihAYl04Yw,2352
-auto_coder-0.1.175.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-auto_coder-0.1.175.dist-info/entry_points.txt,sha256=0nzHtHH4pNcM7xq4EBA2toS28Qelrvcbrr59GqD_0Ak,350
-auto_coder-0.1.175.dist-info/top_level.txt,sha256=Jqc0_uJSw2GwoFQAa9iJxYns-2mWla-9ok_Y3Gcznjk,10
-auto_coder-0.1.175.dist-info/RECORD,,
+auto_coder-0.1.177.dist-info/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
+auto_coder-0.1.177.dist-info/METADATA,sha256=IBRPsgVVwqup2c8xX7d5xeLh2ay5kIiPBrxQthG2JBM,2352
+auto_coder-0.1.177.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+auto_coder-0.1.177.dist-info/entry_points.txt,sha256=0nzHtHH4pNcM7xq4EBA2toS28Qelrvcbrr59GqD_0Ak,350
+auto_coder-0.1.177.dist-info/top_level.txt,sha256=Jqc0_uJSw2GwoFQAa9iJxYns-2mWla-9ok_Y3Gcznjk,10
+auto_coder-0.1.177.dist-info/RECORD,,

autocoder/auto_coder_rag.py

@@ -225,6 +225,11 @@ def main(input_args: Optional[List[str]] = None):
         action="store_true",
         help="Disable automatic window adaptation for documents",
     )
+    serve_parser.add_argument(
+        "--disable_segment_reorder",
+        action="store_true",
+        help="Disable reordering of document segments after retrieval",
+    )
 
     # Tools command
     tools_parser = subparsers.add_parser("tools", help="Various tools")

autocoder/common/__init__.py

@@ -285,8 +285,9 @@ class AutoCoderArgs(pydantic.BaseModel):
 
     monitor_mode: bool = False
     disable_auto_window: bool = False
-    
-    description: Optional[str] = ""
+    disable_segment_reorder: bool = False
+    rag_doc_filter_relevance: int = 5
+    tokenizer_path: Optional[str] = None
     skip_confirm: Optional[bool] = False
     silence: Optional[bool] = False
     exclude_files: Optional[Union[str, List[str]]] = ""

autocoder/rag/document_retriever.py

@@ -159,6 +159,8 @@ def process_file_local(file_path: str) -> List[SourceCode]:
         return v
     except Exception as e:
         logger.error(f"Error processing file {file_path}: {str(e)}")
+        import traceback
+        traceback.print_exc()
         return []
 
 
@@ -392,7 +394,7 @@ class AutoCoderRAGAsyncUpdateQueue:
             elif isinstance(file_list, AddOrUpdateEvent):
                 for file_info in file_list.file_infos:
                     logger.info(f"{file_info[0]} is detected to be updated")
-                    result = process_file_local(file_info)
+                    result = process_file_local(file_info[0])
                     self.update_cache(file_info, result)
 
             self.write_cache()

autocoder/rag/long_context_rag.py

@@ -7,7 +7,6 @@ import byzerllm
 import pandas as pd
 import pathspec
 from byzerllm import ByzerLLM
-from jinja2 import Template
 from loguru import logger
 from openai import OpenAI
 from rich.console import Console
@@ -30,6 +29,11 @@ from autocoder.rag.token_limiter import TokenLimiter
 from tokenizers import Tokenizer
 from autocoder.rag import variable_holder
 
+try:
+    from autocoder_pro.rag.llm_compute import LLMComputeEngine
+except ImportError:
+    LLMComputeEngine = None
+
 
 class LongContextRAG:
     def __init__(
@@ -119,13 +123,13 @@ class LongContextRAG:
             self.monitor_mode,
             ## 确保全文区至少能放下一个文件
             single_file_token_limit=self.full_text_limit - 100,
-            disable_auto_window=self.args.disable_auto_window
+            disable_auto_window=self.args.disable_auto_window,
         )
 
         self.doc_filter = DocFilter(
             self.index_model, self.args, on_ray=self.on_ray, path=self.path
         )
-        
+
         doc_num = 0
         token_num = 0
         token_counts = []
@@ -151,7 +155,7 @@ class LongContextRAG:
             f"  Max doc tokens:    {max(token_counts) if token_counts else 0}\n"
             f"  Min doc tokens:    {min(token_counts) if token_counts else 0}\n"
             f"  Avg doc tokens:    {avg_tokens:.2f}\n"
-            f"  Median doc tokens: {median_tokens:.2f}\n"                        
+            f"  Median doc tokens: {median_tokens:.2f}\n"
         )
 
     def count_tokens(self, text: str) -> int:
@@ -391,8 +395,13 @@ class LongContextRAG:
             relevant_docs_info = []
             for doc in relevant_docs:
                 info = f"- {doc.module_name.replace(self.path,'',1)}"
-                if 'original_docs' in doc.metadata:
-                    original_docs = ", ".join([doc.replace(self.path,"",1) for doc in doc.metadata['original_docs']])                                                                    
+                if "original_docs" in doc.metadata:
+                    original_docs = ", ".join(
+                        [
+                            doc.replace(self.path, "", 1)
+                            for doc in doc.metadata["original_docs"]
+                        ]
+                    )
                     info += f" (Original docs: {original_docs})"
                 relevant_docs_info.append(info)
 
@@ -411,6 +420,7 @@ class LongContextRAG:
                     segment_limit=self.segment_limit,
                     buff_limit=self.buff_limit,
                     llm=self.llm,
+                    disable_segment_reorder=self.args.disable_segment_reorder,
                 )
                 final_relevant_docs = token_limiter.limit_tokens(
                     relevant_docs=relevant_docs,
@@ -442,13 +452,20 @@ class LongContextRAG:
 
             # Add relevant docs information
             final_relevant_docs_info = []
-            for doc in relevant_docs:                
+            for doc in relevant_docs:
                 info = f"- {doc.module_name.replace(self.path,'',1)}"
-                if 'original_docs' in doc.metadata:                    
-                    original_docs = ", ".join([doc.replace(self.path,"",1) for doc in doc.metadata['original_docs']])                                                
+                if "original_docs" in doc.metadata:
+                    original_docs = ", ".join(
+                        [
+                            doc.replace(self.path, "", 1)
+                            for doc in doc.metadata["original_docs"]
+                        ]
+                    )
                     info += f" (Original docs: {original_docs})"
                 if "chunk_ranges" in doc.metadata:
-                    chunk_ranges = json.dumps(doc.metadata['chunk_ranges'],ensure_ascii=False)
+                    chunk_ranges = json.dumps(
+                        doc.metadata["chunk_ranges"], ensure_ascii=False
+                    )
                     info += f" (Chunk ranges: {chunk_ranges})"
                 final_relevant_docs_info.append(info)
 
@@ -464,10 +481,29 @@ class LongContextRAG:
 
             # Log the panel using rich
             console.print(panel)
-            
+
             request_tokens = sum([doc.tokens for doc in relevant_docs])
-            target_model = model or self.llm.default_model_name 
-            logger.info(f"Start to send to model {target_model} with {request_tokens} tokens")
+            target_model = model or self.llm.default_model_name
+            logger.info(
+                f"Start to send to model {target_model} with {request_tokens} tokens"
+            )
+
+            if LLMComputeEngine is not None:
+                llm_compute_engine = LLMComputeEngine(llm=self.llm)
+                new_conversations = llm_compute_engine.process_conversation(
+                    conversations, query, [doc.source_code for doc in relevant_docs]
+                )
+
+                return (
+                    llm_compute_engine.stream_chat_oai(
+                        conversations=new_conversations,
+                        model=model,
+                        role_mapping=role_mapping,
+                        llm_config=llm_config,
+                        delta_mode=True,
+                    ),
+                    context,
+                )
 
             new_conversations = conversations[:-1] + [
                 {
@@ -479,10 +515,6 @@ class LongContextRAG:
                 }
             ]
 
-            # # 将 new_conversations 转化为 JSON 并写入文件
-            # with open('/tmp/rag.json', 'w', encoding='utf-8') as f:
-            #     json.dump(new_conversations, f, ensure_ascii=False, indent=2)
-
             chunks = self.llm.stream_chat_oai(
                 conversations=new_conversations,
                 model=model,
@@ -490,4 +522,5 @@ class LongContextRAG:
                 llm_config=llm_config,
                 delta_mode=True,
             )
+
             return (chunk[0] for chunk in chunks), context

autocoder/rag/token_limiter.py

@@ -17,6 +17,7 @@ class TokenLimiter:
         segment_limit: int,
         buff_limit: int,
         llm,
+        disable_segment_reorder: bool,
     ):
         self.count_tokens = count_tokens
         self.full_text_limit = full_text_limit
@@ -26,6 +27,7 @@ class TokenLimiter:
         self.first_round_full_docs = []
         self.second_round_extracted_docs = []
         self.sencond_round_time = 0
+        self.disable_segment_reorder = disable_segment_reorder
 
     @byzerllm.prompt()
     def extract_relevance_range_from_docs_with_conversation(
@@ -93,8 +95,46 @@ class TokenLimiter:
         token_count = 0
         doc_num_count = 0
 
+        reorder_relevant_docs = []
+
+        ## 文档分段（单个文档过大）和重排序逻辑
+        ## 1. 背景：在检索过程中，许多文档被切割成多个段落（segments）
+        ## 2. 问题：这些segments在召回时因为是按相关分做了排序可能是乱序的，不符合原文顺序，会强化大模型的幻觉。
+        ## 3. 目标：重新排序这些segments，确保来自同一文档的segments保持连续且按正确顺序排列。
+        ## 4. 实现方案：
+        ##    a) 方案一（保留位置）：统一文档的不同segments 根据chunk_index 来置换位置
+        ##    b) 方案二（当前实现）：遍历文档，发现某文档的segment A，立即查找该文档的所有其他segments，
+        ##       对它们进行排序，并将排序后多个segments插入到当前的segment A 位置中。
+        ## TODO:
+        ##     1. 未来根据参数决定是否开启重排以及重排的策略
+        if not self.disable_segment_reorder:
+            num_count = 0
+            for doc in relevant_docs:
+                num_count += 1
+                reorder_relevant_docs.append(doc)
+                if "original_doc" in doc.metadata and "chunk_index" in doc.metadata:
+                    original_doc_name = doc.metadata["original_doc"].module_name
+
+                    temp_docs = []
+                    for temp_doc in relevant_docs[num_count:]:
+                        if (
+                            "original_doc" in temp_doc.metadata
+                            and "chunk_index" in temp_doc.metadata
+                        ):
+                            if (
+                                temp_doc.metadata["original_doc"].module_name
+                                == original_doc_name
+                            ):
+                                if temp_doc not in reorder_relevant_docs:
+                                    temp_docs.append(temp_doc)
+
+                    temp_docs.sort(key=lambda x: x.metadata["chunk_index"])
+                    reorder_relevant_docs.extend(temp_docs)
+        else:
+            reorder_relevant_docs = relevant_docs
+
         ## 非窗口分区实现
-        for doc in relevant_docs:
+        for doc in reorder_relevant_docs:
             doc_tokens = self.count_tokens(doc.source_code)
             doc_num_count += 1
             if token_count + doc_tokens <= self.full_text_limit + self.segment_limit:
@@ -104,12 +144,12 @@ class TokenLimiter:
                 break
 
         ## 如果窗口无法放下所有的相关文档，则需要分区
-        if len(final_relevant_docs) < len(relevant_docs):
+        if len(final_relevant_docs) < len(reorder_relevant_docs):
             ## 先填充full_text分区
             token_count = 0
             new_token_limit = self.full_text_limit
             doc_num_count = 0
-            for doc in relevant_docs:
+            for doc in reorder_relevant_docs:
                 doc_tokens = self.count_tokens(doc.source_code)
                 doc_num_count += 1
                 if token_count + doc_tokens <= new_token_limit:
@@ -130,7 +170,7 @@ class TokenLimiter:
 
             ## 继续填充segment分区
             sencond_round_start_time = time.time()
-            remaining_docs = relevant_docs[len(self.first_round_full_docs) :]
+            remaining_docs = reorder_relevant_docs[len(self.first_round_full_docs) :]
             logger.info(
                 f"first round docs: {len(self.first_round_full_docs)} remaining docs: {len(remaining_docs)} index_filter_workers: {index_filter_workers}"
             )
@@ -219,7 +259,5 @@ class TokenLimiter:
                         f"Failed to process doc {doc.module_name} after {max_retries} attempts: {str(e)}"
                     )
                     return SourceCode(
-                        module_name=doc.module_name,
-                        source_code="",   
-                        tokens= 0                     
+                        module_name=doc.module_name, source_code="", tokens=0
                     )

autocoder/version.py

@@ -1 +1 @@
-__version__ = "0.1.175"
+__version__ = "0.1.177"