auto-coder 0.1.176__py3-none-any.whl → 0.1.178__py3-none-any.whl

{auto_coder-0.1.176.dist-info → auto_coder-0.1.178.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: auto-coder
-Version: 0.1.176
+Version: 0.1.178
 Summary: AutoCoder: AutoCoder
 Author: allwefantasy
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence

{auto_coder-0.1.176.dist-info → auto_coder-0.1.178.dist-info}/RECORD

@@ -7,7 +7,7 @@ autocoder/chat_auto_coder.py,sha256=i5xIuWlTqF0pJz8kXoa-_bW3Ic3SfCFvU2WJIMxrUHU,
 autocoder/chat_auto_coder_lang.py,sha256=QYtu5gWEQmWKVovR_qUZ8plySZarNFX_Onk-1vN9IiA,8524
 autocoder/command_args.py,sha256=ftWw6HnFUZPiQPt1oV-SfpHQe69XN3knaFy1lpROBcU,26854
 autocoder/lang.py,sha256=e-07rYTgimpxS8sm-AxKSmH4kKQX4N05YFHJBg9trVs,12598
-autocoder/version.py,sha256=yiACry4Tn-v8T0DYTTygfQmb9WG4pVkXXkB6IB4a1yg,23
+autocoder/version.py,sha256=EFm__1MLoFEQx_MuXleZfEID925EbP4uHw_aNT8swgI,23
 autocoder/agent/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 autocoder/agent/auto_tool.py,sha256=DBzip-P_T6ZtT2eHexPcusmKYD0h7ufzp7TLwXAY10E,11554
 autocoder/agent/coder.py,sha256=dnITYHqkcOip8zV4lywbkYNH9w7Q3qyYaUArJ4WPrTs,866
@@ -60,9 +60,9 @@ autocoder/pyproject/__init__.py,sha256=-2-ImQVw6e3NQZQOyDlHEP5b4xVs5ur2G5izB-JCa
 autocoder/rag/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 autocoder/rag/api_server.py,sha256=zokIlDJlk7ucRorSLQm80uICO1mecfmn4J2zVqEBskE,6786
 autocoder/rag/doc_filter.py,sha256=LqU8Wi6klwpY9WTHVtkioSHpmo9IWhRz39dzV1gvp6E,9315
-autocoder/rag/document_retriever.py,sha256=plwm8BpC55VJTUWCZyG4HsXYm-niqUsXaBMDLrLgYj0,23348
+autocoder/rag/document_retriever.py,sha256=itypkUdY9vUITMAGhxHvWe-IZpxLi5h0A1mJuIVW6QA,23406
 autocoder/rag/llm_wrapper.py,sha256=xRbTBpLUH43Ah5jplL8WWWU-kjKfNgEJoUntLGBq5F4,2484
-autocoder/rag/long_context_rag.py,sha256=626f5-XFyTxmnbUJ_a9GiaMPuqWhTDVMcg0b0ePW_mQ,19471
+autocoder/rag/long_context_rag.py,sha256=FW4Qv7YpNXOMrNbe28p3hY-baes6nwyVmZ7Qq73J-uc,20173
 autocoder/rag/rag_config.py,sha256=8LwFcTd8OJWWwi1_WY4IzjqgtT6RyE2j4PjxS5cCTDE,802
 autocoder/rag/rag_entry.py,sha256=V1RJ8RGqM30DNPmzymv64rZjNRGWn6kfc8sRy_LECg0,2451
 autocoder/rag/raw_rag.py,sha256=yS2Ur6kG0IRjhCj2_VonwxjY_xls_E62jO5Gz5j2nqE,2952
@@ -70,10 +70,10 @@ autocoder/rag/relevant_utils.py,sha256=OGfp98OXG4jr3jNmtHIeXGPF8mOlIbTnolPIVTZzY
 autocoder/rag/simple_directory_reader.py,sha256=LkKreCkNdEOoL4fNhc3_hDoyyWTQUte4uqextISRz4U,24485
 autocoder/rag/simple_rag.py,sha256=I902EUqOK1WM0Y2WFd7RzDJYofElvTZNLVCBtX5A9rc,14885
 autocoder/rag/token_checker.py,sha256=jc76x6KWmvVxds6W8juZfQGaoErudc2HenG3sNQfSLs,2819
-autocoder/rag/token_counter.py,sha256=9ujfI5xQvwzKpN9XFWQGnXpm0h1sL7kgIJxgposcxNo,2096
-autocoder/rag/token_limiter.py,sha256=nUxaaKJTWEi4J5c5Tz4BkwU4G1B74VxLlMinqu5s41A,10660
+autocoder/rag/token_counter.py,sha256=LReZEYXmWriDI3KYjUvK0E4Gn4MeDJX6RPCfJCmluGY,2110
+autocoder/rag/token_limiter.py,sha256=4cGy2kFCvbsM5CtONfuvLmXgpK_1HZTHehLTob08eks,10959
 autocoder/rag/types.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-autocoder/rag/variable_holder.py,sha256=pDayuCnlKj7-bkn4iUHX5gea9UObddbi3ZnXotmxCs4,45
+autocoder/rag/variable_holder.py,sha256=PFvBjFcR7-fNDD4Vcsc8CpH2Te057vcpwJMxtrfUgKI,75
 autocoder/rag/loaders/__init__.py,sha256=EQHEZ5Cmz-mGP2SllUTvcIbYCnF7W149dNpNItfs0yE,304
 autocoder/rag/loaders/docx_loader.py,sha256=g6Ta8rMUbfgwB8N1qiajhyO6wpaWl7zygAZiKShuioI,174
 autocoder/rag/loaders/excel_loader.py,sha256=Ue8YB1z_kBs8SjIPuBskyM08Q1JiONs_BJZPrzi59oo,896
@@ -95,9 +95,9 @@ autocoder/utils/request_event_queue.py,sha256=r3lo5qGsB1dIjzVQ05dnr0z_9Z3zOkBdP1
 autocoder/utils/request_queue.py,sha256=nwp6PMtgTCiuwJI24p8OLNZjUiprC-TsefQrhMI-yPE,3889
 autocoder/utils/rest.py,sha256=3tXA8KZG6jKz_tddHNLGx77Icee88WcUeesfNsgPno4,8790
 autocoder/utils/tests.py,sha256=BqphrwyycGAvs-5mhH8pKtMZdObwhFtJ5MC_ZAOiLq8,1340
-auto_coder-0.1.176.dist-info/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
-auto_coder-0.1.176.dist-info/METADATA,sha256=-Jm1GW-7-Htzi_6l3MGRGTvl0ytk1ZyMGB2ZpiZoYa8,2352
-auto_coder-0.1.176.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-auto_coder-0.1.176.dist-info/entry_points.txt,sha256=0nzHtHH4pNcM7xq4EBA2toS28Qelrvcbrr59GqD_0Ak,350
-auto_coder-0.1.176.dist-info/top_level.txt,sha256=Jqc0_uJSw2GwoFQAa9iJxYns-2mWla-9ok_Y3Gcznjk,10
-auto_coder-0.1.176.dist-info/RECORD,,
+auto_coder-0.1.178.dist-info/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
+auto_coder-0.1.178.dist-info/METADATA,sha256=X77KLl-OBGY5OArDncTSTJIpfln9pBwS_VOHExh6DqI,2352
+auto_coder-0.1.178.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+auto_coder-0.1.178.dist-info/entry_points.txt,sha256=0nzHtHH4pNcM7xq4EBA2toS28Qelrvcbrr59GqD_0Ak,350
+auto_coder-0.1.178.dist-info/top_level.txt,sha256=Jqc0_uJSw2GwoFQAa9iJxYns-2mWla-9ok_Y3Gcznjk,10
+auto_coder-0.1.178.dist-info/RECORD,,

autocoder/rag/document_retriever.py

@@ -159,6 +159,8 @@ def process_file_local(file_path: str) -> List[SourceCode]:
         return v
     except Exception as e:
         logger.error(f"Error processing file {file_path}: {str(e)}")
+        import traceback
+        traceback.print_exc()
         return []
 
 
@@ -392,7 +394,7 @@ class AutoCoderRAGAsyncUpdateQueue:
             elif isinstance(file_list, AddOrUpdateEvent):
                 for file_info in file_list.file_infos:
                     logger.info(f"{file_info[0]} is detected to be updated")
-                    result = process_file_local(file_info)
+                    result = process_file_local(file_info[0])
                     self.update_cache(file_info, result)
 
             self.write_cache()

autocoder/rag/long_context_rag.py

@@ -7,7 +7,6 @@ import byzerllm
 import pandas as pd
 import pathspec
 from byzerllm import ByzerLLM
-from jinja2 import Template
 from loguru import logger
 from openai import OpenAI
 from rich.console import Console
@@ -28,7 +27,12 @@ from autocoder.rag.token_checker import check_token_limit
 from autocoder.rag.token_counter import RemoteTokenCounter, TokenCounter
 from autocoder.rag.token_limiter import TokenLimiter
 from tokenizers import Tokenizer
-from autocoder.rag import variable_holder
+from autocoder.rag.variable_holder import VariableHolder
+
+try:
+    from autocoder_pro.rag.llm_compute import LLMComputeEngine
+except ImportError:
+    LLMComputeEngine = None
 
 
 class LongContextRAG:
@@ -68,8 +72,8 @@ class LongContextRAG:
         self.on_ray = False
 
         if self.tokenizer_path:
-            variable_holder.TOKENIZER_PATH = self.tokenizer_path
-            variable_holder.TOKENIZER_MODEL = Tokenizer.from_file(self.tokenizer_path)
+            VariableHolder.TOKENIZER_PATH = self.tokenizer_path
+            VariableHolder.TOKENIZER_MODEL = Tokenizer.from_file(self.tokenizer_path)
             self.tokenizer = TokenCounter(self.tokenizer_path)
         else:
             if llm.is_model_exist("deepseek_tokenizer"):
@@ -119,13 +123,13 @@ class LongContextRAG:
             self.monitor_mode,
             ## 确保全文区至少能放下一个文件
             single_file_token_limit=self.full_text_limit - 100,
-            disable_auto_window=self.args.disable_auto_window
+            disable_auto_window=self.args.disable_auto_window,
         )
 
         self.doc_filter = DocFilter(
             self.index_model, self.args, on_ray=self.on_ray, path=self.path
         )
-        
+
         doc_num = 0
         token_num = 0
         token_counts = []
@@ -151,7 +155,7 @@ class LongContextRAG:
             f"  Max doc tokens:    {max(token_counts) if token_counts else 0}\n"
             f"  Min doc tokens:    {min(token_counts) if token_counts else 0}\n"
             f"  Avg doc tokens:    {avg_tokens:.2f}\n"
-            f"  Median doc tokens: {median_tokens:.2f}\n"                        
+            f"  Median doc tokens: {median_tokens:.2f}\n"
         )
 
     def count_tokens(self, text: str) -> int:
@@ -391,8 +395,13 @@ class LongContextRAG:
             relevant_docs_info = []
             for doc in relevant_docs:
                 info = f"- {doc.module_name.replace(self.path,'',1)}"
-                if 'original_docs' in doc.metadata:
-                    original_docs = ", ".join([doc.replace(self.path,"",1) for doc in doc.metadata['original_docs']])                                                                    
+                if "original_docs" in doc.metadata:
+                    original_docs = ", ".join(
+                        [
+                            doc.replace(self.path, "", 1)
+                            for doc in doc.metadata["original_docs"]
+                        ]
+                    )
                     info += f" (Original docs: {original_docs})"
                 relevant_docs_info.append(info)
 
@@ -411,7 +420,7 @@ class LongContextRAG:
                     segment_limit=self.segment_limit,
                     buff_limit=self.buff_limit,
                     llm=self.llm,
-                    disable_segment_reorder = self.args.disable_segment_reorder
+                    disable_segment_reorder=self.args.disable_segment_reorder,
                 )
                 final_relevant_docs = token_limiter.limit_tokens(
                     relevant_docs=relevant_docs,
@@ -443,13 +452,20 @@ class LongContextRAG:
 
             # Add relevant docs information
             final_relevant_docs_info = []
-            for doc in relevant_docs:                
+            for doc in relevant_docs:
                 info = f"- {doc.module_name.replace(self.path,'',1)}"
-                if 'original_docs' in doc.metadata:                    
-                    original_docs = ", ".join([doc.replace(self.path,"",1) for doc in doc.metadata['original_docs']])                                                
+                if "original_docs" in doc.metadata:
+                    original_docs = ", ".join(
+                        [
+                            doc.replace(self.path, "", 1)
+                            for doc in doc.metadata["original_docs"]
+                        ]
+                    )
                     info += f" (Original docs: {original_docs})"
                 if "chunk_ranges" in doc.metadata:
-                    chunk_ranges = json.dumps(doc.metadata['chunk_ranges'],ensure_ascii=False)
+                    chunk_ranges = json.dumps(
+                        doc.metadata["chunk_ranges"], ensure_ascii=False
+                    )
                     info += f" (Chunk ranges: {chunk_ranges})"
                 final_relevant_docs_info.append(info)
 
@@ -465,10 +481,29 @@ class LongContextRAG:
 
             # Log the panel using rich
             console.print(panel)
-            
+
             request_tokens = sum([doc.tokens for doc in relevant_docs])
-            target_model = model or self.llm.default_model_name 
-            logger.info(f"Start to send to model {target_model} with {request_tokens} tokens")
+            target_model = model or self.llm.default_model_name
+            logger.info(
+                f"Start to send to model {target_model} with {request_tokens} tokens"
+            )
+
+            if LLMComputeEngine is not None:
+                llm_compute_engine = LLMComputeEngine(llm=self.llm)
+                new_conversations = llm_compute_engine.process_conversation(
+                    conversations, query, [doc.source_code for doc in relevant_docs]
+                )
+
+                return (
+                    llm_compute_engine.stream_chat_oai(
+                        conversations=new_conversations,
+                        model=model,
+                        role_mapping=role_mapping,
+                        llm_config=llm_config,
+                        delta_mode=True,
+                    ),
+                    context,
+                )
 
             new_conversations = conversations[:-1] + [
                 {
@@ -480,10 +515,6 @@ class LongContextRAG:
                 }
             ]
 
-            # # 将 new_conversations 转化为 JSON 并写入文件
-            # with open('/tmp/rag.json', 'w', encoding='utf-8') as f:
-            #     json.dump(new_conversations, f, ensure_ascii=False, indent=2)
-
             chunks = self.llm.stream_chat_oai(
                 conversations=new_conversations,
                 model=model,
@@ -491,4 +522,5 @@ class LongContextRAG:
                 llm_config=llm_config,
                 delta_mode=True,
             )
+
             return (chunk[0] for chunk in chunks), context

autocoder/rag/token_counter.py

@@ -2,7 +2,7 @@ import time
 from loguru import logger
 from tokenizers import Tokenizer
 from multiprocessing import Pool, cpu_count
-from autocoder.rag.variable_holder import TOKENIZER_MODEL
+from autocoder.rag.variable_holder import VariableHolder
 
 
 class RemoteTokenCounter:
@@ -28,7 +28,7 @@ def initialize_tokenizer(tokenizer_path):
 def count_tokens(text: str) -> int:    
     try:
         # start_time = time.time_ns()
-        encoded = TOKENIZER_MODEL.encode('{"role":"user","content":"' + text + '"}')
+        encoded = VariableHolder.TOKENIZER_MODEL.encode('{"role":"user","content":"' + text + '"}')
         v = len(encoded.ids)
         # elapsed_time = time.time_ns() - start_time
         # logger.info(f"Token counting took {elapsed_time/1000000} ms")

autocoder/rag/token_limiter.py

@@ -17,7 +17,7 @@ class TokenLimiter:
         segment_limit: int,
         buff_limit: int,
         llm,
-        disable_segment_reorder:bool
+        disable_segment_reorder: bool,
     ):
         self.count_tokens = count_tokens
         self.full_text_limit = full_text_limit
@@ -96,8 +96,7 @@ class TokenLimiter:
         doc_num_count = 0
 
         reorder_relevant_docs = []
-        added_docs = set()
-        
+
         ## 文档分段（单个文档过大）和重排序逻辑
         ## 1. 背景：在检索过程中，许多文档被切割成多个段落（segments）
         ## 2. 问题：这些segments在召回时因为是按相关分做了排序可能是乱序的，不符合原文顺序，会强化大模型的幻觉。
@@ -106,20 +105,33 @@ class TokenLimiter:
         ##    a) 方案一（保留位置）：统一文档的不同segments 根据chunk_index 来置换位置
         ##    b) 方案二（当前实现）：遍历文档，发现某文档的segment A，立即查找该文档的所有其他segments，
         ##       对它们进行排序，并将排序后多个segments插入到当前的segment A 位置中。
-        ## TODO: 
-        ##     1. 未来根据参数决定是否开启重排以及重排的策略   
-        if not self.disable_segment_reorder:    
+        ## TODO:
+        ##     1. 未来根据参数决定是否开启重排以及重排的策略
+        if not self.disable_segment_reorder:
+            num_count = 0
             for doc in relevant_docs:
-                if doc.metadata.get('original_doc') and doc.metadata.get('chunk_index'):
-                    if doc.metadata['original_doc'] not in added_docs:
-                        original_doc = doc.metadata['original_doc']
-                        chunks = [d for d in relevant_docs if d.metadata.get('original_doc') == original_doc]
-                        chunks.sort(key=lambda x: x.metadata['chunk_index'])
-                        reorder_relevant_docs.extend(chunks)
-                        added_docs.add(original_doc)
-                elif doc not in added_docs:
-                    reorder_relevant_docs.append(doc)
-                    added_docs.add(doc.module_name)
+                num_count += 1
+                reorder_relevant_docs.append(doc)
+                if "original_doc" in doc.metadata and "chunk_index" in doc.metadata:
+                    original_doc_name = doc.metadata["original_doc"].module_name
+
+                    temp_docs = []
+                    for temp_doc in relevant_docs[num_count:]:
+                        if (
+                            "original_doc" in temp_doc.metadata
+                            and "chunk_index" in temp_doc.metadata
+                        ):
+                            if (
+                                temp_doc.metadata["original_doc"].module_name
+                                == original_doc_name
+                            ):
+                                if temp_doc not in reorder_relevant_docs:
+                                    temp_docs.append(temp_doc)
+
+                    temp_docs.sort(key=lambda x: x.metadata["chunk_index"])
+                    reorder_relevant_docs.extend(temp_docs)
+        else:
+            reorder_relevant_docs = relevant_docs
 
         ## 非窗口分区实现
         for doc in reorder_relevant_docs:
@@ -247,7 +259,5 @@ class TokenLimiter:
                         f"Failed to process doc {doc.module_name} after {max_retries} attempts: {str(e)}"
                     )
                     return SourceCode(
-                        module_name=doc.module_name,
-                        source_code="",   
-                        tokens= 0                     
+                        module_name=doc.module_name, source_code="", tokens=0
                     )

autocoder/rag/variable_holder.py

@@ -1,2 +1,3 @@
-TOKENIZER_PATH = None
-TOKENIZER_MODEL = None
+class VariableHolder:
+    TOKENIZER_PATH = None
+    TOKENIZER_MODEL = None

autocoder/version.py

@@ -1 +1 @@
-__version__ = "0.1.176"
+__version__ = "0.1.178"