auto-coder 0.1.208__py3-none-any.whl → 0.1.211__py3-none-any.whl

autocoder/dispacher/actions/action.py

@@ -5,6 +5,7 @@ from autocoder.common import (
     split_code_into_segments,
     SourceCode,
 )
+from autocoder.common.buildin_tokenizer import BuildinTokenizer
 from autocoder.pyproject import PyProject, Level1PyProject
 from autocoder.tsproject import TSProject
 from autocoder.suffixproject import SuffixProject
@@ -23,9 +24,19 @@ import os
 from autocoder.common.image_to_page import ImageToPage, ImageToPageDirectly
 from autocoder.utils.conversation_store import store_code_model_conversation
 from loguru import logger
+import time
 
 
-class ActionTSProject:
+class BaseAction:
+    def _get_content_length(self, content: str) -> int:
+        try:
+            tokenizer = BuildinTokenizer()
+            return tokenizer.count_tokens(content)
+        except Exception as e:
+            logger.warning(f"Failed to use tokenizer to count tokens, fallback to len(): {e}")
+            return len(content)
+
+class ActionTSProject(BaseAction):
     def __init__(
         self, args: AutoCoderArgs, llm: Optional[byzerllm.ByzerLLM] = None
     ) -> None:
@@ -74,14 +85,15 @@ class ActionTSProject:
         args = self.args
 
         if args.execute and self.llm and not args.human_as_model:
-            if len(content) > self.args.model_max_input_length:
+            content_length = self._get_content_length(content)
+            if content_length > self.args.model_max_input_length:
                 logger.warning(
-                    f"Content length is {len(content)}, which is larger than the maximum input length {self.args.model_max_input_length}. chunk it..."
-                )
-                content = content[: self.args.model_max_input_length]
+                    f"Content(send to model) is {content_length} tokens, which is larger than the maximum input length {self.args.model_max_input_length}"
+                )                
 
-        if args.execute:
+        if args.execute:            
             logger.info("Auto generate the code...")
+            start_time = time.time()
             if args.auto_merge == "diff":
                 generate = CodeAutoGenerateDiff(
                     llm=self.llm, args=self.args, action=self
@@ -104,6 +116,7 @@ class ActionTSProject:
                 generate_result = generate.single_round_run(
                     query=args.query, source_content=content
                 )
+            logger.info(f"Code generation completed in {time.time() - start_time:.2f} seconds")    
             merge_result = None
             if args.execute and args.auto_merge:
                 logger.info("Auto merge the code...")
@@ -141,7 +154,7 @@ class ActionTSProject:
                     file.write(content)
 
 
-class ActionPyScriptProject:
+class ActionPyScriptProject(BaseAction):
     def __init__(
         self, args: AutoCoderArgs, llm: Optional[byzerllm.ByzerLLM] = None
     ) -> None:
@@ -162,6 +175,8 @@ class ActionPyScriptProject:
     def process_content(self, content: str):
         args = self.args
         if args.execute:
+            logger.info("Auto generate the code...")
+            start_time = time.time()
             if args.auto_merge == "diff":
                 generate = CodeAutoGenerateDiff(
                     llm=self.llm, args=self.args, action=self
@@ -184,6 +199,8 @@ class ActionPyScriptProject:
                 generate_result = generate.single_round_run(
                     query=args.query, source_content=content
                 )
+
+            logger.info(f"Code generation completed in {time.time() - start_time:.2f} seconds")    
             merge_result = None
             if args.execute and args.auto_merge:
                 logger.info("Auto merge the code...")
@@ -218,11 +235,13 @@ class ActionPyScriptProject:
                     model=self.llm.default_model_name,
                 )
 
+            end_time = time.time()
+            logger.info(f"Code generation completed in {end_time - start_time:.2f} seconds")
             with open(self.args.target_file, "w") as file:
                 file.write(content)
 
 
-class ActionPyProject:
+class ActionPyProject(BaseAction):
     def __init__(
         self, args: AutoCoderArgs, llm: Optional[byzerllm.ByzerLLM] = None
     ) -> None:
@@ -250,15 +269,15 @@ class ActionPyProject:
         args = self.args
 
         if args.execute and self.llm and not args.human_as_model:
-            if len(content) > self.args.model_max_input_length:
+            content_length = self._get_content_length(content)
+            if content_length > self.args.model_max_input_length:
                 logger.warning(
-                    f'''Content length is {len(content)}(you may collect too much files), which is larger than the maximum input length {self.args.model_max_input_length}. chunk it...'''
+                    f'''Content(send to model) is {content_length} tokens (you may collect too much files), which is larger than the maximum input length {self.args.model_max_input_length}'''
                 )
-                content = content[: self.args.model_max_input_length]
 
         if args.execute:
             logger.info("Auto generate the code...")
-
+            start_time = time.time()
             if args.auto_merge == "diff":
                 generate = CodeAutoGenerateDiff(
                     llm=self.llm, args=self.args, action=self
@@ -283,7 +302,7 @@ class ActionPyProject:
                 generate_result = generate.single_round_run(
                     query=args.query, source_content=content
                 )
-               
+            logger.info(f"Code generation completed in {time.time() - start_time:.2f} seconds")   
             merge_result = None
             if args.execute and args.auto_merge:
                 logger.info("Auto merge the code...")
@@ -316,13 +335,12 @@ class ActionPyProject:
                     instruction=self.args.query,
                     conversations=generate_result.conversations[0],
                     model=self.llm.default_model_name,
-                )
-
+                )            
             with open(args.target_file, "w") as file:
                 file.write(content)
 
 
-class ActionSuffixProject:
+class ActionSuffixProject(BaseAction):
     def __init__(
         self, args: AutoCoderArgs, llm: Optional[byzerllm.ByzerLLM] = None
     ) -> None:
@@ -346,14 +364,15 @@ class ActionSuffixProject:
         args = self.args
 
         if args.execute and self.llm and not args.human_as_model:
-            if len(content) > self.args.model_max_input_length:
+            content_length = self._get_content_length(content)
+            if content_length > self.args.model_max_input_length:
                 logger.warning(
-                    f"Content length is {len(content)}, which is larger than the maximum input length {self.args.model_max_input_length}. chunk it..."
-                )
-                content = content[: self.args.model_max_input_length]
+                    f"Content(send to model) is {content_length} tokens, which is larger than the maximum input length {self.args.model_max_input_length}"
+                )                
 
         if args.execute:
             logger.info("Auto generate the code...")
+            start_time = time.time()
             if args.auto_merge == "diff":
                 generate = CodeAutoGenerateDiff(
                     llm=self.llm, args=self.args, action=self
@@ -377,7 +396,7 @@ class ActionSuffixProject:
                     query=args.query, source_content=content
                 )
               
-
+        logger.info(f"Code generation completed in {time.time() - start_time:.2f} seconds")
         merge_result = None
         if args.execute and args.auto_merge:
             logger.info("Auto merge the code...")

autocoder/rag/cache/simple_cache.py

@@ -98,7 +98,10 @@ class AutoCoderRAGAsyncUpdateQueue(BaseCacheManager):
                     process_file_in_multi_process, files_to_process)
 
             for file_info, result in zip(files_to_process, results):
-                self.update_cache(file_info, result)
+                if result:  # 只有当result不为空时才更新缓存
+                    self.update_cache(file_info, result)
+                else:
+                    logger.warning(f"Empty result for file: {file_info[0]}, skipping cache update")
 
             self.write_cache()
 
@@ -138,7 +141,10 @@ class AutoCoderRAGAsyncUpdateQueue(BaseCacheManager):
                     logger.info(f"{file_info[0]} is detected to be updated")
                     try:
                         result = process_file_local(file_info[0])
-                        self.update_cache(file_info, result)
+                        if result:  # 只有当result不为空时才更新缓存
+                            self.update_cache(file_info, result)
+                        else:
+                            logger.warning(f"Empty result for file: {file_info[0]}, skipping cache update")
                     except Exception as e:
                         logger.error(
                             f"SimpleCache Error in process_queue: {e}")

autocoder/rag/loaders/docx_loader.py

@@ -1,7 +1,7 @@
 from io import BytesIO
 import docx2txt
 from autocoder.utils._markitdown import MarkItDown
-
+import traceback
 
 def extract_text_from_docx_old(docx_path):
     with open(docx_path, "rb") as f:
@@ -16,5 +16,6 @@ def extract_text_from_docx(docx_path):
         md_converter = MarkItDown()
         result = md_converter.convert(docx_path)
         return result.text_content
-    except Exception as e:
+    except (BaseException, Exception) as e:
+        traceback.print_exc()
         return extract_text_from_docx_old(docx_path)

autocoder/rag/loaders/pdf_loader.py

@@ -1,6 +1,7 @@
 from io import BytesIO
 from pypdf import PdfReader
 from autocoder.utils._markitdown import MarkItDown
+import traceback
 
 
 def extract_text_from_pdf_old(file_path):
@@ -18,5 +19,6 @@ def extract_text_from_pdf(file_path):
         md_converter = MarkItDown()
         result = md_converter.convert(file_path)
         return result.text_content
-    except Exception as e:        
+    except (BaseException, Exception) as e:
+        traceback.print_exc()
         return extract_text_from_pdf_old(file_path)

autocoder/rag/long_context_rag.py

@@ -182,19 +182,25 @@ class LongContextRAG:
         使用以下文档和对话历史来提取相关信息。
 
         文档：
+        <documents>
         {% for doc in documents %}
         {{ doc }}
         {% endfor %}
+        </documents>
 
         对话历史：
+        <conversations>
         {% for msg in conversations %}
-        <{{ msg.role }}>: {{ msg.content }}
+        [{{ msg.role }}]: 
+        {{ msg.content }}
+        
         {% endfor %}
+        </conversations>
 
         请根据提供的文档内容、用户对话历史以及最后一个问题，提取并总结文档中与问题相关的重要信息。
         如果文档中没有相关信息，请回复"该文档中没有与问题相关的信息"。
         提取的信息尽量保持和原文中的一样，并且只输出这些信息。
-        """
+        """        
 
     @byzerllm.prompt()
     def _answer_question(
@@ -202,9 +208,11 @@ class LongContextRAG:
     ) -> Generator[str, None, None]:
         """        
         文档：
+        <documents>
         {% for doc in relevant_docs %}
         {{ doc }}
         {% endfor %}
+        </documents>
 
         使用以上文档来回答用户的问题。回答要求：
 

autocoder/rag/utils.py

@@ -9,13 +9,14 @@ import time
 from loguru import logger
 import traceback
 
+
 def process_file_in_multi_process(
     file_info: Tuple[str, str, float]
 ) -> List[SourceCode]:
     start_time = time.time()
     file_path, relative_path, _, _ = file_info
     try:
-        if file_path.endswith(".pdf"):            
+        if file_path.endswith(".pdf"):
             content = extract_text_from_pdf(file_path)
             v = [
                 SourceCode(
@@ -24,7 +25,7 @@ def process_file_in_multi_process(
                     tokens=count_tokens_worker(content),
                 )
             ]
-        elif file_path.endswith(".docx"):            
+        elif file_path.endswith(".docx"):
             content = extract_text_from_docx(file_path)
             v = [
                 SourceCode(
@@ -45,7 +46,8 @@ def process_file_in_multi_process(
             ]
         elif file_path.endswith(".pptx"):
             slides = extract_text_from_ppt(file_path)
-            content = "".join(f"#{slide[0]}\n{slide[1]}\n\n" for slide in slides)
+            content = "".join(
+                f"#{slide[0]}\n{slide[1]}\n\n" for slide in slides)
             v = [
                 SourceCode(
                     module_name=f"##File: {file_path}",
@@ -65,15 +67,16 @@ def process_file_in_multi_process(
             ]
         logger.info(f"Load file {file_path} in {time.time() - start_time}")
         return v
-    except Exception as e:
+    except (BaseException, Exception) as e:
         logger.error(f"Error processing file {file_path}: {str(e)}")
+        logger.error(f"Error type: {type(e).__name__}")
         return []
 
 
 def process_file_local(file_path: str) -> List[SourceCode]:
     start_time = time.time()
     try:
-        if file_path.endswith(".pdf"):            
+        if file_path.endswith(".pdf"):
             content = extract_text_from_pdf(file_path)
             v = [
                 SourceCode(
@@ -82,7 +85,7 @@ def process_file_local(file_path: str) -> List[SourceCode]:
                     tokens=count_tokens(content),
                 )
             ]
-        elif file_path.endswith(".docx"):            
+        elif file_path.endswith(".docx"):
             content = extract_text_from_docx(file_path)
             v = [
                 SourceCode(
@@ -103,7 +106,8 @@ def process_file_local(file_path: str) -> List[SourceCode]:
             ]
         elif file_path.endswith(".pptx"):
             slides = extract_text_from_ppt(file_path)
-            content = "".join(f"#{slide[0]}\n{slide[1]}\n\n" for slide in slides)
+            content = "".join(
+                f"#{slide[0]}\n{slide[1]}\n\n" for slide in slides)
             v = [
                 SourceCode(
                     module_name=f"##File: {file_path}",
@@ -123,7 +127,8 @@ def process_file_local(file_path: str) -> List[SourceCode]:
             ]
         logger.info(f"Load file {file_path} in {time.time() - start_time}")
         return v
-    except Exception as e:
+    except (BaseException, Exception) as e:
         logger.error(f"Error processing file {file_path}: {str(e)}")
+        logger.error(f"Error type: {type(e).__name__}")
         traceback.print_exc()
-        return []
+        return []

autocoder/version.py

@@ -1 +1 @@
-__version__ = "0.1.208"
+__version__ = "0.1.211"