auto-coder 0.1.209__py3-none-any.whl → 0.1.212__py3-none-any.whl

autocoder/dispacher/actions/action.py

@@ -5,6 +5,7 @@ from autocoder.common import (
     split_code_into_segments,
     SourceCode,
 )
+from autocoder.common.buildin_tokenizer import BuildinTokenizer
 from autocoder.pyproject import PyProject, Level1PyProject
 from autocoder.tsproject import TSProject
 from autocoder.suffixproject import SuffixProject
@@ -23,9 +24,19 @@ import os
 from autocoder.common.image_to_page import ImageToPage, ImageToPageDirectly
 from autocoder.utils.conversation_store import store_code_model_conversation
 from loguru import logger
+import time
 
 
-class ActionTSProject:
+class BaseAction:
+    def _get_content_length(self, content: str) -> int:
+        try:
+            tokenizer = BuildinTokenizer()
+            return tokenizer.count_tokens(content)
+        except Exception as e:
+            logger.warning(f"Failed to use tokenizer to count tokens, fallback to len(): {e}")
+            return len(content)
+
+class ActionTSProject(BaseAction):
     def __init__(
         self, args: AutoCoderArgs, llm: Optional[byzerllm.ByzerLLM] = None
     ) -> None:
@@ -74,14 +85,15 @@ class ActionTSProject:
         args = self.args
 
         if args.execute and self.llm and not args.human_as_model:
-            if len(content) > self.args.model_max_input_length:
+            content_length = self._get_content_length(content)
+            if content_length > self.args.model_max_input_length:
                 logger.warning(
-                    f"Content length is {len(content)}, which is larger than the maximum input length {self.args.model_max_input_length}. chunk it..."
-                )
-                content = content[: self.args.model_max_input_length]
+                    f"Content(send to model) is {content_length} tokens, which is larger than the maximum input length {self.args.model_max_input_length}"
+                )                
 
-        if args.execute:
+        if args.execute:            
             logger.info("Auto generate the code...")
+            start_time = time.time()
             if args.auto_merge == "diff":
                 generate = CodeAutoGenerateDiff(
                     llm=self.llm, args=self.args, action=self
@@ -104,6 +116,7 @@ class ActionTSProject:
                 generate_result = generate.single_round_run(
                     query=args.query, source_content=content
                 )
+            logger.info(f"Code generation completed in {time.time() - start_time:.2f} seconds")    
             merge_result = None
             if args.execute and args.auto_merge:
                 logger.info("Auto merge the code...")
@@ -141,7 +154,7 @@ class ActionTSProject:
                     file.write(content)
 
 
-class ActionPyScriptProject:
+class ActionPyScriptProject(BaseAction):
     def __init__(
         self, args: AutoCoderArgs, llm: Optional[byzerllm.ByzerLLM] = None
     ) -> None:
@@ -162,6 +175,8 @@ class ActionPyScriptProject:
     def process_content(self, content: str):
         args = self.args
         if args.execute:
+            logger.info("Auto generate the code...")
+            start_time = time.time()
             if args.auto_merge == "diff":
                 generate = CodeAutoGenerateDiff(
                     llm=self.llm, args=self.args, action=self
@@ -184,6 +199,8 @@ class ActionPyScriptProject:
                 generate_result = generate.single_round_run(
                     query=args.query, source_content=content
                 )
+
+            logger.info(f"Code generation completed in {time.time() - start_time:.2f} seconds")    
             merge_result = None
             if args.execute and args.auto_merge:
                 logger.info("Auto merge the code...")
@@ -218,11 +235,13 @@ class ActionPyScriptProject:
                     model=self.llm.default_model_name,
                 )
 
+            end_time = time.time()
+            logger.info(f"Code generation completed in {end_time - start_time:.2f} seconds")
             with open(self.args.target_file, "w") as file:
                 file.write(content)
 
 
-class ActionPyProject:
+class ActionPyProject(BaseAction):
     def __init__(
         self, args: AutoCoderArgs, llm: Optional[byzerllm.ByzerLLM] = None
     ) -> None:
@@ -250,15 +269,15 @@ class ActionPyProject:
         args = self.args
 
         if args.execute and self.llm and not args.human_as_model:
-            if len(content) > self.args.model_max_input_length:
+            content_length = self._get_content_length(content)
+            if content_length > self.args.model_max_input_length:
                 logger.warning(
-                    f'''Content length is {len(content)}(you may collect too much files), which is larger than the maximum input length {self.args.model_max_input_length}. chunk it...'''
+                    f'''Content(send to model) is {content_length} tokens (you may collect too much files), which is larger than the maximum input length {self.args.model_max_input_length}'''
                 )
-                content = content[: self.args.model_max_input_length]
 
         if args.execute:
             logger.info("Auto generate the code...")
-
+            start_time = time.time()
             if args.auto_merge == "diff":
                 generate = CodeAutoGenerateDiff(
                     llm=self.llm, args=self.args, action=self
@@ -283,7 +302,7 @@ class ActionPyProject:
                 generate_result = generate.single_round_run(
                     query=args.query, source_content=content
                 )
-               
+            logger.info(f"Code generation completed in {time.time() - start_time:.2f} seconds")   
             merge_result = None
             if args.execute and args.auto_merge:
                 logger.info("Auto merge the code...")
@@ -316,13 +335,12 @@ class ActionPyProject:
                     instruction=self.args.query,
                     conversations=generate_result.conversations[0],
                     model=self.llm.default_model_name,
-                )
-
+                )            
             with open(args.target_file, "w") as file:
                 file.write(content)
 
 
-class ActionSuffixProject:
+class ActionSuffixProject(BaseAction):
     def __init__(
         self, args: AutoCoderArgs, llm: Optional[byzerllm.ByzerLLM] = None
     ) -> None:
@@ -346,14 +364,15 @@ class ActionSuffixProject:
         args = self.args
 
         if args.execute and self.llm and not args.human_as_model:
-            if len(content) > self.args.model_max_input_length:
+            content_length = self._get_content_length(content)
+            if content_length > self.args.model_max_input_length:
                 logger.warning(
-                    f"Content length is {len(content)}, which is larger than the maximum input length {self.args.model_max_input_length}. chunk it..."
-                )
-                content = content[: self.args.model_max_input_length]
+                    f"Content(send to model) is {content_length} tokens, which is larger than the maximum input length {self.args.model_max_input_length}"
+                )                
 
         if args.execute:
             logger.info("Auto generate the code...")
+            start_time = time.time()
             if args.auto_merge == "diff":
                 generate = CodeAutoGenerateDiff(
                     llm=self.llm, args=self.args, action=self
@@ -377,7 +396,7 @@ class ActionSuffixProject:
                     query=args.query, source_content=content
                 )
               
-
+        logger.info(f"Code generation completed in {time.time() - start_time:.2f} seconds")
         merge_result = None
         if args.execute and args.auto_merge:
             logger.info("Auto merge the code...")

autocoder/version.py

@@ -1 +1 @@
-__version__ = "0.1.209"
+__version__ = "0.1.212"