auto-coder 0.1.400__py3-none-any.whl → 2.0.0__py3-none-any.whl

autocoder/common/autocoderargs_parser/token_parser.py

@@ -0,0 +1,290 @@
+import re
+import ast
+import operator
+from typing import Union, Optional, Dict, Any
+from ..llms import LLMManager, LLMModel
+
+
+class TokenParser:
+    """Token 数量解析器，支持多种格式的 token 数量表示"""
+    
+    # 单位转换映射
+    UNIT_MULTIPLIERS = {
+        'k': 1024,
+        'm': 1024 * 1024,
+        'g': 1024 * 1024 * 1024,
+        'kb': 1024,
+        'mb': 1024 * 1024,
+        'gb': 1024 * 1024 * 1024,
+    }
+    
+    # 支持的数学运算符
+    OPERATORS = {
+        ast.Add: operator.add,
+        ast.Sub: operator.sub,
+        ast.Mult: operator.mul,
+        ast.Div: operator.truediv,
+        ast.FloorDiv: operator.floordiv,
+        ast.Mod: operator.mod,
+        ast.Pow: operator.pow,
+    }
+    
+    def __init__(self, llm_manager: Optional[LLMManager] = None):
+        """
+        初始化 TokenParser
+        
+        Args:
+            llm_manager: LLM 管理器，用于获取模型信息
+        """
+        self.llm_manager = llm_manager or LLMManager()
+    
+    def parse_token_value(self, value: Union[str, int, float], 
+                         code_model: Optional[str] = None) -> int:
+        """
+        解析 token 数量值
+        
+        Args:
+            value: 要解析的值，可以是字符串、整数或浮点数
+            code_model: 代码模型名称，用于 float 类型的解析
+            
+        Returns:
+            解析后的 token 数量（整数）
+            
+        Raises:
+            ValueError: 当解析失败时
+        """
+        if isinstance(value, int):
+            return max(0, value)
+        
+        if isinstance(value, float):
+            return self._parse_float_value(value, code_model)
+        
+        if isinstance(value, str):
+            return self._parse_string_value(value)
+        
+        raise ValueError(f"Unsupported value type: {type(value)}")
+    
+    def _parse_float_value(self, value: float, code_model: Optional[str] = None) -> int:
+        """
+        解析浮点数值，基于模型的 context_window
+        
+        Args:
+            value: 浮点数值（0-1 之间表示比例）
+            code_model: 模型名称
+            
+        Returns:
+            计算后的 token 数量
+        """
+        if not (0 <= value <= 1):
+            raise ValueError(f"Float value must be between 0 and 1, got: {value}")
+        
+        context_window = self._get_context_window(code_model)
+        return int(context_window * value)
+    
+    def _parse_string_value(self, value: str) -> int:
+        """
+        解析字符串值，支持纯数字、单位和数学表达式
+        
+        Args:
+            value: 字符串值
+            
+        Returns:
+            解析后的 token 数量
+        """
+        # 移除空格
+        value = value.strip()
+        
+        if not value:
+            raise ValueError("Empty string value")
+        
+        # 先尝试解析纯数字字符串（整数或浮点数）
+        numeric_result = self._try_parse_numeric_string(value)
+        if numeric_result is not None:
+            return numeric_result
+        
+        # 再尝试解析带单位的简单数字
+        unit_result = self._try_parse_with_unit(value)
+        if unit_result is not None:
+            return unit_result
+        
+        # 最后尝试解析数学表达式
+        return self._parse_math_expression(value)
+    
+    def _try_parse_numeric_string(self, value: str) -> Optional[int]:
+        """
+        尝试解析纯数字字符串（整数或浮点数）
+        
+        Args:
+            value: 字符串值
+            
+        Returns:
+            解析结果，如果不是纯数字则返回 None
+        """
+        try:
+            # 使用正则表达式检查是否为纯数字字符串
+            
+            # 匹配整数（包括负数）
+            if re.match(r'^-?\d+$', value):
+                return max(0, int(value))
+            
+            # 匹配浮点数（包括负数）
+            if re.match(r'^-?\d+\.\d+$', value) or re.match(r'^-?\d+\.$', value) or re.match(r'^-?\.\d+$', value):
+                float_value = float(value)
+                return max(0, int(float_value))
+                
+            return None
+                
+        except (ValueError, TypeError):
+            return None
+    
+    def _try_parse_with_unit(self, value: str) -> Optional[int]:
+        """
+        尝试解析带单位的数字（如 100k, 1.5m）
+        
+        Args:
+            value: 字符串值
+            
+        Returns:
+            解析结果，如果不匹配则返回 None
+        """
+        # 匹配数字+单位的模式
+        pattern = r'^(\d+(?:\.\d+)?)\s*([kmgKMG](?:[bB]?)?)$'
+        match = re.match(pattern, value)
+        
+        if not match:
+            return None
+        
+        number_str, unit = match.groups()
+        number = float(number_str)
+        unit_lower = unit.lower()
+        
+        if unit_lower in self.UNIT_MULTIPLIERS:
+            return int(number * self.UNIT_MULTIPLIERS[unit_lower])
+        
+        return None
+    
+    def _parse_math_expression(self, expression: str) -> int:
+        """
+        安全地解析数学表达式
+        
+        Args:
+            expression: 数学表达式字符串
+            
+        Returns:
+            计算结果
+            
+        Raises:
+            ValueError: 当表达式无效时
+        """
+        try:
+            # 解析 AST
+            tree = ast.parse(expression, mode='eval')
+            result = self._evaluate_ast_node(tree.body)
+            
+            # 确保结果是正整数
+            if isinstance(result, (int, float)):
+                return max(0, int(result))
+            else:
+                raise ValueError(f"Expression result is not a number: {result}")
+                
+        except (SyntaxError, ValueError, TypeError) as e:
+            raise ValueError(f"Invalid expression '{expression}': {e}")
+    
+    def _evaluate_ast_node(self, node: ast.AST) -> Union[int, float]:
+        """
+        递归计算 AST 节点
+        
+        Args:
+            node: AST 节点
+            
+        Returns:
+            计算结果
+        """
+        if isinstance(node, ast.Constant):  # Python 3.8+
+            return node.value
+        elif isinstance(node, ast.Num):  # Python < 3.8
+            return node.n
+        elif isinstance(node, ast.BinOp):
+            left = self._evaluate_ast_node(node.left)
+            right = self._evaluate_ast_node(node.right)
+            op = self.OPERATORS.get(type(node.op))
+            if op is None:
+                raise ValueError(f"Unsupported operator: {type(node.op)}")
+            
+            # 检查除零情况
+            if isinstance(node.op, (ast.Div, ast.FloorDiv, ast.Mod)) and right == 0:
+                raise ValueError("Division by zero")
+                
+            return op(left, right)
+        elif isinstance(node, ast.UnaryOp):
+            operand = self._evaluate_ast_node(node.operand)
+            if isinstance(node.op, ast.UAdd):
+                return +operand
+            elif isinstance(node.op, ast.USub):
+                return -operand
+            else:
+                raise ValueError(f"Unsupported unary operator: {type(node.op)}")
+        else:
+            raise ValueError(f"Unsupported AST node type: {type(node)}")
+    
+    def _get_context_window(self, model_name: Optional[str] = None) -> int:
+        """
+        获取模型的 context_window
+        
+        Args:
+            model_name: 模型名称
+            
+        Returns:
+            context_window 大小，如果未找到或为 0 则返回默认值 120*1000
+        """
+        if not model_name:
+            return 120 * 1000
+        
+        try:
+            model = self.llm_manager.get_model(model_name)
+            if model and model.context_window > 0:
+                return model.context_window
+        except Exception:
+            # 忽略获取模型信息时的错误
+            pass
+        
+        return 120 * 1000
+    
+    def validate_token_value(self, value: Union[str, int, float], 
+                           min_tokens: int = 0, 
+                           max_tokens: Optional[int] = None) -> bool:
+        """
+        验证 token 值是否在有效范围内
+        
+        Args:
+            value: 要验证的值
+            min_tokens: 最小 token 数
+            max_tokens: 最大 token 数，None 表示无限制
+            
+        Returns:
+            是否有效
+        """
+        try:
+            parsed_value = self.parse_token_value(value)
+            if parsed_value < min_tokens:
+                return False
+            if max_tokens is not None and parsed_value > max_tokens:
+                return False
+            return True
+        except (ValueError, TypeError):
+            return False
+    
+    def get_supported_units(self) -> Dict[str, int]:
+        """获取支持的单位及其倍数"""
+        return self.UNIT_MULTIPLIERS.copy()
+    
+    def get_examples(self) -> Dict[str, str]:
+        """获取使用示例"""
+        return {
+            "整数": "51200",
+            "纯数字字符串": "1024, 8, 1024.5",
+            "带单位": "50k, 1m, 2.5k",
+            "数学表达式": "50*1024, 100*1000, 2**16",
+            "浮点数（比例）": "0.1 (表示 context_window 的 10%)",
+            "复合表达式": "50*1024 + 10k"
+        } 

autocoder/common/buildin_tokenizer.py

@@ -1,4 +1,4 @@
-import pkg_resources
+import importlib.resources as resources
 from tokenizers import Tokenizer
 from typing import Optional
 
@@ -14,9 +14,7 @@ class BuildinTokenizer:
 
     def _initialize(self) -> None:
         try:
-            tokenizer_path = pkg_resources.resource_filename(
-                "autocoder", "data/tokenizer.json"
-            )
+            tokenizer_path = str(resources.files("autocoder") / "data" / "tokenizer.json")
         except FileNotFoundError:
             tokenizer_path = None
             

autocoder/common/code_auto_generate.py

@@ -2,22 +2,31 @@ from typing import List, Dict, Tuple
 from autocoder.common.types import Mode
 from autocoder.common import AutoCoderArgs
 import byzerllm
-from autocoder.utils.queue_communicate import queue_communicate, CommunicateEvent, CommunicateEventType
 from autocoder.common import sys_prompt
 from concurrent.futures import ThreadPoolExecutor
 from autocoder.common.types import CodeGenerateResult
-from autocoder.common.utils_code_auto_generate import chat_with_continue,stream_chat_with_continue,ChatWithContinueResult
+from autocoder.common.utils_code_auto_generate import (
+    chat_with_continue,
+    stream_chat_with_continue,
+    ChatWithContinueResult,
+)
 from autocoder.utils.auto_coder_utils.chat_stream_out import stream_out
 from autocoder.common.stream_out_type import CodeGenerateStreamOutType
 from autocoder.common.auto_coder_lang import get_message_with_format
 import json
 from autocoder.common.printer import Printer
-from autocoder.rag.token_counter import count_tokens
+from autocoder.common.tokens import count_string_tokens as count_tokens
 from autocoder.utils import llms as llm_utils
 from autocoder.common import SourceCodeList
 from autocoder.privacy.model_filter import ModelPathFilter
 from autocoder.memory.active_context_manager import ActiveContextManager
-from autocoder.run_context import get_run_context,RunMode
+from autocoder.run_context import get_run_context, RunMode
+import os
+import uuid
+from datetime import datetime
+from autocoder.common.save_formatted_log import save_formatted_log
+import platform
+
 
 class CodeAutoGenerate:
     def __init__(
@@ -25,7 +34,7 @@ class CodeAutoGenerate:
     ) -> None:
         self.llm = llm
         self.args = args
-        self.action = action        
+        self.action = action
         self.generate_times_same_model = args.generate_times_same_model
         if not self.llm:
             raise ValueError(
@@ -51,13 +60,22 @@ class CodeAutoGenerate:
         {{ instruction }}
 
         """
-    
 
     @byzerllm.prompt(llm=lambda self: self.llm)
     def single_round_instruction(
-        self, instruction: str, content: str, context: str = "", package_context: str = ""
+        self,
+        instruction: str,
+        content: str,
+        context: str = "",
+        package_context: str = "",
     ) -> str:
         """
+        <env>
+        当前工作目录: {{ current_dir }}
+        操作系统: {{ os_info }}
+        当前时间: {{ current_time }}
+        </env>
+
         {%- if structure %}
         ====
         {{ structure }}
@@ -95,7 +113,7 @@ class CodeAutoGenerate:
         ##File: {{ key }}
         {{ value }}
         </user_rule>
-        {% endfor %}        
+        {% endfor %}
         {% endif %}
 
         ====
@@ -120,18 +138,28 @@ class CodeAutoGenerate:
 
         请确保每份代码的完整性，而不要只生成修改部分。
         """
-        
+        # 收集环境信息
+        current_dir = os.getcwd()
+        os_info = f"{platform.system()} {platform.release()}"
+        current_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+
         if not self.args.include_project_structure:
             return {
-                "structure": "",                
+                "structure": "",
+                "current_dir": current_dir,
+                "os_info": os_info,
+                "current_time": current_time,
             }
-        
+
         return {
             "structure": (
                 self.action.pp.get_tree_like_directory_structure()
                 if self.action
                 else ""
-            )
+            ),
+            "current_dir": current_dir,
+            "os_info": os_info,
+            "current_time": current_time,
         }
 
     def single_round_run(
@@ -139,8 +167,8 @@ class CodeAutoGenerate:
     ) -> Tuple[List[str], Dict[str, str]]:
         llm_config = {"human_as_model": self.args.human_as_model}
 
-        # Apply model filter for code_llm  
-        printer = Printer()      
+        # Apply model filter for code_llm
+        printer = Printer()
         for llm in self.llms:
             model_filter = ModelPathFilter.from_model_object(llm, self.args)
             filtered_sources = []
@@ -148,20 +176,25 @@ class CodeAutoGenerate:
                 if model_filter.is_accessible(source.module_name):
                     filtered_sources.append(source)
                 else:
-                    printer.print_in_terminal("index_file_filtered", 
-                                               style="yellow",
-                                               file_path=source.module_name, 
-                                               model_name=",".join(llm_utils.get_llm_names(llm)))
-            
-        source_code_list = SourceCodeList(filtered_sources)
+                    printer.print_in_terminal(
+                        "index_file_filtered",
+                        style="yellow",
+                        file_path=source.module_name,
+                        model_name=",".join(llm_utils.get_llm_names(llm)),
+                    )
+
+            source_code_list = SourceCodeList(filtered_sources)
+
         source_content = source_code_list.to_str()
 
         # 获取包上下文信息
         package_context = ""
-        
+
         if self.args.enable_active_context:
             # 初始化活动上下文管理器
-            active_context_manager = ActiveContextManager(self.llm, self.args.source_dir)
+            active_context_manager = ActiveContextManager(
+                self.llm, self.args.source_dir
+            )
             # 获取活动上下文信息
             result = active_context_manager.load_active_contexts_for_files(
                 [source.module_name for source in source_code_list.sources]
@@ -170,32 +203,52 @@ class CodeAutoGenerate:
             if result.contexts:
                 package_context_parts = []
                 for dir_path, context in result.contexts.items():
-                    package_context_parts.append(f"<package_info>{context.content}</package_info>")
-                
+                    package_context_parts.append(
+                        f"<package_info>{context.content}</package_info>"
+                    )
+
                 package_context = "\n".join(package_context_parts)
 
         if self.args.template == "common":
             init_prompt = self.single_round_instruction.prompt(
-                instruction=query, content=source_content, context=self.args.context,
-                package_context=package_context
+                instruction=query,
+                content=source_content,
+                context=self.args.context,
+                package_context=package_context,
             )
         elif self.args.template == "auto_implement":
             init_prompt = self.auto_implement_function.prompt(
                 instruction=query, content=source_content
             )
 
-        with open(self.args.target_file, "w",encoding="utf-8") as file:
+        with open(self.args.target_file, "w", encoding="utf-8") as file:
             file.write(init_prompt)
 
         conversations = []
 
         if self.args.system_prompt and self.args.system_prompt.strip() == "claude":
-            conversations.append({"role": "system", "content": sys_prompt.claude_sys_prompt.prompt()})
+            conversations.append(
+                {"role": "system", "content": sys_prompt.claude_sys_prompt.prompt()}
+            )
         elif self.args.system_prompt:
             conversations.append({"role": "system", "content": self.args.system_prompt})
-        
+
         conversations.append({"role": "user", "content": init_prompt})
 
+        # 保存 conversations 到 manuals 目录
+        try:
+            conversations_json = json.dumps(conversations, ensure_ascii=False, indent=2)
+            save_formatted_log(
+                project_root=self.args.source_dir,
+                json_text=conversations_json,
+                suffix="conversations",
+                log_subdir="manuals",
+            )
+        except Exception as e:
+            printer.print_in_terminal(
+                "conversations_save_failed", style="red", error=str(e)
+            )
+
         conversations_list = []
         results = []
         input_tokens_count = 0
@@ -205,90 +258,112 @@ class CodeAutoGenerate:
         model_names = []
 
         printer = Printer()
-        estimated_input_tokens = count_tokens(json.dumps(conversations, ensure_ascii=False))
-        printer.print_in_terminal("estimated_input_tokens_in_generate", style="yellow",
-                                  estimated_input_tokens_in_generate=estimated_input_tokens,
-                                  generate_mode="wholefile"
-                                  )
+        estimated_input_tokens = count_tokens(
+            json.dumps(conversations, ensure_ascii=False)
+        )
+        printer.print_in_terminal(
+            "estimated_input_tokens_in_generate",
+            style="yellow",
+            estimated_input_tokens_in_generate=estimated_input_tokens,
+            generate_mode="wholefile",
+        )
 
         if not self.args.human_as_model or get_run_context().mode == RunMode.WEB:
-            with ThreadPoolExecutor(max_workers=len(self.llms) * self.generate_times_same_model) as executor:
+            with ThreadPoolExecutor(
+                max_workers=len(self.llms) * self.generate_times_same_model
+            ) as executor:
                 futures = []
                 count = 0
                 for llm in self.llms:
-                    model_names_list = llm_utils.get_llm_names(llm) 
+                    model_names_list = llm_utils.get_llm_names(llm)
                     model_name = None
                     if model_names_list:
-                        model_name = model_names_list[0]  
+                        model_name = model_names_list[0]
 
                     for _ in range(self.generate_times_same_model):
                         model_names.append(model_name)
                         if count == 0:
+
                             def job():
                                 stream_generator = stream_chat_with_continue(
-                                    llm=llm, 
-                                    conversations=conversations, 
+                                    llm=llm,
+                                    conversations=conversations,
                                     llm_config=llm_config,
-                                    args=self.args
+                                    args=self.args,
                                 )
                                 full_response, last_meta = stream_out(
-                                stream_generator,
-                                model_name=model_name,
-                                title=get_message_with_format(
-                                    "code_generate_title", model_name=model_name),
-                                args=self.args,
-                                extra_meta={
-                                    "stream_out_type": CodeGenerateStreamOutType.CODE_GENERATE.value
-                                })
+                                    stream_generator,
+                                    model_name=model_name,
+                                    title=get_message_with_format(
+                                        "code_generate_title", model_name=model_name
+                                    ),
+                                    args=self.args,
+                                    extra_meta={
+                                        "stream_out_type": CodeGenerateStreamOutType.CODE_GENERATE.value
+                                    },
+                                )
                                 return ChatWithContinueResult(
                                     content=full_response,
                                     input_tokens_count=last_meta.input_tokens_count,
-                                    generated_tokens_count=last_meta.generated_tokens_count
+                                    generated_tokens_count=last_meta.generated_tokens_count,
                                 )
+
                             futures.append(executor.submit(job))
-                        else:                                
-                            futures.append(executor.submit(
-                                chat_with_continue, 
-                                llm=llm, 
-                                conversations=conversations, 
-                                llm_config=llm_config,
-                                args=self.args
-                            ))
+                        else:
+                            futures.append(
+                                executor.submit(
+                                    chat_with_continue,
+                                    llm=llm,
+                                    conversations=conversations,
+                                    llm_config=llm_config,
+                                    args=self.args,
+                                )
+                            )
                         count += 1
                 temp_results = [future.result() for future in futures]
                 for result in temp_results:
                     results.append(result.content)
                     input_tokens_count += result.input_tokens_count
                     generated_tokens_count += result.generated_tokens_count
-                    model_info = llm_utils.get_model_info(model_name, self.args.product_mode)
+                    model_info = llm_utils.get_model_info(
+                        model_name, self.args.product_mode
+                    )
                     input_cost = model_info.get("input_price", 0) if model_info else 0
                     output_cost = model_info.get("output_price", 0) if model_info else 0
-                    input_tokens_cost += input_cost * result.input_tokens_count / 1000000
-                    generated_tokens_cost += output_cost * result.generated_tokens_count / 1000000
-            
+                    input_tokens_cost += (
+                        input_cost * result.input_tokens_count / 1000000
+                    )
+                    generated_tokens_cost += (
+                        output_cost * result.generated_tokens_count / 1000000
+                    )
+
             for result in results:
                 conversations_list.append(
-                    conversations + [{"role": "assistant", "content": result}])
-        else:            
+                    conversations + [{"role": "assistant", "content": result}]
+                )
+        else:
             for _ in range(self.args.human_model_num):
                 single_result = chat_with_continue(
-                    llm=self.llms[0], 
-                    conversations=conversations, 
+                    llm=self.llms[0],
+                    conversations=conversations,
                     llm_config=llm_config,
-                    args=self.args
-                )                
+                    args=self.args,
+                )
                 results.append(single_result.content)
                 input_tokens_count += single_result.input_tokens_count
                 generated_tokens_count += single_result.generated_tokens_count
-                conversations_list.append(conversations + [{"role": "assistant", "content": single_result.content}])
-        
+                conversations_list.append(
+                    conversations
+                    + [{"role": "assistant", "content": single_result.content}]
+                )
+
         statistics = {
             "input_tokens_count": input_tokens_count,
             "generated_tokens_count": generated_tokens_count,
             "input_tokens_cost": input_tokens_cost,
-            "generated_tokens_cost": generated_tokens_cost
-        }        
-    
-        return CodeGenerateResult(contents=results, conversations=conversations_list, metadata=statistics)
+            "generated_tokens_cost": generated_tokens_cost,
+        }
 
-    
+        return CodeGenerateResult(
+            contents=results, conversations=conversations_list, metadata=statistics
+        )