auto-coder 1.0.0__py3-none-any.whl → 2.0.1__py3-none-any.whl

autocoder/common/autocoderargs_parser/__init__.py

@@ -0,0 +1,14 @@
+"""
+Auto-Coder AutoCoderArgs 参数解析模块
+
+提供 AutoCoderArgs 参数的高级解析功能，包括：
+- 支持多种数据类型的统一解析
+- Token 数量单位解析（k, m, g 等）
+- 数学表达式计算
+- 基于模型配置的动态解析
+"""
+
+from .parser import AutoCoderArgsParser
+from .token_parser import TokenParser
+
+__all__ = ["AutoCoderArgsParser", "TokenParser"] 

autocoder/common/autocoderargs_parser/parser.py

@@ -0,0 +1,184 @@
+from typing import Union, Optional, Dict, Any, Type
+from pydantic import Field
+from ..llms import LLMManager
+from .token_parser import TokenParser
+
+
+class AutoCoderArgsParser:
+    """AutoCoderArgs 参数解析器，提供高级参数解析功能"""
+    
+    def __init__(self, llm_manager: Optional[LLMManager] = None):
+        """
+        初始化参数解析器
+        
+        Args:
+            llm_manager: LLM 管理器实例
+        """
+        self.llm_manager = llm_manager or LLMManager()
+        self.token_parser = TokenParser(self.llm_manager)
+    
+    def parse_conversation_prune_safe_zone_tokens(self, 
+                                                 value: Union[str, int, float],
+                                                 code_model: Optional[str] = None) -> int:
+        """
+        解析 conversation_prune_safe_zone_tokens 参数
+        
+        Args:
+            value: 参数值，支持字符串、整数、浮点数
+            code_model: 代码模型名称，用于浮点数解析
+            
+        Returns:
+            解析后的 token 数量
+            
+        Examples:
+            >>> parser = AutoCoderArgsParser()
+            >>> parser.parse_conversation_prune_safe_zone_tokens(51200)
+            51200
+            >>> parser.parse_conversation_prune_safe_zone_tokens("50k")
+            51200
+            >>> parser.parse_conversation_prune_safe_zone_tokens("50*1024")
+            51200
+            >>> parser.parse_conversation_prune_safe_zone_tokens(0.1, "deepseek/v3")
+            # 返回 deepseek/v3 context_window 的 10%
+        """
+        try:
+            return self.token_parser.parse_token_value(value, code_model)
+        except ValueError as e:
+            # 如果解析失败，返回默认值并记录警告
+            default_value = 50 * 1024
+            print(f"Warning: Failed to parse conversation_prune_safe_zone_tokens '{value}': {e}. Using default value: {default_value}")
+            return default_value
+    
+    def parse_context_prune_safe_zone_tokens(self,
+                                           value: Union[str, int, float],
+                                           code_model: Optional[str] = None) -> int:
+        """
+        解析 context_prune_safe_zone_tokens 参数
+        
+        Args:
+            value: 参数值
+            code_model: 代码模型名称
+            
+        Returns:
+            解析后的 token 数量
+        """
+        try:
+            return self.token_parser.parse_token_value(value, code_model)
+        except ValueError as e:
+            default_value = 24 * 1024
+            print(f"Warning: Failed to parse context_prune_safe_zone_tokens '{value}': {e}. Using default value: {default_value}")
+            return default_value
+    
+    def parse_token_field(self, 
+                         field_name: str,
+                         value: Union[str, int, float],
+                         code_model: Optional[str] = None,
+                         default_value: Optional[int] = None) -> int:
+        """
+        通用的 token 字段解析方法
+        
+        Args:
+            field_name: 字段名称
+            value: 字段值
+            code_model: 代码模型名称
+            default_value: 默认值，如果为 None 则使用 50*1024
+            
+        Returns:
+            解析后的 token 数量
+        """
+        try:
+            return self.token_parser.parse_token_value(value, code_model)
+        except ValueError as e:
+            fallback_value = default_value or 50 * 1024
+            print(f"Warning: Failed to parse {field_name} '{value}': {e}. Using default value: {fallback_value}")
+            return fallback_value
+    
+    def validate_parsed_args(self, args_dict: Dict[str, Any], 
+                           code_model: Optional[str] = None) -> Dict[str, Any]:
+        """
+        验证和解析整个参数字典
+        
+        Args:
+            args_dict: 参数字典
+            code_model: 代码模型名称
+            
+        Returns:
+            验证和解析后的参数字典
+        """
+        # 定义需要解析的 token 字段及其默认值
+        token_fields = {
+            'conversation_prune_safe_zone_tokens': 50 * 1024,
+            'context_prune_safe_zone_tokens': 24 * 1024,
+            'context_prune_sliding_window_size': 1000,
+            'context_prune_sliding_window_overlap': 100,
+            'rag_params_max_tokens': 500000,
+            'rag_context_window_limit': 120000,
+            'rag_duckdb_vector_dim': 1024,
+            'rag_duckdb_query_top_k': 10000,
+            'rag_emb_dim': 1024,
+            'rag_emb_text_size': 1024,
+            'hybrid_index_max_output_tokens': 1000000,
+            'data_cells_max_num': 2000,
+        }
+        
+        parsed_dict = args_dict.copy()
+        
+        for field_name, default_value in token_fields.items():
+            if field_name in parsed_dict:
+                original_value = parsed_dict[field_name]
+                # 如果值不是简单的 int，则进行解析
+                if not isinstance(original_value, int):
+                    parsed_dict[field_name] = self.parse_token_field(
+                        field_name, original_value, code_model, default_value
+                    )
+        
+        return parsed_dict
+    
+    def get_parsing_info(self) -> Dict[str, Any]:
+        """
+        获取解析器信息和使用示例
+        
+        Returns:
+            包含解析器信息的字典
+        """
+        return {
+            "supported_types": ["int", "float", "str"],
+            "string_formats": self.token_parser.get_examples(),
+            "supported_units": self.token_parser.get_supported_units(),
+            "float_behavior": "Float values (0-1) are treated as percentages of the code_model's context_window",
+            "fallback_context_window": 50 * 1024,
+            "safety_notes": [
+                "Invalid expressions fall back to default values",
+                "Negative results are clamped to 0",
+                "Division by zero is handled gracefully"
+            ]
+        }
+    
+    def create_flexible_field(self, 
+                             default_value: Union[str, int, float],
+                             description: str,
+                             min_value: Optional[int] = None,
+                             max_value: Optional[int] = None) -> Field:
+        """
+        创建支持灵活解析的 Pydantic 字段
+        
+        Args:
+            default_value: 默认值
+            description: 字段描述
+            min_value: 最小值
+            max_value: 最大值
+            
+        Returns:
+            Pydantic Field 对象
+        """
+        field_kwargs = {
+            "default": default_value,
+            "description": description
+        }
+        
+        if min_value is not None:
+            field_kwargs["ge"] = min_value
+        if max_value is not None:
+            field_kwargs["le"] = max_value
+            
+        return Field(**field_kwargs) 

autocoder/common/autocoderargs_parser/tests/__init__.py

@@ -0,0 +1 @@
+# 测试模块初始化文件 

autocoder/common/autocoderargs_parser/tests/test_args_parser.py

@@ -0,0 +1,235 @@
+import pytest
+import tempfile
+from pathlib import Path
+from unittest.mock import Mock
+
+from autocoder.common.autocoderargs_parser import AutoCoderArgsParser, TokenParser
+from autocoder.common.llms import LLMManager, LLMModel
+
+
+@pytest.fixture
+def mock_llm_manager():
+    """创建 mock LLM manager fixture"""
+    mock_manager = Mock(spec=LLMManager)
+    
+    # 设置默认的模型信息
+    mock_model = Mock(spec=LLMModel)
+    mock_model.context_window = 32768  # deepseek/v3 的默认值
+    mock_manager.get_model.return_value = mock_model
+    
+    return mock_manager
+
+
+@pytest.fixture
+def args_parser(mock_llm_manager):
+    """创建 AutoCoderArgsParser 实例 fixture"""
+    return AutoCoderArgsParser(mock_llm_manager)
+
+
+class TestAutoCoderArgsParser:
+    """AutoCoderArgsParser 测试类"""
+    
+    @pytest.mark.parametrize("value,expected", [
+        (51200, 51200),  # 整数
+        ("50k", 51200),  # 带单位字符串
+        ("50*1024", 51200),  # 数学表达式
+        (0.1, 3276),  # 浮点数比例 (32768 * 0.1)
+    ])
+    def test_parse_conversation_prune_safe_zone_tokens(self, args_parser, value, expected):
+        """测试 conversation_prune_safe_zone_tokens 解析"""
+        result = args_parser.parse_conversation_prune_safe_zone_tokens(value, "deepseek/v3")
+        assert result == expected
+    
+    def test_parse_conversation_prune_safe_zone_tokens_invalid(self, args_parser, capsys):
+        """测试无效值的处理"""
+        result = args_parser.parse_conversation_prune_safe_zone_tokens("invalid", "deepseek/v3")
+        
+        # 应该返回默认值
+        assert result == 50 * 1024
+        
+        # 应该有警告信息
+        captured = capsys.readouterr()
+        assert "Warning: Failed to parse conversation_prune_safe_zone_tokens" in captured.out
+    
+    @pytest.mark.parametrize("value,expected", [
+        (24576, 24576),  # 整数
+        ("24k", 24576),  # 带单位字符串
+        ("24*1024", 24576),  # 数学表达式
+        (0.05, 1638),  # 浮点数比例 (32768 * 0.05)
+    ])
+    def test_parse_context_prune_safe_zone_tokens(self, args_parser, value, expected):
+        """测试 context_prune_safe_zone_tokens 解析"""
+        result = args_parser.parse_context_prune_safe_zone_tokens(value, "deepseek/v3")
+        assert result == expected
+    
+    def test_parse_context_prune_safe_zone_tokens_invalid(self, args_parser, capsys):
+        """测试 context_prune_safe_zone_tokens 无效值处理"""
+        result = args_parser.parse_context_prune_safe_zone_tokens("invalid", "deepseek/v3")
+        
+        # 应该返回默认值
+        assert result == 24 * 1024
+        
+        # 应该有警告信息
+        captured = capsys.readouterr()
+        assert "Warning: Failed to parse context_prune_safe_zone_tokens" in captured.out
+    
+    @pytest.mark.parametrize("field_name,value,default_value,expected", [
+        ("rag_params_max_tokens", "500k", 500000, 512000),
+        ("data_cells_max_num", "2k", 2000, 2048),
+        ("context_prune_sliding_window_size", 1000, 1000, 1000),
+        ("hybrid_index_max_output_tokens", "1m", 1000000, 1048576),
+    ])
+    def test_parse_token_field(self, args_parser, field_name, value, default_value, expected):
+        """测试通用 token 字段解析"""
+        result = args_parser.parse_token_field(field_name, value, "deepseek/v3", default_value)
+        assert result == expected
+    
+    def test_parse_token_field_invalid(self, args_parser, capsys):
+        """测试通用 token 字段无效值处理"""
+        field_name = "test_field"
+        result = args_parser.parse_token_field(field_name, "invalid", "deepseek/v3", 1000)
+        
+        # 应该返回提供的默认值
+        assert result == 1000
+        
+        # 应该有警告信息
+        captured = capsys.readouterr()
+        assert f"Warning: Failed to parse {field_name}" in captured.out
+    
+    def test_validate_parsed_args(self, args_parser):
+        """测试批量参数验证和解析"""
+        args_dict = {
+            "conversation_prune_safe_zone_tokens": "50k",
+            "context_prune_safe_zone_tokens": "24*1024",
+            "rag_params_max_tokens": 0.5,  # 比例值
+            "data_cells_max_num": "2k",
+            "other_field": "not_a_token_field",  # 非 token 字段
+            "regular_int_field": 12345  # 已经是 int 的字段
+        }
+        
+        parsed_dict = args_parser.validate_parsed_args(args_dict, "deepseek/v3")
+        
+        # 检查 token 字段被正确解析
+        assert parsed_dict["conversation_prune_safe_zone_tokens"] == 51200
+        assert parsed_dict["context_prune_safe_zone_tokens"] == 24576
+        assert parsed_dict["rag_params_max_tokens"] == 16384  # 32768 * 0.5
+        assert parsed_dict["data_cells_max_num"] == 2048
+        
+        # 检查非 token 字段保持不变
+        assert parsed_dict["other_field"] == "not_a_token_field"
+        assert parsed_dict["regular_int_field"] == 12345
+    
+    def test_validate_parsed_args_with_non_token_fields_only(self, args_parser):
+        """测试只包含非 token 字段的参数字典"""
+        args_dict = {
+            "model": "deepseek/v3",
+            "query": "test query",
+            "output": "result.txt",
+            "skip_confirm": True
+        }
+        
+        parsed_dict = args_parser.validate_parsed_args(args_dict, "deepseek/v3")
+        
+        # 所有字段应该保持不变
+        assert parsed_dict == args_dict
+    
+    def test_get_parsing_info(self, args_parser):
+        """测试获取解析器信息"""
+        info = args_parser.get_parsing_info()
+        
+        # 检查必要的信息字段
+        assert "supported_types" in info
+        assert "string_formats" in info
+        assert "supported_units" in info
+        assert "float_behavior" in info
+        assert "fallback_context_window" in info
+        assert "safety_notes" in info
+        
+        # 检查支持的类型
+        assert "int" in info["supported_types"]
+        assert "float" in info["supported_types"]
+        assert "str" in info["supported_types"]
+        
+        # 检查字符串格式示例
+        assert isinstance(info["string_formats"], dict)
+        assert "整数" in info["string_formats"]
+        
+        # 检查支持的单位
+        assert isinstance(info["supported_units"], dict)
+        assert "k" in info["supported_units"]
+        assert info["supported_units"]["k"] == 1024
+    
+    def test_float_value_with_zero_context_window(self, args_parser, mock_llm_manager):
+        """测试当模型 context_window 为 0 时的回退机制"""
+        # Mock 模型的 context_window 为 0
+        mock_model = Mock(spec=LLMModel)
+        mock_model.context_window = 0
+        mock_llm_manager.get_model.return_value = mock_model
+        
+        result = args_parser.parse_conversation_prune_safe_zone_tokens(0.1, "test/model")
+        # 应该使用默认值 120*1000
+        expected = int((120 * 1000) * 0.1)
+        assert result == expected
+    
+    def test_float_value_with_missing_model(self, args_parser, mock_llm_manager):
+        """测试当模型不存在时的回退机制"""
+        # Mock 获取模型返回 None
+        mock_llm_manager.get_model.return_value = None
+        
+        result = args_parser.parse_conversation_prune_safe_zone_tokens(0.1, "nonexistent/model")
+        # 应该使用默认值 120*1000
+        expected = int((120 * 1000) * 0.1)
+        assert result == expected
+    
+    def test_args_parser_uses_token_parser(self, args_parser):
+        """测试 AutoCoderArgsParser 正确使用 TokenParser"""
+        # 确保 AutoCoderArgsParser 内部使用 TokenParser
+        assert hasattr(args_parser, 'token_parser')
+        assert isinstance(args_parser.token_parser, TokenParser)
+        assert args_parser.token_parser.llm_manager == args_parser.llm_manager
+
+
+class TestAutoCoderArgsParserIntegration:
+    """AutoCoderArgsParser 集成测试"""
+    
+    def test_integration_with_real_llm_manager(self):
+        """测试与真实 LLM Manager 的集成"""
+        with tempfile.TemporaryDirectory() as temp_dir:
+            models_json = Path(temp_dir) / "models.json"
+            
+            # 创建真实的 LLM Manager（使用临时目录）
+            from autocoder.common.llms import LLMManager
+            llm_manager = LLMManager(str(models_json))
+            
+            # 创建 AutoCoderArgsParser
+            parser = AutoCoderArgsParser(llm_manager)
+            
+            # 测试基本功能
+            result = parser.parse_conversation_prune_safe_zone_tokens("50k")
+            assert result == 51200
+            
+            # 测试浮点数（由于使用默认模型，会使用回退值）
+            result = parser.parse_conversation_prune_safe_zone_tokens(0.1)
+            expected = int((120 * 1000) * 0.1)
+            assert result == expected
+    
+    @pytest.mark.parametrize("field_value", [
+        "50k",
+        "50*1024", 
+        0.1,
+        51200
+    ])
+    def test_all_supported_formats(self, field_value):
+        """测试所有支持的格式都能正常工作"""
+        # 使用临时 LLM Manager
+        with tempfile.TemporaryDirectory() as temp_dir:
+            models_json = Path(temp_dir) / "models.json"
+            
+            from autocoder.common.llms import LLMManager
+            llm_manager = LLMManager(str(models_json))
+            parser = AutoCoderArgsParser(llm_manager)
+            
+            # 所有格式都应该成功解析，不抛出异常
+            result = parser.parse_conversation_prune_safe_zone_tokens(field_value)
+            assert isinstance(result, int)
+            assert result >= 0 

autocoder/common/autocoderargs_parser/tests/test_token_parser.py

@@ -0,0 +1,195 @@
+import pytest
+import tempfile
+from pathlib import Path
+from unittest.mock import Mock
+
+from autocoder.common.autocoderargs_parser.token_parser import TokenParser
+from autocoder.common.llms import LLMManager, LLMModel
+
+
+@pytest.fixture
+def mock_llm_manager():
+    """创建 mock LLM manager fixture"""
+    return Mock(spec=LLMManager)
+
+
+@pytest.fixture
+def token_parser(mock_llm_manager):
+    """创建 TokenParser 实例 fixture"""
+    return TokenParser(mock_llm_manager)
+
+
+@pytest.fixture
+def temp_dir():
+    """创建临时目录 fixture"""
+    with tempfile.TemporaryDirectory() as temp_dir:
+        yield temp_dir
+
+
+class TestTokenParser:
+    """TokenParser 测试类"""
+    
+    def test_parse_int_value(self, token_parser):
+        """测试整数值解析"""
+        result = token_parser.parse_token_value(51200)
+        assert result == 51200
+        
+        # 测试负数会被转为 0
+        result = token_parser.parse_token_value(-100)
+        assert result == 0
+    
+    def test_parse_float_value(self, token_parser, mock_llm_manager):
+        """测试浮点数值解析"""
+        # Mock 模型信息
+        mock_model = Mock(spec=LLMModel)
+        mock_model.context_window = 100000
+        mock_llm_manager.get_model.return_value = mock_model
+        
+        result = token_parser.parse_token_value(0.1, "test/model")
+        assert result == 10000  # 100000 * 0.1
+        
+        # 测试无效的浮点数范围
+        with pytest.raises(ValueError):
+            token_parser.parse_token_value(1.5, "test/model")
+        
+        with pytest.raises(ValueError):
+            token_parser.parse_token_value(-0.1, "test/model")
+    
+    def test_parse_float_value_fallback(self, token_parser, mock_llm_manager):
+        """测试浮点数解析的回退机制"""
+        # Mock 获取模型失败
+        mock_llm_manager.get_model.return_value = None
+        
+        result = token_parser.parse_token_value(0.1)
+        expected = int((120 * 1000) * 0.1)  # 默认 context_window
+        assert result == expected
+    
+    @pytest.mark.parametrize("input_str,expected", [
+        ("100k", 100 * 1024),
+        ("1m", 1024 * 1024),
+        ("2.5k", int(2.5 * 1024)),
+        ("1g", 1024 * 1024 * 1024),
+        ("50K", 50 * 1024),  # 大写
+        ("1M", 1024 * 1024),  # 大写
+        ("100kb", 100 * 1024),  # 带 b
+        ("2MB", 2 * 1024 * 1024),  # 大写带 b
+    ])
+    def test_parse_unit_strings(self, token_parser, input_str, expected):
+        """测试带单位的字符串解析"""
+        result = token_parser.parse_token_value(input_str)
+        assert result == expected
+    
+    @pytest.mark.parametrize("input_str,expected", [
+        ("1024", 1024),        # 纯整数字符串
+        ("0", 0),              # 零
+        ("8", 8),              # 单个数字
+        ("123456", 123456),    # 大数字
+        ("1024.0", 1024),      # 浮点数字符串（整数值）
+        ("1024.5", 1024),      # 浮点数字符串（小数会被截断）
+        ("1024.", 1024),       # 末尾有小数点
+        (".5", 0),             # 纯小数（< 1 的会被转为 0）
+        ("0.8", 0),            # 小数（< 1 的会被转为 0）
+        ("100.99", 100),       # 小数截断
+        ("-10", 0),            # 负数会被转为 0
+        ("-100.5", 0),         # 负浮点数会被转为 0
+    ])
+    def test_parse_numeric_strings(self, token_parser, input_str, expected):
+        """测试纯数字字符串解析"""
+        result = token_parser.parse_token_value(input_str)
+        assert result == expected
+    
+    @pytest.mark.parametrize("input_str,expected", [
+        ("00123", 123),        # 前导零
+        ("123.000", 123),      # 后续零
+        ("0000", 0),           # 多个零
+        ("1.0000", 1),         # 浮点数后续零
+        ("0001.5", 1),         # 前导零加浮点数
+    ])
+    def test_parse_numeric_strings_edge_cases(self, token_parser, input_str, expected):
+        """测试纯数字字符串解析的边界情况"""
+        result = token_parser.parse_token_value(input_str)
+        assert result == expected
+    
+    def test_parse_priority_order(self, token_parser):
+        """测试解析优先级：纯数字 > 带单位 > 数学表达式"""
+        # 确保纯数字不会被误认为是数学表达式
+        assert token_parser.parse_token_value("1024") == 1024
+        assert token_parser.parse_token_value("1024.0") == 1024
+        
+        # 确保带单位的解析优先于数学表达式（虽然这种情况不太可能发生冲突）
+        assert token_parser.parse_token_value("1k") == 1024
+        assert token_parser.parse_token_value("1m") == 1024 * 1024
+        
+        # 确保数学表达式正常工作
+        assert token_parser.parse_token_value("1+1") == 2
+        assert token_parser.parse_token_value("2*2") == 4
+    
+    @pytest.mark.parametrize("expr,expected", [
+        ("50*1024", 50 * 1024),
+        ("100*1000", 100 * 1000),
+        ("2**16", 2**16),
+        ("1024+512", 1024 + 512),
+        ("2048-512", 2048 - 512),
+        ("100*10+50", 100 * 10 + 50),
+        ("(100+50)*10", (100 + 50) * 10),
+        ("1024//2", 1024 // 2),
+        ("1024%100", 1024 % 100),
+    ])
+    def test_parse_math_expressions(self, token_parser, expr, expected):
+        """测试数学表达式解析"""
+        result = token_parser.parse_token_value(expr)
+        assert result == expected
+    
+    @pytest.mark.parametrize("invalid_expr", [
+        "invalid_string",
+        "100x",  # 无效单位
+        "1024 + undefined_var",  # 未定义变量
+        "1024 / 0",  # 除零
+        "",  # 空字符串
+        "   ",  # 只有空格
+    ])
+    def test_parse_invalid_expressions(self, token_parser, invalid_expr):
+        """测试无效表达式处理"""
+        with pytest.raises(ValueError):
+            token_parser.parse_token_value(invalid_expr)
+    
+    def test_validate_token_value(self, token_parser):
+        """测试值验证功能"""
+        # 有效值
+        assert token_parser.validate_token_value(1024) is True
+        assert token_parser.validate_token_value("50k") is True
+        assert token_parser.validate_token_value("50*1024") is True
+        
+        # 范围验证
+        assert token_parser.validate_token_value(1024, min_tokens=500, max_tokens=2000) is True
+        assert token_parser.validate_token_value(100, min_tokens=500, max_tokens=2000) is False
+        assert token_parser.validate_token_value(3000, min_tokens=500, max_tokens=2000) is False
+        
+        # 无效值
+        assert token_parser.validate_token_value("invalid") is False
+    
+    def test_get_supported_units(self, token_parser):
+        """测试获取支持的单位"""
+        units = token_parser.get_supported_units()
+        assert 'k' in units
+        assert 'm' in units
+        assert 'g' in units
+        assert units['k'] == 1024
+        assert units['m'] == 1024 * 1024
+    
+    def test_get_examples(self, token_parser):
+        """测试获取使用示例"""
+        examples = token_parser.get_examples()
+        assert isinstance(examples, dict)
+        assert "整数" in examples
+        assert "带单位" in examples
+        assert "数学表达式" in examples
+    
+    @pytest.mark.parametrize("invalid_value", [
+        [1, 2, 3],  # 列表类型不支持
+        {"value": 1024},  # 字典类型不支持
+    ])
+    def test_unsupported_type(self, token_parser, invalid_value):
+        """测试不支持的类型"""
+        with pytest.raises(ValueError):
+            token_parser.parse_token_value(invalid_value)