auto-coder 0.1.255__py3-none-any.whl → 0.1.257__py3-none-any.whl

autocoder/index/index.py

@@ -9,6 +9,7 @@ from autocoder.index.symbols_utils import (
     SymbolType,
     symbols_info_to_str,
 )
+from autocoder.privacy.model_filter import ModelPathFilter
 from concurrent.futures import ThreadPoolExecutor, as_completed
 import threading
 
@@ -17,6 +18,7 @@ import hashlib
 
 from autocoder.common.printer import Printer
 from autocoder.common.auto_coder_lang import get_message
+from autocoder.utils.llms import get_llm_names, get_model_info
 from autocoder.index.types import (
     IndexItem,
     TargetFile,
@@ -30,6 +32,9 @@ class IndexManager:
     ):
         self.sources = sources
         self.source_dir = args.source_dir
+        # Initialize model filter for index_llm and index_filter_llm
+        self.index_model_filter = None
+        self.index_filter_model_filter = None
         self.anti_quota_limit = (
             args.index_model_anti_quota_limit or args.anti_quota_limit
         )
@@ -46,6 +51,12 @@ class IndexManager:
             self.index_filter_llm = llm
 
         self.llm = llm
+        
+        # Initialize model filters
+        if self.index_llm:
+            self.index_model_filter = ModelPathFilter.from_model_object(self.index_llm, args)
+        if self.index_filter_llm:
+            self.index_filter_model_filter = ModelPathFilter.from_model_object(self.index_filter_llm, args)
         self.args = args
         self.max_input_length = (
             args.index_model_max_input_length or args.model_max_input_length
@@ -194,6 +205,17 @@ class IndexManager:
         ext = os.path.splitext(file_path)[1].lower()
         if ext in [".md", ".html", ".txt", ".doc", ".pdf"]:
             return True
+            
+        # Check model filter restrictions
+        if self.index_model_filter and not self.index_model_filter.is_accessible(file_path):
+            self.printer.print_in_terminal(
+                "index_file_filtered",
+                style="yellow",
+                file_path=file_path,
+                model_name=",".join(get_llm_names(self.index_llm))
+            )
+            return True
+            
         return False
 
     def build_index_for_single_source(self, source: SourceCode):   
@@ -212,8 +234,29 @@ class IndexManager:
         model_name = ",".join(get_llm_names(self.index_llm))
 
         try:
+            # 获取模型名称列表
+            model_names = get_llm_names(self.index_llm)
+            model_name = ",".join(model_names)
+
+            # 获取模型价格信息
+            model_info_map = {}
+            for name in model_names:
+                info = get_model_info(name, self.args.product_mode)
+                if info:
+                    model_info_map[name] = {
+                        "input_price": info.get("input_price", 0.0),
+                        "output_price": info.get("output_price", 0.0)
+                    }
+
             start_time = time.monotonic()
             source_code = source.source_code
+            
+            # 统计token和成本
+            total_input_tokens = 0
+            total_output_tokens = 0
+            total_input_cost = 0.0
+            total_output_cost = 0.0
+            
             if len(source.source_code) > self.max_input_length:
                 self.printer.print_in_terminal(
                     "index_file_too_large",
@@ -227,15 +270,38 @@ class IndexManager:
                 )
                 symbols = []
                 for chunk in chunks:
+                    meta_holder = byzerllm.MetaHolder()
                     chunk_symbols = self.get_all_file_symbols.with_llm(
-                        self.index_llm).run(source.module_name, chunk)
+                        self.index_llm).with_meta(meta_holder).run(source.module_name, chunk)
                     time.sleep(self.anti_quota_limit)
                     symbols.append(chunk_symbols)
+                    
+                    if meta_holder.get_meta():
+                        meta_dict = meta_holder.get_meta()
+                        total_input_tokens += meta_dict.get("input_tokens_count", 0)
+                        total_output_tokens += meta_dict.get("generated_tokens_count", 0)
+                        
                 symbols = "\n".join(symbols)
             else:
+                meta_holder = byzerllm.MetaHolder()
                 symbols = self.get_all_file_symbols.with_llm(
-                    self.index_llm).run(source.module_name, source_code)
+                    self.index_llm).with_meta(meta_holder).run(source.module_name, source_code)
                 time.sleep(self.anti_quota_limit)
+                
+                if meta_holder.get_meta():
+                    meta_dict = meta_holder.get_meta()
+                    total_input_tokens += meta_dict.get("input_tokens_count", 0)
+                    total_output_tokens += meta_dict.get("generated_tokens_count", 0)
+            
+            # 计算总成本
+            for name in model_names:
+                info = model_info_map.get(name, {})
+                total_input_cost += (total_input_tokens * info.get("input_price", 0.0)) / 1000000
+                total_output_cost += (total_output_tokens * info.get("output_price", 0.0)) / 1000000
+            
+            # 四舍五入到4位小数
+            total_input_cost = round(total_input_cost, 4)
+            total_output_cost = round(total_output_cost, 4)
             
             self.printer.print_in_terminal(
                 "index_update_success",
@@ -243,7 +309,11 @@ class IndexManager:
                 file_path=file_path,
                 md5=md5,
                 duration=time.monotonic() - start_time,
-                model_name=model_name
+                model_name=model_name,
+                input_tokens=total_input_tokens,
+                output_tokens=total_output_tokens,
+                input_cost=total_input_cost,
+                output_cost=total_output_cost
             )
 
         except Exception as e:
@@ -263,6 +333,10 @@ class IndexManager:
             "symbols": symbols,
             "last_modified": os.path.getmtime(file_path),
             "md5": md5,
+            "input_tokens_count": total_input_tokens,
+            "generated_tokens_count": total_output_tokens,
+            "input_tokens_cost": total_input_cost,
+            "generated_tokens_cost": total_output_cost
         }
 
     def build_index(self):
@@ -290,6 +364,11 @@ class IndexManager:
 
         updated_sources = []
 
+        total_input_tokens = 0
+        total_output_tokens = 0
+        total_input_cost = 0.0
+        total_output_cost = 0.0
+
         with ThreadPoolExecutor(max_workers=self.args.index_build_workers) as executor:
 
             wait_to_build_files = []
@@ -346,6 +425,10 @@ class IndexManager:
                         num_files=num_files
                     )
                     module_name = result["module_name"]
+                    total_input_tokens += result["input_tokens_count"]
+                    total_output_tokens += result["generated_tokens_count"]
+                    total_input_cost += result["input_tokens_cost"]
+                    total_output_cost += result["generated_tokens_cost"]
                     index_data[module_name] = result
                     updated_sources.append(module_name)
                     if len(updated_sources) > 5:
@@ -357,12 +440,19 @@ class IndexManager:
         if updated_sources or keys_to_remove:
             with open(self.index_file, "w") as file:
                 json.dump(index_data, file, ensure_ascii=False, indent=2)
+
+            print("")    
             self.printer.print_in_terminal(
                 "index_file_saved",
                 style="green",
                 updated_files=len(updated_sources),
-                removed_files=len(keys_to_remove)
+                removed_files=len(keys_to_remove),
+                input_tokens=total_input_tokens,
+                output_tokens=total_output_tokens,
+                input_cost=total_input_cost,
+                output_cost=total_output_cost
             )
+            print("")
 
         return index_data
 

autocoder/models.py

@@ -110,6 +110,20 @@ def save_models(models: List[Dict]) -> None:
         json.dump(models, f, indent=2, ensure_ascii=False)
 
 
+def add_and_activate_models(models: List[Dict]) -> None:
+    """
+    添加模型
+    """
+    exits_models = load_models()
+    for model in models:        
+        if model["name"] not in [m["name"] for m in exits_models]:
+            exits_models.append(model)
+    save_models(exits_models)
+
+    for model in models:
+        if "api_key" in model:
+            update_model_with_api_key(model["name"], model["api_key"])
+
 def get_model_by_name(name: str) -> Dict:
     """
     根据模型名称查找模型
@@ -127,11 +141,23 @@ def update_model_input_price(name: str, price: float) -> bool:
     """更新模型输入价格
     
     Args:
-        name: 模型名称
-        price: 输入价格(M/百万input tokens)
+        name (str): 要更新的模型名称，必须与models.json中的记录匹配
+        price (float): 新的输入价格，单位：美元/百万tokens。必须大于等于0
         
     Returns:
-        bool: 是否更新成功
+        bool: 是否成功找到并更新了模型价格
+        
+    Raises:
+        ValueError: 如果price为负数时抛出
+        
+    Example:
+        >>> update_model_input_price("gpt-4", 3.0)
+        True
+        
+    Notes:
+        1. 价格设置后会立即生效并保存到models.json
+        2. 实际费用计算时会按实际使用量精确到小数点后6位
+        3. 设置价格为0表示该模型当前不可用
     """
     if price < 0:
         raise ValueError("Price cannot be negative")
@@ -151,11 +177,23 @@ def update_model_output_price(name: str, price: float) -> bool:
     """更新模型输出价格
     
     Args:
-        name: 模型名称
-        price: 输出价格(M/百万output tokens)
+        name (str): 要更新的模型名称，必须与models.json中的记录匹配
+        price (float): 新的输出价格，单位：美元/百万tokens。必须大于等于0
         
     Returns:
-        bool: 是否更新成功
+        bool: 是否成功找到并更新了模型价格
+        
+    Raises:
+        ValueError: 如果price为负数时抛出
+        
+    Example:
+        >>> update_model_output_price("gpt-4", 6.0)
+        True
+        
+    Notes:
+        1. 输出价格通常比输入价格高30%-50%
+        2. 对于按token计费的API，实际收费按(input_tokens * input_price + output_tokens * output_price)计算
+        3. 价格变更会影响所有依赖模型计费的功能（如成本预测、用量监控等）
     """
     if price < 0:
         raise ValueError("Price cannot be negative")

autocoder/privacy/__init__.py

@@ -0,0 +1,3 @@
+from .model_filter import ModelPathFilter
+
+__all__ = ["ModelPathFilter"]

autocoder/privacy/model_filter.py

@@ -0,0 +1,100 @@
+import re
+import yaml
+from pathlib import Path
+from typing import Dict, List, Optional
+from autocoder.common import AutoCoderArgs
+from autocoder.utils import llms as llm_utils
+
+
+class ModelPathFilter:
+    def __init__(self,
+                 model_name: str,
+                 args: AutoCoderArgs,
+                 default_forbidden: List[str] = None):
+        """
+        模型路径过滤器
+        :param model_name: 当前使用的模型名称
+        :param args: 自动编码器参数
+        :param default_forbidden: 默认禁止路径规则
+        """
+        self.model_name = model_name
+        if args.model_filter_path:
+            self.config_path = Path(args.model_filter_path)
+        else:
+            self.config_path = Path(args.source_dir, ".model_filters.yml")
+        self.default_forbidden = default_forbidden or []
+        self._rules_cache: Dict[str, List[re.Pattern]] = {}
+        self._load_rules()
+
+    def _load_rules(self):
+        """加载并编译正则规则"""
+        if not self.config_path.exists():            
+            return
+
+        with open(self.config_path, 'r', encoding="utf-8") as f:
+            config = yaml.safe_load(f)
+
+        model_rules = config.get('model_filters', {}).get(self.model_name, {})
+        all_rules = model_rules.get('forbidden_paths', []) + self.default_forbidden
+
+        # 预编译正则表达式
+        self._rules_cache[self.model_name] = [
+            re.compile(rule) for rule in all_rules
+        ]
+
+    def is_accessible(self, file_path: str) -> bool:
+        """
+        检查文件路径是否符合访问规则
+        :return: True表示允许访问,False表示禁止
+        """
+        # 优先使用模型专属规则
+        patterns = self._rules_cache.get(self.model_name, [])
+
+        # 回退到默认规则
+        if not patterns and self.default_forbidden:
+            patterns = [re.compile(rule) for rule in self.default_forbidden]
+
+        # 如果路径为空或None,直接返回True
+        if not file_path:
+            return True
+
+        return not any(pattern.search(file_path) for pattern in patterns)
+
+    def add_temp_rule(self, rule: str):
+        """
+        添加临时规则
+        :param rule: 正则表达式规则
+        """
+        patterns = self._rules_cache.get(self.model_name, [])
+        patterns.append(re.compile(rule))
+        self._rules_cache[self.model_name] = patterns
+
+    def reload_rules(self):
+        """重新加载规则配置"""
+        self._rules_cache.clear()
+        self._load_rules()
+
+    def has_rules(self):
+        """检查是否存在规则"""
+        return bool(self._rules_cache.get(self.model_name, []))
+
+    @classmethod
+    def from_model_object(cls,
+                         llm_obj,
+                         args: AutoCoderArgs,
+                         default_forbidden: Optional[List[str]] = None):
+        """
+        从LLM对象创建过滤器
+        :param llm_obj: ByzerLLM实例或类似对象
+        :param args: 自动编码器参数
+        :param default_forbidden: 默认禁止路径规则
+        """
+        model_name = ",".join(llm_utils.get_llm_names(llm_obj))
+        if not model_name:
+            raise ValueError(f"{model_name} is not found")
+
+        return cls(
+            model_name=model_name,
+            args=args,
+            default_forbidden=default_forbidden
+        )

autocoder/pyproject/__init__.py

@@ -116,6 +116,7 @@ class PyProject:
             "actions",
             ".vscode",
             ".idea",
+            "venv",
         ]
 
     @byzerllm.prompt()

autocoder/suffixproject/__init__.py

@@ -56,6 +56,7 @@ class SuffixProject:
             ".vscode",
             "actions",
             ".idea",
+            "venv",
         ]
 
     @byzerllm.prompt()

autocoder/tsproject/__init__.py

@@ -48,6 +48,7 @@ class TSProject:
             "actions",
             ".vscode",
             ".idea",
+            "venv",
         ]
 
     @byzerllm.prompt()

autocoder/utils/llms.py

@@ -3,9 +3,15 @@ from typing import Union,Optional
 
 def get_llm_names(llm: Union[byzerllm.ByzerLLM, byzerllm.SimpleByzerLLM,str],target_model_type:Optional[str]=None):
    if target_model_type is None:
+      if isinstance(llm,list):
+         return [_llm.default_model_name for _llm in llm]
       return [llm.default_model_name for llm in [llm] if llm.default_model_name]
+   
    llms = llm.get_sub_client(target_model_type) 
+   
    if llms is None:
+      if isinstance(llm,list):
+         return [_llm.default_model_name for _llm in llm]
       return [llm.default_model_name for llm in [llm] if llm.default_model_name]
    elif isinstance(llms, list):
        return [llm.default_model_name for llm in llms if llm.default_model_name]
@@ -14,6 +20,27 @@ def get_llm_names(llm: Union[byzerllm.ByzerLLM, byzerllm.SimpleByzerLLM,str],tar
    else:
       return [llm.default_model_name for llm in [llms] if llm.default_model_name]
 
+def get_model_info(model_names: str, product_mode: str):    
+    from autocoder import models as models_module
+    def get_model_by_name(model_name: str):
+        try:
+            return models_module.get_model_by_name(model_name)
+        except Exception as e:
+            return None
+    
+    if product_mode == "pro":
+        return None
+
+    if product_mode == "lite":
+        if "," in model_names:
+            # Multiple code models specified
+            model_names = model_names.split(",")
+            for _, model_name in enumerate(model_names):
+                return get_model_by_name(model_name)
+        else:
+            # Single code model
+            return get_model_by_name(model_names)
+
 def get_single_llm(model_names: str, product_mode: str):
     from autocoder import models as models_module
     if product_mode == "pro":

autocoder/utils/model_provider_selector.py

@@ -0,0 +1,192 @@
+from prompt_toolkit.shortcuts import radiolist_dialog, input_dialog
+from prompt_toolkit.validation import Validator, ValidationError
+from rich.console import Console
+from typing import Optional, Dict, Any, List
+from autocoder.common.printer import Printer
+import re
+from pydantic import BaseModel
+
+from autocoder.models import process_api_key_path
+
+class ProviderInfo(BaseModel):
+    name: str
+    endpoint: str
+    r1_model: str
+    v3_model: str
+    api_key: str
+    r1_input_price: float
+    r1_output_price: float
+    v3_input_price: float
+    v3_output_price: float    
+
+
+PROVIDER_INFO_LIST = [
+    ProviderInfo(
+        name="volcano",
+        endpoint="https://ark.cn-beijing.volces.com/api/v3",   
+        r1_model="",
+        v3_model="",
+        api_key="",
+        r1_input_price=2.0,
+        r1_output_price=8.0,
+        v3_input_price=1.0,
+        v3_output_price=4.0,
+    ), 
+    ProviderInfo(
+        name="siliconFlow",
+        endpoint="https://api.siliconflow.cn/v1",        
+        r1_model="Pro/deepseek-ai/DeepSeek-R1",
+        v3_model="Pro/deepseek-ai/DeepSeek-V3",
+        api_key="",
+        r1_input_price=2.0,
+        r1_output_price=4.0,
+        v3_input_price=4.0,
+        v3_output_price=16.0,
+    ),
+    ProviderInfo(
+        name="deepseek",
+        endpoint="https://api.deepseek.com/v1",
+        r1_model="deepseek-reasoner",
+        v3_model="deepseek-chat",
+        api_key="",
+        r1_input_price=4.0,
+        r1_output_price=16.0,
+        v3_input_price=2.0,
+        v3_output_price=8.0,
+    ),
+]
+
+class VolcanoEndpointValidator(Validator):
+    def validate(self, document):
+        text = document.text
+        pattern = r'^ep-\d{14}-[a-z0-9]{5}$'
+        if not re.match(pattern, text):
+            raise ValidationError(
+                message='Invalid endpoint format. Should be like: ep-20250204215011-vzbsg',
+                cursor_position=len(text)
+            )
+
+class ModelProviderSelector:
+    def __init__(self):
+        self.printer = Printer()
+        self.console = Console()
+
+    def to_models_json(self, provider_info: ProviderInfo) -> List[Dict[str, Any]]:
+        """
+        Convert provider info to models.json format.
+        Returns a list of model configurations matching the format in models.py default_models_list.
+        
+        Args:
+            provider_info: ProviderInfo object containing provider details
+            
+        Returns:
+            List[Dict[str, Any]]: List of model configurations
+        """
+        models = []
+        
+        # Add R1 model (for reasoning/design/review)
+        if provider_info.r1_model:
+            models.append({
+                "name": f"r1_chat",
+                "description": f"{provider_info.name} R1 is for design/review",
+                "model_name": provider_info.r1_model,
+                "model_type": "saas/openai",
+                "base_url": provider_info.endpoint,
+                "api_key": provider_info.api_key,
+                "api_key_path": f"r1_chat",
+                "is_reasoning": True,
+                "input_price": provider_info.r1_input_price,
+                "output_price": provider_info.r1_output_price,
+                "average_speed": 0.0
+            })
+            
+        # Add V3 model (for coding)
+        if provider_info.v3_model:
+            models.append({
+                "name": f"v3_chat",
+                "description": f"{provider_info.name} Chat is for coding",
+                "model_name": provider_info.v3_model,
+                "model_type": "saas/openai",
+                "base_url": provider_info.endpoint,
+                "api_key": provider_info.api_key,
+                "api_key_path": f"v3_chat",
+                "is_reasoning": False,
+                "input_price": provider_info.v3_input_price,
+                "output_price": provider_info.v3_output_price,
+                "average_speed": 0.0
+            })
+            
+        return models
+        
+    def select_provider(self) -> Optional[Dict[str, Any]]:
+        """
+        Let user select a model provider and input necessary credentials.
+        Returns a dictionary with provider info or None if cancelled.
+        """
+        result = radiolist_dialog(
+            title=self.printer.get_message_from_key("model_provider_select_title"),
+            text=self.printer.get_message_from_key("model_provider_select_text"),
+            values=[
+                ("volcano", self.printer.get_message_from_key("model_provider_volcano")),
+                ("siliconflow", self.printer.get_message_from_key("model_provider_guiji")),
+                ("deepseek", self.printer.get_message_from_key("model_provider_deepseek"))
+            ]
+        ).run()
+        
+        if result is None:
+            return None
+
+    
+        provider_info = None
+        for provider in PROVIDER_INFO_LIST:
+            if provider.name == result:
+                provider_info = provider
+                break
+        
+        if result == "volcano":
+            # Get R1 endpoint
+            r1_endpoint = input_dialog(
+                title=self.printer.get_message_from_key("model_provider_api_key_title"),
+                text=self.printer.get_message_from_key("model_provider_volcano_r1_text"),
+                validator=VolcanoEndpointValidator()
+            ).run()
+            
+            if r1_endpoint is None:
+                return None
+            
+            provider_info.r1_model = r1_endpoint
+            
+            # Get V3 endpoint
+            v3_endpoint = input_dialog(
+                title=self.printer.get_message_from_key("model_provider_api_key_title"),
+                text=self.printer.get_message_from_key("model_provider_volcano_v3_text"),
+                validator=VolcanoEndpointValidator()
+            ).run()
+            
+            if v3_endpoint is None:
+                return None
+                
+            provider_info.v3_model = v3_endpoint
+        
+        # Get API key for all providers
+        api_key = input_dialog(
+            title=self.printer.get_message_from_key("model_provider_api_key_title"),
+            text=self.printer.get_message_from_key(f"model_provider_{result}_api_key_text"),
+            password=True
+        ).run()
+        
+        if api_key is None:
+            return None
+            
+        provider_info.api_key = api_key
+        
+        self.printer.print_panel(
+            self.printer.get_message_from_key("model_provider_selected"),
+            text_options={"justify": "left"},
+            panel_options={
+                "title": self.printer.get_message_from_key("model_provider_success_title"),
+                "border_style": "green"
+            }
+        )
+        
+        return provider_info

autocoder/version.py

@@ -1 +1 @@
-__version__ = "0.1.255"
+__version__ = "0.1.257"