auto-coder 0.1.256__py3-none-any.whl → 0.1.257__py3-none-any.whl

autocoder/dispacher/actions/action.py

@@ -27,6 +27,8 @@ from loguru import logger
 import time
 from autocoder.common.printer import Printer
 from autocoder.utils.llms import get_llm_names
+from autocoder.privacy.model_filter import ModelPathFilter
+from autocoder.common import SourceCodeList
 
 
 class BaseAction:    
@@ -55,14 +57,15 @@ class ActionTSProject(BaseAction):
         self.pp = pp
         pp.run()
 
-        source_code = pp.output()
+        # source_code = pp.output()
+        source_code_list = SourceCodeList(pp.sources)
         if self.llm:
             if args.in_code_apply:
                 old_query = args.query
                 args.query = (args.context or "") + "\n\n" + args.query
-            source_code = build_index_and_filter_files(
+            source_code_list = build_index_and_filter_files(
                 llm=self.llm, args=args, sources=pp.sources
-            )
+            )            
             if args.in_code_apply:
                 args.query = old_query
 
@@ -81,17 +84,21 @@ class ActionTSProject(BaseAction):
                 html_path=html_path,
                 max_iter=self.args.image_max_iter,
             )
-
+            html_code = ""
             with open(html_path, "r") as f:
                 html_code = f.read()
-                source_code = f"##File: {html_path}\n{html_code}\n\n" + source_code
+            
+            source_code_list.sources.append(SourceCode( 
+                module_name=html_path,
+                source_code=html_code,
+                tag="IMAGE"))            
 
-        self.process_content(source_code)
+        self.process_content(source_code_list)
         return True
 
-    def process_content(self, content: str):
+    def process_content(self, source_code_list: SourceCodeList):
         args = self.args
-
+        content = source_code_list.to_str()
         if args.execute and self.llm and not args.human_as_model:
             content_length = self._get_content_length(content)
             if content_length > self.args.model_max_input_length:
@@ -116,13 +123,14 @@ class ActionTSProject(BaseAction):
                 )
             else:
                 generate = CodeAutoGenerate(llm=self.llm, args=self.args, action=self)
+                        
             if self.args.enable_multi_round_generate:
                 generate_result = generate.multi_round_run(
-                    query=args.query, source_content=content
+                    query=args.query, source_code_list=source_code_list
                 )
             else:
                 generate_result = generate.single_round_run(
-                    query=args.query, source_content=content
+                    query=args.query, source_code_list=source_code_list
                 )
             elapsed_time = time.time() - start_time
             speed = generate_result.metadata.get('generated_tokens_count', 0) / elapsed_time if elapsed_time > 0 else 0
@@ -191,11 +199,12 @@ class ActionPyScriptProject(BaseAction):
         pp = Level1PyProject(
             script_path=args.script_path, package_name=args.package_name
         )
-        content = pp.run()
-        self.process_content(content)
+        pp.run()
+        source_code_list = SourceCodeList(pp.sources)
+        self.process_content(source_code_list)
         return True
 
-    def process_content(self, content: str):
+    def process_content(self, source_code_list: SourceCodeList):
         args = self.args
         if args.execute:
             self.printer.print_in_terminal("code_generation_start")
@@ -216,11 +225,11 @@ class ActionPyScriptProject(BaseAction):
                 generate = CodeAutoGenerate(llm=self.llm, args=self.args, action=self)
             if self.args.enable_multi_round_generate:
                 generate_result = generate.multi_round_run(
-                    query=args.query, source_content=content
+                    query=args.query, source_code_list=source_code_list
                 )
             else:
                 generate_result = generate.single_round_run(
-                    query=args.query, source_content=content
+                    query=args.query, source_code_list=source_code_list
                 )
 
             elapsed_time = time.time() - start_time
@@ -293,24 +302,24 @@ class ActionPyProject(BaseAction):
         pp = PyProject(args=self.args, llm=self.llm)
         self.pp = pp
         pp.run(packages=args.py_packages.split(",") if args.py_packages else [])            
-        source_code = pp.output()                
+        source_code_list = SourceCodeList(pp.sources)                
 
         if self.llm:
             old_query = args.query
             if args.in_code_apply:
                 args.query = (args.context or "") + "\n\n" + args.query
-            source_code = build_index_and_filter_files(
+            source_code_list = build_index_and_filter_files(
                 llm=self.llm, args=args, sources=pp.sources
             )
             if args.in_code_apply:
                 args.query = old_query
 
-        self.process_content(source_code)
+        self.process_content(source_code_list)
         return True
 
-    def process_content(self, content: str):
+    def process_content(self, source_code_list: SourceCodeList):
         args = self.args
-
+        content = source_code_list.to_str()
         if args.execute and self.llm and not args.human_as_model:
             content_length = self._get_content_length(content)
             if content_length > self.args.model_max_input_length:
@@ -342,11 +351,11 @@ class ActionPyProject(BaseAction):
 
             if self.args.enable_multi_round_generate:
                 generate_result = generate.multi_round_run(
-                    query=args.query, source_content=content
+                    query=args.query, source_code_list=source_code_list
                 )
             else:
                 generate_result = generate.single_round_run(
-                    query=args.query, source_content=content
+                    query=args.query, source_code_list=source_code_list
                 )
             elapsed_time = time.time() - start_time
             speed = generate_result.metadata.get('generated_tokens_count', 0) / elapsed_time if elapsed_time > 0 else 0
@@ -414,20 +423,21 @@ class ActionSuffixProject(BaseAction):
         pp = SuffixProject(args=args, llm=self.llm)
         self.pp = pp
         pp.run()
-        source_code = pp.output()
+        source_code_list = SourceCodeList(pp.sources)
         if self.llm:
             if args.in_code_apply:
                 old_query = args.query
                 args.query = (args.context or "") + "\n\n" + args.query
-            source_code = build_index_and_filter_files(
+            source_code_list = build_index_and_filter_files(
                 llm=self.llm, args=args, sources=pp.sources
             )
             if args.in_code_apply:
                 args.query = old_query
-        self.process_content(source_code)
+        self.process_content(source_code_list)
 
-    def process_content(self, content: str):
+    def process_content(self, source_code_list: SourceCodeList):
         args = self.args
+        content = source_code_list.to_str()
 
         if args.execute and self.llm and not args.human_as_model:
             content_length = self._get_content_length(content)
@@ -455,11 +465,11 @@ class ActionSuffixProject(BaseAction):
                 generate = CodeAutoGenerate(llm=self.llm, args=self.args, action=self)
             if self.args.enable_multi_round_generate:
                 generate_result = generate.multi_round_run(
-                    query=args.query, source_content=content
+                    query=args.query, source_code_list=source_code_list
                 )
             else:
                 generate_result = generate.single_round_run(
-                    query=args.query, source_content=content
+                    query=args.query, source_code_list=source_code_list
                 )
               
         elapsed_time = time.time() - start_time

autocoder/dispacher/actions/plugins/action_regex_project.py

@@ -15,6 +15,7 @@ from autocoder.utils.conversation_store import store_code_model_conversation
 from autocoder.common.printer import Printer
 import time
 from autocoder.utils.llms import get_llm_names
+from autocoder.common import SourceCodeList
 from loguru import logger
 class ActionRegexProject:
     def __init__(
@@ -36,20 +37,21 @@ class ActionRegexProject:
         pp = RegexProject(args=args, llm=self.llm)
         self.pp = pp
         pp.run()
-        source_code = pp.output()
+        source_code_list = SourceCodeList(pp.sources)
         if self.llm:
             if args.in_code_apply:
                 old_query = args.query
                 args.query = (args.context or "") + "\n\n" + args.query
-            source_code = build_index_and_filter_files(
+            source_code_list = build_index_and_filter_files(
                 llm=self.llm, args=args, sources=pp.sources
             )
             if args.in_code_apply:
                 args.query = old_query
-        self.process_content(source_code)
+        self.process_content(source_code_list)
 
-    def process_content(self, content: str):
+    def process_content(self, source_code_list: SourceCodeList):
         args = self.args
+        content = source_code_list.to_str()
 
         if args.execute and self.llm and not args.human_as_model:
             if len(content) > self.args.model_max_input_length:
@@ -78,11 +80,11 @@ class ActionRegexProject:
                 generate = CodeAutoGenerate(llm=self.llm, args=self.args, action=self)
             if self.args.enable_multi_round_generate:
                 generate_result = generate.multi_round_run(
-                    query=args.query, source_content=content
+                    query=args.query, source_code_list=source_code_list
                 )
             else:
                 generate_result = generate.single_round_run(
-                    query=args.query, source_content=content
+                    query=args.query, source_code_list=source_code_list
                 )
 
             elapsed_time = time.time() - start_time

autocoder/index/entry.py

@@ -23,10 +23,11 @@ from autocoder.index.filter.quick_filter import QuickFilter
 from autocoder.index.filter.normal_filter import NormalFilter
 from autocoder.index.index import IndexManager
 from loguru import logger
+from autocoder.common import SourceCodeList
 
 def build_index_and_filter_files(
     llm, args: AutoCoderArgs, sources: List[SourceCode]
-) -> str:
+) -> SourceCodeList:
     # Initialize timing and statistics
     total_start_time = time.monotonic()
     stats = {
@@ -253,7 +254,8 @@ def build_index_and_filter_files(
         for file in final_filenames:
             print(f"{file} - {final_files[file].reason}")
 
-    source_code = ""
+    source_code = ""  
+    source_code_list = SourceCodeList(sources=[])
     depulicated_sources = set()
 
     for file in sources:
@@ -263,7 +265,7 @@ def build_index_and_filter_files(
             depulicated_sources.add(file.module_name)
             source_code += f"##File: {file.module_name}\n"
             source_code += f"{file.source_code}\n\n"
-
+            source_code_list.sources.append(file)
     if args.request_id and not args.skip_events:
         queue_communicate.send_event(
             request_id=args.request_id,
@@ -339,4 +341,4 @@ def build_index_and_filter_files(
             )
         )
 
-    return source_code
+    return source_code_list

autocoder/index/index.py

@@ -9,6 +9,7 @@ from autocoder.index.symbols_utils import (
     SymbolType,
     symbols_info_to_str,
 )
+from autocoder.privacy.model_filter import ModelPathFilter
 from concurrent.futures import ThreadPoolExecutor, as_completed
 import threading
 
@@ -17,6 +18,7 @@ import hashlib
 
 from autocoder.common.printer import Printer
 from autocoder.common.auto_coder_lang import get_message
+from autocoder.utils.llms import get_llm_names, get_model_info
 from autocoder.index.types import (
     IndexItem,
     TargetFile,
@@ -30,6 +32,9 @@ class IndexManager:
     ):
         self.sources = sources
         self.source_dir = args.source_dir
+        # Initialize model filter for index_llm and index_filter_llm
+        self.index_model_filter = None
+        self.index_filter_model_filter = None
         self.anti_quota_limit = (
             args.index_model_anti_quota_limit or args.anti_quota_limit
         )
@@ -46,6 +51,12 @@ class IndexManager:
             self.index_filter_llm = llm
 
         self.llm = llm
+        
+        # Initialize model filters
+        if self.index_llm:
+            self.index_model_filter = ModelPathFilter.from_model_object(self.index_llm, args)
+        if self.index_filter_llm:
+            self.index_filter_model_filter = ModelPathFilter.from_model_object(self.index_filter_llm, args)
         self.args = args
         self.max_input_length = (
             args.index_model_max_input_length or args.model_max_input_length
@@ -194,6 +205,17 @@ class IndexManager:
         ext = os.path.splitext(file_path)[1].lower()
         if ext in [".md", ".html", ".txt", ".doc", ".pdf"]:
             return True
+            
+        # Check model filter restrictions
+        if self.index_model_filter and not self.index_model_filter.is_accessible(file_path):
+            self.printer.print_in_terminal(
+                "index_file_filtered",
+                style="yellow",
+                file_path=file_path,
+                model_name=",".join(get_llm_names(self.index_llm))
+            )
+            return True
+            
         return False
 
     def build_index_for_single_source(self, source: SourceCode):   
@@ -212,8 +234,29 @@ class IndexManager:
         model_name = ",".join(get_llm_names(self.index_llm))
 
         try:
+            # 获取模型名称列表
+            model_names = get_llm_names(self.index_llm)
+            model_name = ",".join(model_names)
+
+            # 获取模型价格信息
+            model_info_map = {}
+            for name in model_names:
+                info = get_model_info(name, self.args.product_mode)
+                if info:
+                    model_info_map[name] = {
+                        "input_price": info.get("input_price", 0.0),
+                        "output_price": info.get("output_price", 0.0)
+                    }
+
             start_time = time.monotonic()
             source_code = source.source_code
+            
+            # 统计token和成本
+            total_input_tokens = 0
+            total_output_tokens = 0
+            total_input_cost = 0.0
+            total_output_cost = 0.0
+            
             if len(source.source_code) > self.max_input_length:
                 self.printer.print_in_terminal(
                     "index_file_too_large",
@@ -227,15 +270,38 @@ class IndexManager:
                 )
                 symbols = []
                 for chunk in chunks:
+                    meta_holder = byzerllm.MetaHolder()
                     chunk_symbols = self.get_all_file_symbols.with_llm(
-                        self.index_llm).run(source.module_name, chunk)
+                        self.index_llm).with_meta(meta_holder).run(source.module_name, chunk)
                     time.sleep(self.anti_quota_limit)
                     symbols.append(chunk_symbols)
+                    
+                    if meta_holder.get_meta():
+                        meta_dict = meta_holder.get_meta()
+                        total_input_tokens += meta_dict.get("input_tokens_count", 0)
+                        total_output_tokens += meta_dict.get("generated_tokens_count", 0)
+                        
                 symbols = "\n".join(symbols)
             else:
+                meta_holder = byzerllm.MetaHolder()
                 symbols = self.get_all_file_symbols.with_llm(
-                    self.index_llm).run(source.module_name, source_code)
+                    self.index_llm).with_meta(meta_holder).run(source.module_name, source_code)
                 time.sleep(self.anti_quota_limit)
+                
+                if meta_holder.get_meta():
+                    meta_dict = meta_holder.get_meta()
+                    total_input_tokens += meta_dict.get("input_tokens_count", 0)
+                    total_output_tokens += meta_dict.get("generated_tokens_count", 0)
+            
+            # 计算总成本
+            for name in model_names:
+                info = model_info_map.get(name, {})
+                total_input_cost += (total_input_tokens * info.get("input_price", 0.0)) / 1000000
+                total_output_cost += (total_output_tokens * info.get("output_price", 0.0)) / 1000000
+            
+            # 四舍五入到4位小数
+            total_input_cost = round(total_input_cost, 4)
+            total_output_cost = round(total_output_cost, 4)
             
             self.printer.print_in_terminal(
                 "index_update_success",
@@ -243,7 +309,11 @@ class IndexManager:
                 file_path=file_path,
                 md5=md5,
                 duration=time.monotonic() - start_time,
-                model_name=model_name
+                model_name=model_name,
+                input_tokens=total_input_tokens,
+                output_tokens=total_output_tokens,
+                input_cost=total_input_cost,
+                output_cost=total_output_cost
             )
 
         except Exception as e:
@@ -263,6 +333,10 @@ class IndexManager:
             "symbols": symbols,
             "last_modified": os.path.getmtime(file_path),
             "md5": md5,
+            "input_tokens_count": total_input_tokens,
+            "generated_tokens_count": total_output_tokens,
+            "input_tokens_cost": total_input_cost,
+            "generated_tokens_cost": total_output_cost
         }
 
     def build_index(self):
@@ -290,6 +364,11 @@ class IndexManager:
 
         updated_sources = []
 
+        total_input_tokens = 0
+        total_output_tokens = 0
+        total_input_cost = 0.0
+        total_output_cost = 0.0
+
         with ThreadPoolExecutor(max_workers=self.args.index_build_workers) as executor:
 
             wait_to_build_files = []
@@ -346,6 +425,10 @@ class IndexManager:
                         num_files=num_files
                     )
                     module_name = result["module_name"]
+                    total_input_tokens += result["input_tokens_count"]
+                    total_output_tokens += result["generated_tokens_count"]
+                    total_input_cost += result["input_tokens_cost"]
+                    total_output_cost += result["generated_tokens_cost"]
                     index_data[module_name] = result
                     updated_sources.append(module_name)
                     if len(updated_sources) > 5:
@@ -357,12 +440,19 @@ class IndexManager:
         if updated_sources or keys_to_remove:
             with open(self.index_file, "w") as file:
                 json.dump(index_data, file, ensure_ascii=False, indent=2)
+
+            print("")    
             self.printer.print_in_terminal(
                 "index_file_saved",
                 style="green",
                 updated_files=len(updated_sources),
-                removed_files=len(keys_to_remove)
+                removed_files=len(keys_to_remove),
+                input_tokens=total_input_tokens,
+                output_tokens=total_output_tokens,
+                input_cost=total_input_cost,
+                output_cost=total_output_cost
             )
+            print("")
 
         return index_data
 

autocoder/models.py

@@ -110,6 +110,20 @@ def save_models(models: List[Dict]) -> None:
         json.dump(models, f, indent=2, ensure_ascii=False)
 
 
+def add_and_activate_models(models: List[Dict]) -> None:
+    """
+    添加模型
+    """
+    exits_models = load_models()
+    for model in models:        
+        if model["name"] not in [m["name"] for m in exits_models]:
+            exits_models.append(model)
+    save_models(exits_models)
+
+    for model in models:
+        if "api_key" in model:
+            update_model_with_api_key(model["name"], model["api_key"])
+
 def get_model_by_name(name: str) -> Dict:
     """
     根据模型名称查找模型

autocoder/privacy/__init__.py

@@ -0,0 +1,3 @@
+from .model_filter import ModelPathFilter
+
+__all__ = ["ModelPathFilter"]

autocoder/privacy/model_filter.py

@@ -0,0 +1,100 @@
+import re
+import yaml
+from pathlib import Path
+from typing import Dict, List, Optional
+from autocoder.common import AutoCoderArgs
+from autocoder.utils import llms as llm_utils
+
+
+class ModelPathFilter:
+    def __init__(self,
+                 model_name: str,
+                 args: AutoCoderArgs,
+                 default_forbidden: List[str] = None):
+        """
+        模型路径过滤器
+        :param model_name: 当前使用的模型名称
+        :param args: 自动编码器参数
+        :param default_forbidden: 默认禁止路径规则
+        """
+        self.model_name = model_name
+        if args.model_filter_path:
+            self.config_path = Path(args.model_filter_path)
+        else:
+            self.config_path = Path(args.source_dir, ".model_filters.yml")
+        self.default_forbidden = default_forbidden or []
+        self._rules_cache: Dict[str, List[re.Pattern]] = {}
+        self._load_rules()
+
+    def _load_rules(self):
+        """加载并编译正则规则"""
+        if not self.config_path.exists():            
+            return
+
+        with open(self.config_path, 'r', encoding="utf-8") as f:
+            config = yaml.safe_load(f)
+
+        model_rules = config.get('model_filters', {}).get(self.model_name, {})
+        all_rules = model_rules.get('forbidden_paths', []) + self.default_forbidden
+
+        # 预编译正则表达式
+        self._rules_cache[self.model_name] = [
+            re.compile(rule) for rule in all_rules
+        ]
+
+    def is_accessible(self, file_path: str) -> bool:
+        """
+        检查文件路径是否符合访问规则
+        :return: True表示允许访问,False表示禁止
+        """
+        # 优先使用模型专属规则
+        patterns = self._rules_cache.get(self.model_name, [])
+
+        # 回退到默认规则
+        if not patterns and self.default_forbidden:
+            patterns = [re.compile(rule) for rule in self.default_forbidden]
+
+        # 如果路径为空或None,直接返回True
+        if not file_path:
+            return True
+
+        return not any(pattern.search(file_path) for pattern in patterns)
+
+    def add_temp_rule(self, rule: str):
+        """
+        添加临时规则
+        :param rule: 正则表达式规则
+        """
+        patterns = self._rules_cache.get(self.model_name, [])
+        patterns.append(re.compile(rule))
+        self._rules_cache[self.model_name] = patterns
+
+    def reload_rules(self):
+        """重新加载规则配置"""
+        self._rules_cache.clear()
+        self._load_rules()
+
+    def has_rules(self):
+        """检查是否存在规则"""
+        return bool(self._rules_cache.get(self.model_name, []))
+
+    @classmethod
+    def from_model_object(cls,
+                         llm_obj,
+                         args: AutoCoderArgs,
+                         default_forbidden: Optional[List[str]] = None):
+        """
+        从LLM对象创建过滤器
+        :param llm_obj: ByzerLLM实例或类似对象
+        :param args: 自动编码器参数
+        :param default_forbidden: 默认禁止路径规则
+        """
+        model_name = ",".join(llm_utils.get_llm_names(llm_obj))
+        if not model_name:
+            raise ValueError(f"{model_name} is not found")
+
+        return cls(
+            model_name=model_name,
+            args=args,
+            default_forbidden=default_forbidden
+        )