auto-coder 0.1.205__py3-none-any.whl → 0.1.207__py3-none-any.whl

{auto_coder-0.1.205.dist-info → auto_coder-0.1.207.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: auto-coder
-Version: 0.1.205
+Version: 0.1.207
 Summary: AutoCoder: AutoCoder
 Author: allwefantasy
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence

{auto_coder-0.1.205.dist-info → auto_coder-0.1.207.dist-info}/RECORD

@@ -1,14 +1,16 @@
 autocoder/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 autocoder/auto_coder.py,sha256=IP4aSmuZh3HaIzLIThTKHF0PB38eTPUU1GDdfuJBam4,38971
 autocoder/auto_coder_lang.py,sha256=Rtupq6N3_HT7JRhDKdgCBcwRaiAnyCOR_Gsp4jUomrI,3229
-autocoder/auto_coder_rag.py,sha256=0JAQNHbrCiWJMFipr1ZIe9sTBNxSlHJj1T6tfeRt_Zc,17622
+autocoder/auto_coder_rag.py,sha256=UNP69PTJA_Iz0MIF_RS26o9slSUvm3f3D7yQBxGYKAY,21578
 autocoder/auto_coder_server.py,sha256=XU9b4SBH7zjPPXaTWWHV4_zJm-XYa6njuLQaplYJH_c,20290
-autocoder/chat_auto_coder.py,sha256=GuJzVOgscEpQ3xObo3RN5X6eJNUWYFBYC01TH_qYn54,86906
+autocoder/benchmark.py,sha256=Ypomkdzd1T3GE6dRICY3Hj547dZ6_inqJbBJIp5QMco,4423
+autocoder/chat_auto_coder.py,sha256=PtES0XppQ4OTuMucQZFXN5j-6rXS4Rc7RL5gmBy3GZU,87182
 autocoder/chat_auto_coder_lang.py,sha256=zU9VRY-l80fZnLJ0Op8A3wq27UhQHh9WcpSYU4SmnqU,8708
 autocoder/command_args.py,sha256=nmVD3xgLKKNt3fLJ4wVNUdJBtBOkfAdZ2ZmaRMlS7qg,29891
 autocoder/lang.py,sha256=Ajng6m7towmx-cvQfEHPFp43iEfddPvr8ju5GH4H8qA,13819
-autocoder/version.py,sha256=D55oj5ULfDLw7xtwrHsVQheA8-tdoVBOPKnQO1dgeNw,24
+autocoder/version.py,sha256=Zou96E1H6FClEDuqHKcUc9kgUuTRvVu0lQVcFzPxZRY,24
 autocoder/agent/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+autocoder/agent/auto_filegroup.py,sha256=CW7bqp0FW1GIEMnl-blyAc2UGT7O9Mom0q66ITz1ckM,6635
 autocoder/agent/auto_tool.py,sha256=DBzip-P_T6ZtT2eHexPcusmKYD0h7ufzp7TLwXAY10E,11554
 autocoder/agent/coder.py,sha256=x6bdJwDuETGg9ebQnYlUWCxCtQcDGg73LtI6McpWslQ,72034
 autocoder/agent/designer.py,sha256=EpRbzO58Xym3GrnppIT1Z8ZFAlnNfgzHbIzZ3PX-Yv8,27037
@@ -20,6 +22,7 @@ autocoder/common/ShellClient.py,sha256=fM1q8t_XMSbLBl2zkCNC2J9xuyKN3eXzGm6hHhqL2
 autocoder/common/__init__.py,sha256=szE6bhkWT_KDTxY73hDQDRYCaH7DDTrBuEQBxzqAEOM,11201
 autocoder/common/anything2images.py,sha256=0ILBbWzY02M-CiWB-vzuomb_J1hVdxRcenAfIrAXq9M,25283
 autocoder/common/audio.py,sha256=Kn9nWKQddWnUrAz0a_ZUgjcu4VUU_IcZBigT7n3N3qc,7439
+autocoder/common/chunk_validation.py,sha256=BrR_ZWavW8IANuueEE7hS8NFAwEvm8TX34WnPx_1hs8,3030
 autocoder/common/cleaner.py,sha256=NU72i8C6o9m0vXExab7nao5bstBUsfJFcj11cXa9l4U,1089
 autocoder/common/code_auto_execute.py,sha256=4KXGmiGObr_B1d6tzV9dwS6MifCSc3Gm4j2d6ildBXQ,6867
 autocoder/common/code_auto_generate.py,sha256=bREGPj2yNQ1oy35ivHRHQSXAdmIsPaW4HQ9BkO7Ytco,8099
@@ -38,12 +41,14 @@ autocoder/common/git_utils.py,sha256=btK45sxvfm4tX3fBRNUPRZoGQuZuOEQrWSAwLy1yoLw
 autocoder/common/image_to_page.py,sha256=O0cNO_vHHUP-fP4GXiVojShmNqkPnZXeIyiY1MRLpKg,13936
 autocoder/common/interpreter.py,sha256=62-dIakOunYB4yjmX8SHC0Gdy2h8NtxdgbpdqRZJ5vk,2833
 autocoder/common/llm_rerank.py,sha256=FbvtCzaR661Mt2wn0qsuiEL1Y3puD6jeIJS4zg_e7Bs,3260
+autocoder/common/recall_validation.py,sha256=Avt9Q9dX3kG6Pf2zsdlOHmsjd-OeSj7U1PFBDp_Cve0,1700
 autocoder/common/screenshots.py,sha256=_gA-z1HxGjPShBrtgkdideq58MG6rqFB2qMUJKjrycs,3769
 autocoder/common/search.py,sha256=_ZX03ph89rDPGMY1OrfqaDfxsDR-flh6YEHixherjwM,16616
 autocoder/common/search_replace.py,sha256=GphFkc57Hb673CAwmbiocqTbw8vrV7TrZxtOhD0332g,22147
 autocoder/common/sys_prompt.py,sha256=JlexfjZt554faqbgkCmzOJqYUzDHfbnxly5ugFfHfEE,26403
 autocoder/common/text.py,sha256=KGRQq314GHBmY4MWG8ossRoQi1_DTotvhxchpn78c-k,1003
 autocoder/common/types.py,sha256=_kl8q2oCAZe-HqU9LjQgSqVHAw4ojMojlF58-bp2xiE,354
+autocoder/data/tokenizer.json,sha256=QfO_ZCE9qMAS2L0IcaWKH99wRj6PCPEQ3bsQgvUp9mk,4607451
 autocoder/db/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 autocoder/db/store.py,sha256=tFT66bP2ZKIqZip-uhLkHRSLaaOAUUDZfozJwcqix3c,1908
 autocoder/dispacher/__init__.py,sha256=YoA64dIxnx4jcE1pwSfg81sjkQtjDkhddkfac1-cMWo,1230
@@ -71,7 +76,7 @@ autocoder/rag/relevant_utils.py,sha256=OGfp98OXG4jr3jNmtHIeXGPF8mOlIbTnolPIVTZzY
 autocoder/rag/simple_directory_reader.py,sha256=LkKreCkNdEOoL4fNhc3_hDoyyWTQUte4uqextISRz4U,24485
 autocoder/rag/simple_rag.py,sha256=I902EUqOK1WM0Y2WFd7RzDJYofElvTZNLVCBtX5A9rc,14885
 autocoder/rag/token_checker.py,sha256=jc76x6KWmvVxds6W8juZfQGaoErudc2HenG3sNQfSLs,2819
-autocoder/rag/token_counter.py,sha256=LReZEYXmWriDI3KYjUvK0E4Gn4MeDJX6RPCfJCmluGY,2110
+autocoder/rag/token_counter.py,sha256=C-Lwc4oIjJpZDEqp9WLHGOe6hb4yhrdJpMtkrtp_1qc,2125
 autocoder/rag/token_limiter.py,sha256=dtIxCtHswZ2ut-XKbx8_SiWyv-xqnR1WAIcmh6f8Ktw,11137
 autocoder/rag/types.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 autocoder/rag/utils.py,sha256=MOMy0l2_YQ8foWnQQSvYmYAwghAWJ8_wVrdZh5DSaTg,4822
@@ -100,15 +105,16 @@ autocoder/utils/conversation_store.py,sha256=sz-hhY7sttPAUOAQU6Pze-5zJc3j0_Emj22
 autocoder/utils/llm_client_interceptors.py,sha256=FEHNXoFZlCjAHQcjPRyX8FOMjo6rPXpO2AJ2zn2KTTo,901
 autocoder/utils/log_capture.py,sha256=I-bsJFLWoGUiX-GKoZsH9kWJCKSV7ZlUnRt7jh-fOL0,1548
 autocoder/utils/multi_turn.py,sha256=unK9OpqVRbK6uIcTKXgggX2wNmyj7s5eyEAQ2xUwHoM,88
+autocoder/utils/operate_config_api.py,sha256=99YAKsuUFLPwrRvj0CJal_bAPgyiXWMma6ZKMU56thw,5790
 autocoder/utils/print_table.py,sha256=ZMRhCA9DD0FUfKyJBWd5bDdj1RrtPtgOMWSJwtvZcLs,403
 autocoder/utils/queue_communicate.py,sha256=buyEzdvab1QA4i2QKbq35rG5v_9x9PWVLWWMTznWcYM,6832
 autocoder/utils/request_event_queue.py,sha256=r3lo5qGsB1dIjzVQ05dnr0z_9Z3zOkBdP1vmRciKdi4,2095
 autocoder/utils/request_queue.py,sha256=nwp6PMtgTCiuwJI24p8OLNZjUiprC-TsefQrhMI-yPE,3889
 autocoder/utils/rest.py,sha256=HawagAap3wMIDROGhY1730zSZrJR_EycODAA5qOj83c,8807
 autocoder/utils/tests.py,sha256=BqphrwyycGAvs-5mhH8pKtMZdObwhFtJ5MC_ZAOiLq8,1340
-auto_coder-0.1.205.dist-info/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
-auto_coder-0.1.205.dist-info/METADATA,sha256=ZD3gLn49tV98cAyXzJB7r4NSoD-tx3sDx66039FbNUY,2575
-auto_coder-0.1.205.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
-auto_coder-0.1.205.dist-info/entry_points.txt,sha256=0nzHtHH4pNcM7xq4EBA2toS28Qelrvcbrr59GqD_0Ak,350
-auto_coder-0.1.205.dist-info/top_level.txt,sha256=Jqc0_uJSw2GwoFQAa9iJxYns-2mWla-9ok_Y3Gcznjk,10
-auto_coder-0.1.205.dist-info/RECORD,,
+auto_coder-0.1.207.dist-info/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
+auto_coder-0.1.207.dist-info/METADATA,sha256=CIJPaXnDWOPKCFvDnjZHWQEmftK9Jde99NTbFdxANng,2575
+auto_coder-0.1.207.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
+auto_coder-0.1.207.dist-info/entry_points.txt,sha256=0nzHtHH4pNcM7xq4EBA2toS28Qelrvcbrr59GqD_0Ak,350
+auto_coder-0.1.207.dist-info/top_level.txt,sha256=Jqc0_uJSw2GwoFQAa9iJxYns-2mWla-9ok_Y3Gcznjk,10
+auto_coder-0.1.207.dist-info/RECORD,,

autocoder/agent/auto_filegroup.py

@@ -0,0 +1,202 @@
+from typing import List, Dict, Optional, Any, Tuple
+import os
+import yaml
+from loguru import logger
+import byzerllm
+import pydantic
+
+
+class FileGroup(pydantic.BaseModel):
+    name: str
+    description: str
+    queries: List[str]
+    urls: List[str]    
+
+
+class FileGroups(pydantic.BaseModel):
+    groups: List[FileGroup]
+
+
+def load_yaml_config(yaml_file: str) -> Dict:
+    """加载YAML配置文件"""
+    try:
+        with open(yaml_file, 'r', encoding='utf-8') as f:
+            return yaml.safe_load(f)
+    except Exception as e:
+        logger.error(f"Error loading yaml file {yaml_file}: {str(e)}")
+        return {}
+
+
+class AutoFileGroup:
+    def __init__(self, llm: byzerllm.ByzerLLM, 
+                 project_dir: str, 
+                 skip_diff: bool = False,
+                 group_num_limit: int = 10,
+                 file_size_limit: int = 100):
+        """
+        初始化AutoFileGroup
+
+        Args:
+            actions_dir: 包含YAML文件的目录
+        """
+        self.project_dir = project_dir
+        self.actions_dir = os.path.join(project_dir, "actions")
+        self.llm = llm
+        self.file_size_limit = file_size_limit
+        self.skip_diff = skip_diff
+        self.group_num_limit = group_num_limit
+
+    @byzerllm.prompt()
+    def group_by_similarity(self, querie_with_urls: List[Tuple[str, List[str], str]]) -> str:
+        """
+        分析多个开发任务的关联性，将相互关联的任务进行分组。
+
+        输入说明：
+        querie_with_urls 包含多个开发任务信息，每个任务由以下部分组成：
+        1. query: 任务需求描述
+        2. urls: 需要修改的文件路径列表
+        3. diff: Git diff信息，展示具体的代码修改
+
+        示例数据：
+        <queries>
+        {% for query,urls,diff in querie_with_urls %}
+        ## {{ query }}        
+
+        修改的文件:
+        {% for url in urls %}
+        - {{ url }}
+        {% endfor %}
+        {% if diff %}
+
+        代码变更:
+        ```diff
+        {{ diff }}
+        ```
+        {% endif %}        
+        {% endfor %}
+        </queries>
+
+        分组规则：
+        1. 每个分组至少包含2个query
+        2. 根据以下维度判断任务的关联性：
+           - 功能相似性：任务是否属于同一个功能模块
+           - 文件关联：修改的文件是否有重叠或紧密关联
+           - 代码依赖：代码修改是否存在依赖关系
+           - 业务目的：任务的最终业务目标是否一致
+        3. 输出的分组数量最多不超过 {{ group_num_limit }}
+
+        期望输出：
+        返回符合以下格式的JSON:
+        {
+          "groups": [
+            {
+              "name": "分组名称",
+              "description": "分组的功能概述，描述该组任务的共同目标",
+              "queries": ["相关的query1", "相关的query2"],
+              "urls": ["相关的文件1", "相关的文件2"]
+            }
+          ]
+        }
+
+        特别说明：
+        1. 分组名称应该简洁且具有描述性，能反映该组任务的主要特征
+        2. 分组描述应突出任务间的共同点和关联性
+        3. 返回的urls应该是该组任务涉及的所有相关文件的并集
+        """
+        return {
+            "group_num_limit": self.group_num_limit
+        }
+
+
+    def group_files(self) -> List[Dict]:
+        """
+        根据YAML文件中的query和urls进行文件分组，并获取相关的git commit信息
+
+        Returns:
+            List[Dict]: 分组结果列表
+        """
+        import git
+        import hashlib
+
+        # 获取所有YAML文件
+        action_files = [
+            f for f in os.listdir(self.actions_dir)
+            if f[:3].isdigit() and "_" in f and f.endswith('.yml')
+        ]
+
+        # 按序号排序
+        def get_seq(name):
+            return int(name.split("_")[0])
+        
+        # 获取最新的action文件列表
+        action_files = sorted(action_files, key=get_seq)
+        action_files.reverse()
+
+        action_files = action_files[:self.file_size_limit]
+
+        querie_with_urls_and_diffs = []
+        repo = git.Repo(self.project_dir)
+
+        # 收集所有query、urls和对应的commit diff
+        for yaml_file in action_files:
+            yaml_path = os.path.join(self.actions_dir, yaml_file)
+            config = load_yaml_config(yaml_path)
+
+            if not config:
+                continue
+
+            query = config.get('query', '')
+            urls = config.get('urls', [])
+
+            if query and urls:  
+                commit_diff = ""
+                if not self.skip_diff:
+                    # 计算文件的MD5用于匹配commit
+                    file_md5 = hashlib.md5(open(yaml_path, 'rb').read()).hexdigest()
+                    response_id = f"auto_coder_{yaml_file}_{file_md5}"
+                    # 查找对应的commit                   
+                    try:
+                        for commit in repo.iter_commits():
+                            if response_id in commit.message:
+                                if commit.parents:
+                                    parent = commit.parents[0]
+                                    commit_diff = repo.git.diff(
+                                        parent.hexsha, commit.hexsha)
+                                else:
+                                    commit_diff = repo.git.show(commit.hexsha)
+                                break
+                    except git.exc.GitCommandError as e:
+                        logger.error(f"Git命令执行错误: {str(e)}")
+                    except Exception as e:
+                        logger.error(f"获取commit diff时出错: {str(e)}")
+
+                querie_with_urls_and_diffs.append((query, urls, commit_diff))
+
+        if not querie_with_urls_and_diffs:
+            return []
+
+        # 使用LLM进行分组
+        try:            
+            result = self.group_by_similarity.with_llm(self.llm).with_return_type(FileGroups).run(
+                querie_with_urls=querie_with_urls_and_diffs
+            )
+            return result.groups
+        except Exception as e:
+            import traceback
+            traceback.print_exc()
+            logger.error(f"Error during grouping: {str(e)}")
+            return []
+
+
+def create_file_groups(actions_dir: str) -> List[Dict]:
+    """
+    创建文件分组的便捷函数
+
+    Args:
+        actions_dir: YAML文件所在目录
+
+    Returns:
+        List[Dict]: 分组结果，每个字典包含name, queries和urls
+    """
+    grouper = AutoFileGroup(actions_dir)
+    return grouper.group_files()

autocoder/auto_coder_rag.py

@@ -18,8 +18,10 @@ from rich.console import Console
 from rich.table import Table
 import os
 from loguru import logger
+import asyncio
 
 from autocoder.rag.document_retriever import process_file_local
+import pkg_resources
 from autocoder.rag.token_counter import TokenCounter
 
 if platform.system() == "Windows":
@@ -139,6 +141,13 @@ def initialize_system():
 
 def main(input_args: Optional[List[str]] = None):
 
+    try:
+        tokenizer_path = pkg_resources.resource_filename(
+            "autocoder", "data/tokenizer.json"
+        )
+    except FileNotFoundError:
+        tokenizer_path = None
+
     system_lang, _ = locale.getdefaultlocale()
     lang = "zh" if system_lang and system_lang.startswith("zh") else "en"
     desc = lang_desc[lang]
@@ -146,18 +155,38 @@ def main(input_args: Optional[List[str]] = None):
     subparsers = parser.add_subparsers(dest="command", help="Available commands")
 
     # Build hybrid index command
-    build_index_parser = subparsers.add_parser("build_hybrid_index", help="Build hybrid index for RAG")
-    build_index_parser.add_argument("--quick", action="store_true", help="Skip system initialization")
+    build_index_parser = subparsers.add_parser(
+        "build_hybrid_index", help="Build hybrid index for RAG"
+    )
+    build_index_parser.add_argument(
+        "--quick", action="store_true", help="Skip system initialization"
+    )
     build_index_parser.add_argument("--file", default="", help=desc["file"])
-    build_index_parser.add_argument("--model", default="deepseek_chat", help=desc["model"])
-    build_index_parser.add_argument("--index_model", default="", help=desc["index_model"])
+    build_index_parser.add_argument(
+        "--model", default="deepseek_chat", help=desc["model"]
+    )
+    build_index_parser.add_argument(
+        "--index_model", default="", help=desc["index_model"]
+    )
     build_index_parser.add_argument("--emb_model", default="", help=desc["emb_model"])
-    build_index_parser.add_argument("--ray_address", default="auto", help=desc["ray_address"])
-    build_index_parser.add_argument("--required_exts", default="", help=desc["doc_build_parse_required_exts"])
-    build_index_parser.add_argument("--source_dir", default=".", help="Source directory path")
-    build_index_parser.add_argument("--tokenizer_path", default="", help="Path to tokenizer file")
-    build_index_parser.add_argument("--doc_dir", default="", help="Document directory path")
-    build_index_parser.add_argument("--enable_hybrid_index", action="store_true", help="Enable hybrid index")
+    build_index_parser.add_argument(
+        "--ray_address", default="auto", help=desc["ray_address"]
+    )
+    build_index_parser.add_argument(
+        "--required_exts", default="", help=desc["doc_build_parse_required_exts"]
+    )
+    build_index_parser.add_argument(
+        "--source_dir", default=".", help="Source directory path"
+    )
+    build_index_parser.add_argument(
+        "--tokenizer_path", default=tokenizer_path, help="Path to tokenizer file"
+    )
+    build_index_parser.add_argument(
+        "--doc_dir", default="", help="Document directory path"
+    )
+    build_index_parser.add_argument(
+        "--enable_hybrid_index", action="store_true", help="Enable hybrid index"
+    )
 
     # Serve command
     serve_parser = subparsers.add_parser("serve", help="Start the RAG server")
@@ -220,7 +249,7 @@ def main(input_args: Optional[List[str]] = None):
     serve_parser.add_argument("--ssl_certfile", default="", help="")
     serve_parser.add_argument("--response_role", default="assistant", help="")
     serve_parser.add_argument("--doc_dir", default="", help="")
-    serve_parser.add_argument("--tokenizer_path", default="", help="")
+    serve_parser.add_argument("--tokenizer_path", default=tokenizer_path, help="")
     serve_parser.add_argument(
         "--collections", default="", help="Collection name for indexing"
     )
@@ -282,7 +311,7 @@ def main(input_args: Optional[List[str]] = None):
 
     serve_parser.add_argument(
         "--without_contexts",
-        action="store_true", 
+        action="store_true",
         help="Whether to return responses without contexts. only works when pro plugin is installed",
     )
 
@@ -304,14 +333,73 @@ def main(input_args: Optional[List[str]] = None):
         help="The model used for question answering",
     )
 
+    # Benchmark command
+    benchmark_parser = subparsers.add_parser(
+        "benchmark", help="Benchmark LLM client performance"
+    )
+    benchmark_parser.add_argument(
+        "--model", default="deepseek_chat", help="Model to benchmark"
+    )
+    benchmark_parser.add_argument(
+        "--parallel", type=int, default=10, help="Number of parallel requests"
+    )
+    benchmark_parser.add_argument(
+        "--rounds", type=int, default=1, help="Number of rounds to run"
+    )
+    benchmark_parser.add_argument(
+        "--type",
+        choices=["openai", "byzerllm"],
+        default="byzerllm",
+        help="Client type to benchmark",
+    )
+    benchmark_parser.add_argument(
+        "--api_key", default="", help="OpenAI API key for OpenAI client"
+    )
+    benchmark_parser.add_argument(
+        "--base_url", default="", help="Base URL for OpenAI client"
+    )
+    benchmark_parser.add_argument(
+        "--query", default="Hello, how are you?", help="Query to use for benchmarking"
+    )
+
     # Tools command
     tools_parser = subparsers.add_parser("tools", help="Various tools")
     tools_subparsers = tools_parser.add_subparsers(dest="tool", help="Available tools")
 
     # Count tool
     count_parser = tools_subparsers.add_parser("count", help="Count tokens in a file")
+
+    # Recall validation tool
+    recall_parser = tools_subparsers.add_parser(
+        "recall", help="Validate recall model performance"
+    )
+    recall_parser.add_argument(
+        "--model", required=True, help="Model to use for recall validation"
+    )
+    recall_parser.add_argument(
+        "--content", default=None, help="Content to validate against"
+    )
+    recall_parser.add_argument(
+        "--query", default=None, help="Query to use for validation"
+    )
+
+    # Add chunk model validation tool
+    chunk_parser = tools_subparsers.add_parser(
+        "chunk", help="Validate chunk model performance"
+    )
+    chunk_parser.add_argument(
+        "--model", required=True, help="Model to use for chunk validation"
+    )
+    chunk_parser.add_argument(
+        "--content", default=None, help="Content to validate against"
+    )
+    chunk_parser.add_argument(
+        "--query", default=None, help="Query to use for validation"
+    )
     count_parser.add_argument(
-        "--tokenizer_path", required=True, help="Path to the tokenizer"
+        "--tokenizer_path",
+        default=tokenizer_path,        
+        help="Path to the tokenizer",
     )
     count_parser.add_argument(
         "--file", required=True, help="Path to the file to count tokens"
@@ -319,7 +407,22 @@ def main(input_args: Optional[List[str]] = None):
 
     args = parser.parse_args(input_args)
 
-    if args.command == "serve":
+    if args.command == "benchmark":
+        from .benchmark import benchmark_openai, benchmark_byzerllm
+
+        if args.type == "openai":
+            if not args.api_key:
+                print("OpenAI API key is required for OpenAI client benchmark")
+                return
+            asyncio.run(
+                benchmark_openai(
+                    args.model, args.parallel, args.api_key, args.base_url, args.rounds, args.query
+                )
+            )
+        else:  # byzerllm
+            benchmark_byzerllm(args.model, args.parallel, args.rounds, args.query)
+
+    elif args.command == "serve":
         if not args.quick:
             initialize_system()
         server_args = ServerArgs(
@@ -337,14 +440,17 @@ def main(input_args: Optional[List[str]] = None):
             }
         )
 
-        if auto_coder_args.enable_hybrid_index:     
-            # 尝试连接storage                
+        if auto_coder_args.enable_hybrid_index:
+            # 尝试连接storage
             try:
                 from byzerllm.apps.byzer_storage.simple_api import ByzerStorage
+
                 storage = ByzerStorage("byzerai_store", "rag", "files")
                 storage.retrieval.cluster_info("byzerai_store")
             except Exception as e:
-                logger.error("When enable_hybrid_index is true, ByzerStorage must be started")
+                logger.error(
+                    "When enable_hybrid_index is true, ByzerStorage must be started"
+                )
                 logger.error("Please run 'byzerllm storage start' first")
                 return
         else:
@@ -369,12 +475,14 @@ def main(input_args: Optional[List[str]] = None):
             llm.setup_sub_client("qa_model", qa_model)
 
         # 当启用hybrid_index时,检查必要的组件
-        if auto_coder_args.enable_hybrid_index:                           
+        if auto_coder_args.enable_hybrid_index:
             if not llm.is_model_exist("emb"):
-                logger.error("When enable_hybrid_index is true, an 'emb' model must be deployed")
+                logger.error(
+                    "When enable_hybrid_index is true, an 'emb' model must be deployed"
+                )
                 return
             llm.setup_default_emb_model_name("emb")
-                
+
         if server_args.doc_dir:
             auto_coder_args.rag_type = "simple"
             rag = RAGFactory.get_rag(
@@ -391,7 +499,7 @@ def main(input_args: Optional[List[str]] = None):
     elif args.command == "build_hybrid_index":
         if not args.quick:
             initialize_system()
-            
+
         auto_coder_args = AutoCoderArgs(
             **{
                 arg: getattr(args, arg)
@@ -402,25 +510,30 @@ def main(input_args: Optional[List[str]] = None):
 
         auto_coder_args.enable_hybrid_index = True
         auto_coder_args.rag_type = "simple"
-                        
+
         try:
             from byzerllm.apps.byzer_storage.simple_api import ByzerStorage
+
             storage = ByzerStorage("byzerai_store", "rag", "files")
             storage.retrieval.cluster_info("byzerai_store")
         except Exception as e:
-            logger.error("When enable_hybrid_index is true, ByzerStorage must be started")
+            logger.error(
+                "When enable_hybrid_index is true, ByzerStorage must be started"
+            )
             logger.error("Please run 'byzerllm storage start' first")
             return
-        
+
         llm = byzerllm.ByzerLLM()
         llm.setup_default_model_name(args.model)
 
         # 当启用hybrid_index时,检查必要的组件
-        if auto_coder_args.enable_hybrid_index:                           
+        if auto_coder_args.enable_hybrid_index:
             if not llm.is_model_exist("emb"):
-                logger.error("When enable_hybrid_index is true, an 'emb' model must be deployed")
+                logger.error(
+                    "When enable_hybrid_index is true, an 'emb' model must be deployed"
+                )
                 return
-            llm.setup_default_emb_model_name("emb")        
+            llm.setup_default_emb_model_name("emb")
 
         rag = RAGFactory.get_rag(
             llm=llm,
@@ -428,19 +541,41 @@ def main(input_args: Optional[List[str]] = None):
             path=args.doc_dir,
             tokenizer_path=args.tokenizer_path,
         )
-        
+
         if hasattr(rag.document_retriever, "cacher"):
             rag.document_retriever.cacher.build_cache()
         else:
-            logger.error("The document retriever does not support hybrid index building")
+            logger.error(
+                "The document retriever does not support hybrid index building"
+            )
+
+    elif args.command == "tools":
+        if args.tool == "count":
+            # auto-coder.rag tools count --tokenizer_path /Users/allwefantasy/Downloads/tokenizer.json --file /Users/allwefantasy/data/yum/schema/schema.xlsx
+            count_tokens(args.tokenizer_path, args.file)
+        elif args.tool == "recall":
+            from .common.recall_validation import validate_recall
+
+            llm = byzerllm.ByzerLLM.from_default_model(args.model)
+
+            content = None if not args.content else [args.content]
+            result = validate_recall(llm, content=content, query=args.query)
+            print(f"Recall Validation Result:\n{result}")
+        elif args.tool == "chunk":
+            from .common.chunk_validation import validate_chunk
 
-    elif args.command == "tools" and args.tool == "count":
-        # auto-coder.rag tools count --tokenizer_path /Users/allwefantasy/Downloads/tokenizer.json --file /Users/allwefantasy/data/yum/schema/schema.xlsx
-        count_tokens(args.tokenizer_path, args.file)
+            llm = byzerllm.ByzerLLM.from_default_model(args.model)
+            content = None if not args.content else [args.content]
+            result = validate_chunk(llm, content=content, query=args.query)
+            print(f"Chunk Model Validation Result:\n{result}")
 
 
 def count_tokens(tokenizer_path: str, file_path: str):
-    token_counter = TokenCounter(tokenizer_path)
+    from autocoder.rag.variable_holder import VariableHolder
+    from tokenizers import Tokenizer
+    VariableHolder.TOKENIZER_PATH = tokenizer_path
+    VariableHolder.TOKENIZER_MODEL = Tokenizer.from_file(tokenizer_path)
+    token_counter = TokenCounter(tokenizer_path)    
     source_codes = process_file_local(file_path)
 
     console = Console()