autocoder-nano 0.1.25__py3-none-any.whl → 0.1.27__py3-none-any.whl

autocoder_nano/index/entry.py

@@ -0,0 +1,166 @@
+import os
+import time
+from typing import List, Dict
+
+from loguru import logger
+from rich.console import Console
+from rich.table import Table
+
+from autocoder_nano.index.index_manager import IndexManager
+from autocoder_nano.llm_types import SourceCode, TargetFile, VerifyFileRelevance, AutoCoderArgs
+from autocoder_nano.llm_client import AutoLLM
+
+console = Console()
+
+
+def build_index_and_filter_files(args: AutoCoderArgs, llm: AutoLLM, sources: List[SourceCode]) -> str:
+    def get_file_path(_file_path):
+        if _file_path.startswith("##"):
+            return _file_path.strip()[2:]
+        return _file_path
+
+    final_files: Dict[str, TargetFile] = {}
+    logger.info("第一阶段：处理 REST/RAG/Search 资源...")
+    for source in sources:
+        if source.tag in ["REST", "RAG", "SEARCH"]:
+            final_files[get_file_path(source.module_name)] = TargetFile(
+                file_path=source.module_name, reason="Rest/Rag/Search"
+            )
+
+    if not args.skip_build_index and llm:
+        logger.info("第二阶段：为所有文件构建索引...")
+        index_manager = IndexManager(args=args, llm=llm, source_codes=sources)
+        index_data = index_manager.build_index()
+        indexed_files_count = len(index_data) if index_data else 0
+        logger.info(f"总索引文件数: {indexed_files_count}")
+
+        if not args.skip_filter_index and args.index_filter_level >= 1:
+            logger.info("第三阶段：执行 Level 1 过滤(基于查询) ...")
+            target_files = index_manager.get_target_files_by_query(args.query)
+            if target_files:
+                for file in target_files.file_list:
+                    file_path = file.file_path.strip()
+                    final_files[get_file_path(file_path)] = file
+
+            if target_files is not None and args.index_filter_level >= 2:
+                logger.info("第四阶段：执行 Level 2 过滤（基于相关文件）...")
+                related_files = index_manager.get_related_files(
+                    [file.file_path for file in target_files.file_list]
+                )
+                if related_files is not None:
+                    for file in related_files.file_list:
+                        file_path = file.file_path.strip()
+                        final_files[get_file_path(file_path)] = file
+
+            # 如果 Level 1 filtering 和 Level 2 filtering 都未获取路径，则使用全部文件
+            if not final_files:
+                logger.warning("Level 1, Level 2 过滤未找到相关文件, 将使用所有文件 ...")
+                for source in sources:
+                    final_files[get_file_path(source.module_name)] = TargetFile(
+                        file_path=source.module_name,
+                        reason="No related files found, use all files",
+                    )
+
+            logger.info("第五阶段：执行相关性验证 ...")
+            verified_files = {}
+            temp_files = list(final_files.values())
+            verification_results = []
+
+            def _print_verification_results(results):
+                table = Table(title="文件相关性验证结果", expand=True, show_lines=True)
+                table.add_column("文件路径", style="cyan", no_wrap=True)
+                table.add_column("得分", justify="right", style="green")
+                table.add_column("状态", style="yellow")
+                table.add_column("原因/错误")
+                if result:
+                    for _file_path, _score, _status, _reason in results:
+                        table.add_row(_file_path,
+                                      str(_score) if _score is not None else "N/A", _status, _reason)
+                console.print(table)
+
+            def _verify_single_file(single_file: TargetFile):
+                for _source in sources:
+                    if _source.module_name == single_file.file_path:
+                        file_content = _source.source_code
+                        try:
+                            _result = index_manager.verify_file_relevance.with_llm(llm).with_return_type(
+                                VerifyFileRelevance).run(
+                                file_content=file_content,
+                                query=args.query
+                            )
+                            if _result.relevant_score >= args.verify_file_relevance_score:
+                                verified_files[single_file.file_path] = TargetFile(
+                                    file_path=single_file.file_path,
+                                    reason=f"Score:{_result.relevant_score}, {_result.reason}"
+                                )
+                                return single_file.file_path, _result.relevant_score, "PASS", _result.reason
+                            else:
+                                return single_file.file_path, _result.relevant_score, "FAIL", _result.reason
+                        except Exception as e:
+                            error_msg = str(e)
+                            verified_files[single_file.file_path] = TargetFile(
+                                file_path=single_file.file_path,
+                                reason=f"Verification failed: {error_msg}"
+                            )
+                            return single_file.file_path, None, "ERROR", error_msg
+                return
+
+            for pending_verify_file in temp_files:
+                result = _verify_single_file(pending_verify_file)
+                if result:
+                    verification_results.append(result)
+                time.sleep(args.anti_quota_limit)
+
+            _print_verification_results(verification_results)
+            # Keep all files, not just verified ones
+            final_files = verified_files
+
+    logger.info("第六阶段：筛选文件并应用限制条件 ...")
+    if args.index_filter_file_num > 0:
+        logger.info(f"从 {len(final_files)} 个文件中获取前 {args.index_filter_file_num} 个文件(Limit)")
+    final_filenames = [file.file_path for file in final_files.values()]
+    if not final_filenames:
+        logger.warning("未找到目标文件，你可能需要重新编写查询并重试.")
+    if args.index_filter_file_num > 0:
+        final_filenames = final_filenames[: args.index_filter_file_num]
+
+    def _shorten_path(path: str, keep_levels: int = 3) -> str:
+        """
+        优化长路径显示，保留最后指定层级
+        示例：/a/b/c/d/e/f.py -> .../c/d/e/f.py
+        """
+        parts = path.split(os.sep)
+        if len(parts) > keep_levels:
+            return ".../" + os.sep.join(parts[-keep_levels:])
+        return path
+
+    def _print_selected(data):
+        table = Table(title="代码上下文文件", expand=True, show_lines=True)
+        table.add_column("文件路径", style="cyan")
+        table.add_column("原因", style="cyan")
+        for _file, _reason in data:
+            # 路径截取优化：保留最后 3 级路径
+            _processed_path = _shorten_path(_file, keep_levels=3)
+            table.add_row(_processed_path, _reason)
+        console.print(table)
+
+    logger.info("第七阶段：准备最终输出 ...")
+    _print_selected(
+        [
+            (file.file_path, file.reason)
+            for file in final_files.values()
+            if file.file_path in final_filenames
+        ]
+    )
+    result_source_code = ""
+    depulicated_sources = set()
+
+    for file in sources:
+        if file.module_name in final_filenames:
+            if file.module_name in depulicated_sources:
+                continue
+            depulicated_sources.add(file.module_name)
+            result_source_code += f"##File: {file.module_name}\n"
+            result_source_code += f"{file.source_code}\n\n"
+
+    return result_source_code

autocoder_nano/index/index_manager.py

@@ -0,0 +1,410 @@
+import hashlib
+import json
+import os
+import time
+from typing import List, Optional
+
+from loguru import logger
+
+from autocoder_nano.index.symbols_utils import extract_symbols, symbols_info_to_str
+from autocoder_nano.llm_client import AutoLLM
+from autocoder_nano.llm_prompt import prompt
+from autocoder_nano.llm_types import SourceCode, AutoCoderArgs, IndexItem, SymbolType, FileList
+
+
+class IndexManager:
+    def __init__(self, args: AutoCoderArgs, source_codes: List[SourceCode], llm: AutoLLM = None):
+        self.args = args
+        self.sources = source_codes
+        self.source_dir = args.source_dir
+        self.index_dir = os.path.join(self.source_dir, ".auto-coder")
+        self.index_file = os.path.join(self.index_dir, "index.json")
+        self.llm = llm
+        self.llm.setup_default_model_name(args.chat_model)
+        self.max_input_length = args.model_max_input_length  # 模型输入最大长度
+        # 使用 time.sleep(self.anti_quota_limit) 防止超过 API 频率限制
+        self.anti_quota_limit = args.anti_quota_limit
+        # 如果索引目录不存在,则创建它
+        if not os.path.exists(self.index_dir):
+            os.makedirs(self.index_dir)
+
+    def build_index(self):
+        """ 构建或更新索引，使用多线程处理多个文件，并将更新后的索引数据写入文件 """
+        if os.path.exists(self.index_file):
+            with open(self.index_file, "r") as file:  # 读缓存
+                index_data = json.load(file)
+        else:  # 首次 build index
+            logger.info("首次生成索引.")
+            index_data = {}
+
+        @prompt()
+        def error_message(source_dir: str, file_path: str):
+            """
+            The source_dir is different from the path in index file (e.g. file_path:{{ file_path }} source_dir:{{
+            source_dir }}). You may need to replace the prefix with the source_dir in the index file or Just delete
+            the index file to rebuild it.
+            """
+
+        for item in index_data.keys():
+            if not item.startswith(self.source_dir):
+                logger.warning(error_message(source_dir=self.source_dir, file_path=item))
+                break
+
+        updated_sources = []
+        wait_to_build_files = []
+        for source in self.sources:
+            source_code = source.source_code
+            md5 = hashlib.md5(source_code.encode("utf-8")).hexdigest()
+            if source.module_name not in index_data or index_data[source.module_name]["md5"] != md5:
+                wait_to_build_files.append(source)
+        counter = 0
+        num_files = len(wait_to_build_files)
+        total_files = len(self.sources)
+        logger.info(f"总文件数: {total_files}, 需要索引文件数: {num_files}")
+
+        for source in wait_to_build_files:
+            build_result = self.build_index_for_single_source(source)
+            if build_result is not None:
+                counter += 1
+                logger.info(f"正在构建索引:{counter}/{num_files}...")
+                module_name = build_result["module_name"]
+                index_data[module_name] = build_result
+                updated_sources.append(module_name)
+        if updated_sources:
+            with open(self.index_file, "w") as fp:
+                json_str = json.dumps(index_data, indent=2, ensure_ascii=False)
+                fp.write(json_str)
+        return index_data
+
+    def split_text_into_chunks(self, text):
+        """ 文本分块,将大文本分割成适合 LLM 处理的小块 """
+        lines = text.split("\n")
+        chunks = []
+        current_chunk = []
+        current_length = 0
+        for line in lines:
+            if current_length + len(line) + 1 <= self.max_input_length:
+                current_chunk.append(line)
+                current_length += len(line) + 1
+            else:
+                chunks.append("\n".join(current_chunk))
+                current_chunk = [line]
+                current_length = len(line) + 1
+        if current_chunk:
+            chunks.append("\n".join(current_chunk))
+        return chunks
+
+    @prompt()
+    def get_all_file_symbols(self, path: str, code: str) -> str:
+        """
+        你的目标是从给定的代码中获取代码里的符号，需要获取的符号类型包括：
+
+        1. 函数
+        2. 类
+        3. 变量
+        4. 所有导入语句
+
+        如果没有任何符号,返回空字符串就行。
+        如果有符号，按如下格式返回:
+
+        ```
+        {符号类型}: {符号名称}, {符号名称}, ...
+        ```
+
+        注意：
+        1. 直接输出结果，不要尝试使用任何代码
+        2. 不要分析代码的内容和目的
+        3. 用途的长度不能超过100字符
+        4. 导入语句的分隔符为^^
+
+        下面是一段示例：
+
+        ## 输入
+        下列是文件 /test.py 的源码：
+
+        import os
+        import time
+        from loguru import logger
+        import byzerllm
+
+        a = ""
+
+        @byzerllm.prompt(render="jinja")
+        def auto_implement_function_template(instruction:str, content:str)->str:
+
+        ## 输出
+        用途：主要用于提供自动实现函数模板的功能。
+        函数：auto_implement_function_template
+        变量：a
+        类：
+        导入语句：import os^^import time^^from loguru import logger^^import byzerllm
+
+        现在，让我们开始一个新的任务:
+
+        ## 输入
+        下列是文件 {{ path }} 的源码：
+
+        {{ code }}
+
+        ## 输出
+        """
+
+    def build_index_for_single_source(self, source: SourceCode):
+        """ 处理单个源文件，提取符号信息并存储元数据 """
+        file_path = source.module_name
+        if not os.path.exists(file_path):  # 过滤不存在的文件
+            return None
+
+        ext = os.path.splitext(file_path)[1].lower()
+        if ext in [".md", ".html", ".txt", ".doc", ".pdf"]:  # 过滤文档文件
+            return None
+
+        if source.source_code.strip() == "":
+            return None
+
+        md5 = hashlib.md5(source.source_code.encode("utf-8")).hexdigest()
+
+        try:
+            start_time = time.monotonic()
+            source_code = source.source_code
+            if len(source.source_code) > self.max_input_length:
+                logger.warning(
+                    f"警告[构建索引]: 源代码({source.module_name})长度过长 "
+                    f"({len(source.source_code)}) > 模型最大输入长度({self.max_input_length})，"
+                    f"正在分割为多个块..."
+                )
+                chunks = self.split_text_into_chunks(source_code)
+                symbols_list = []
+                for chunk in chunks:
+                    chunk_symbols = self.get_all_file_symbols.with_llm(self.llm).run(source.module_name, chunk)
+                    time.sleep(self.anti_quota_limit)
+                    symbols_list.append(chunk_symbols.output)
+                symbols = "\n".join(symbols_list)
+            else:
+                single_symbols = self.get_all_file_symbols.with_llm(self.llm).run(source.module_name, source_code)
+                symbols = single_symbols.output
+                time.sleep(self.anti_quota_limit)
+
+            logger.info(f"解析并更新索引：文件 {file_path}（MD5: {md5}），耗时 {time.monotonic() - start_time:.2f} 秒")
+        except Exception as e:
+            logger.warning(f"源文件 {file_path} 处理失败: {e}")
+            return None
+
+        return {
+            "module_name": source.module_name,
+            "symbols": symbols,
+            "last_modified": os.path.getmtime(file_path),
+            "md5": md5,
+        }
+
+    @prompt()
+    def _get_target_files_by_query(self, indices: str, query: str) -> str:
+        """
+        下面是已知文件以及对应的符号信息：
+
+        {{ indices }}
+
+        用户的问题是：
+
+        {{ query }}
+
+        现在，请根据用户的问题以及前面的文件和符号信息，寻找相关文件路径。返回结果按如下格式：
+
+        ```json
+        {
+            "file_list": [
+                {
+                    "file_path": "path/to/file.py",
+                    "reason": "The reason why the file is the target file"
+                },
+                {
+                    "file_path": "path/to/file.py",
+                    "reason": "The reason why the file is the target file"
+                }
+            ]
+        }
+        ```
+
+        如果没有找到，返回如下 json 即可：
+
+        ```json
+            {"file_list": []}
+        ```
+
+        请严格遵循以下步骤：
+
+        1. 识别特殊标记：
+           - 查找query中的 `@` 符号，它后面的内容是用户关注的文件路径。
+           - 查找query中的 `@@` 符号，它后面的内容是用户关注的符号（如函数名、类名、变量名）。
+
+        2. 匹配文件路径：
+           - 对于 `@` 标记，在indices中查找包含该路径的所有文件。
+           - 路径匹配应该是部分匹配，因为用户可能只提供了路径的一部分。
+
+        3. 匹配符号：
+           - 对于 `@@` 标记，在indices中所有文件的符号信息中查找该符号。
+           - 检查函数、类、变量等所有符号类型。
+
+        4. 分析依赖关系：
+           - 利用 "导入语句" 信息确定文件间的依赖关系。
+           - 如果找到了相关文件，也包括与之直接相关的依赖文件。
+
+        5. 考虑文件用途：
+           - 使用每个文件的 "用途" 信息来判断其与查询的相关性。
+
+        6. 请严格按格式要求返回结果,无需额外的说明
+
+        请确保结果的准确性和完整性，包括所有可能相关的文件。
+        """
+
+    def read_index(self) -> List[IndexItem]:
+        """ 读取并解析索引文件，将其转换为 IndexItem 对象列表 """
+        if not os.path.exists(self.index_file):
+            return []
+
+        with open(self.index_file, "r") as file:
+            index_data = json.load(file)
+
+        index_items = []
+        for module_name, data in index_data.items():
+            index_item = IndexItem(
+                module_name=module_name,
+                symbols=data["symbols"],
+                last_modified=data["last_modified"],
+                md5=data["md5"]
+            )
+            index_items.append(index_item)
+
+        return index_items
+
+    def _get_meta_str(self, includes: Optional[List[SymbolType]] = None):
+        index_items = self.read_index()
+        current_chunk = []
+        for item in index_items:
+            symbols_str = item.symbols
+            if includes:
+                symbol_info = extract_symbols(symbols_str)
+                symbols_str = symbols_info_to_str(symbol_info, includes)
+
+            item_str = f"##{item.module_name}\n{symbols_str}\n\n"
+            if len(current_chunk) > self.args.filter_batch_size:
+                yield "".join(current_chunk)
+                current_chunk = [item_str]
+            else:
+                current_chunk.append(item_str)
+        if current_chunk:
+            yield "".join(current_chunk)
+
+    def get_target_files_by_query(self, query: str):
+        """ 根据查询条件查找相关文件，考虑不同过滤级别 """
+        all_results = []
+        completed = 0
+        total = 0
+
+        includes = None
+        if self.args.index_filter_level == 0:
+            includes = [SymbolType.USAGE]
+        if self.args.index_filter_level >= 1:
+            includes = None
+
+        for chunk in self._get_meta_str(includes=includes):
+            result = self._get_target_files_by_query.with_llm(self.llm).with_return_type(FileList).run(chunk, query)
+            if result is not None:
+                all_results.extend(result.file_list)
+                completed += 1
+            else:
+                logger.warning(f"无法找到分块的目标文件。原因可能是模型响应未返回 JSON 格式数据，或返回的 JSON 为空。")
+            total += 1
+            time.sleep(self.anti_quota_limit)
+
+        logger.info(f"已完成 {completed}/{total} 个分块(基于查询条件)")
+        all_results = list({file.file_path: file for file in all_results}.values())
+        if self.args.index_filter_file_num > 0:
+            limited_results = all_results[: self.args.index_filter_file_num]
+            return FileList(file_list=limited_results)
+        return FileList(file_list=all_results)
+
+    @prompt()
+    def _get_related_files(self, indices: str, file_paths: str) -> str:
+        """
+        下面是所有文件以及对应的符号信息：
+
+        {{ indices }}
+
+        请参考上面的信息，找到被下列文件使用或者引用到的文件列表：
+
+        {{ file_paths }}
+
+        请按如下格式进行输出：
+
+        ```json
+        {
+            "file_list": [
+                {
+                    "file_path": "path/to/file.py",
+                    "reason": "The reason why the file is the target file"
+                },
+                {
+                    "file_path": "path/to/file.py",
+                    "reason": "The reason why the file is the target file"
+                }
+            ]
+        }
+        ```
+
+        如果没有相关的文件，输出如下 json 即可：
+
+        ```json
+        {"file_list": []}
+        ```
+
+        注意，
+        1. 找到的文件名必须出现在上面的文件列表中
+        2. 原因控制在20字以内, 且使用中文
+        3. 请严格按格式要求返回结果,无需额外的说明
+        """
+
+    def get_related_files(self, file_paths: List[str]):
+        """ 根据文件路径查询相关文件 """
+        all_results = []
+
+        completed = 0
+        total = 0
+
+        for chunk in self._get_meta_str():
+            result = self._get_related_files.with_llm(self.llm).with_return_type(
+                FileList).run(chunk, "\n".join(file_paths))
+            if result is not None:
+                all_results.extend(result.file_list)
+                completed += 1
+            else:
+                logger.warning(f"无法找到与分块相关的文件。原因可能是模型限制或查询条件与文件不匹配。")
+            total += 1
+            time.sleep(self.anti_quota_limit)
+        logger.info(f"已完成 {completed}/{total} 个分块(基于相关文件)")
+        all_results = list({file.file_path: file for file in all_results}.values())
+        return FileList(file_list=all_results)
+
+    @prompt()
+    def verify_file_relevance(self, file_content: str, query: str) -> str:
+        """
+        请验证下面的文件内容是否与用户问题相关:
+
+        文件内容:
+        {{ file_content }}
+
+        用户问题:
+        {{ query }}
+
+        相关是指，需要依赖这个文件提供上下文，或者需要修改这个文件才能解决用户的问题。
+        请给出相应的可能性分数：0-10，并结合用户问题，理由控制在50字以内，并且使用中文。
+        请严格按格式要求返回结果。
+        格式如下:
+
+        ```json
+        {
+            "relevant_score": 0-10,
+            "reason": "这是相关的原因..."
+        }
+        ```
+        """

autocoder_nano/index/symbols_utils.py

@@ -0,0 +1,43 @@
+import re
+from typing import List
+
+from autocoder_nano.llm_types import SymbolsInfo, SymbolType
+
+
+def extract_symbols(text: str) -> SymbolsInfo:
+    patterns = {
+        "usage": r"用途：(.+)",
+        "functions": r"函数：(.+)",
+        "variables": r"变量：(.+)",
+        "classes": r"类：(.+)",
+        "import_statements": r"导入语句：(.+)",
+    }
+
+    info = SymbolsInfo()
+    for field, pattern in patterns.items():
+        match = re.search(pattern, text)
+        if match:
+            value = match.group(1).strip()
+            if field == "import_statements":
+                value = [v.strip() for v in value.split("^^")]
+            elif field == "functions" or field == "variables" or field == "classes":
+                value = [v.strip() for v in value.split(",")]
+            setattr(info, field, value)
+
+    return info
+
+
+def symbols_info_to_str(info: SymbolsInfo, symbol_types: List[SymbolType]) -> str:
+    result = []
+    for symbol_type in symbol_types:
+        value = getattr(info, symbol_type.value)
+        if value:
+            if symbol_type == SymbolType.IMPORT_STATEMENTS:
+                value_str = "^^".join(value)
+            elif symbol_type in [SymbolType.FUNCTIONS, SymbolType.VARIABLES, SymbolType.CLASSES,]:
+                value_str = ",".join(value)
+            else:
+                value_str = value
+            result.append(f"{symbol_type.value}：{value_str}")
+
+    return "\n".join(result)

autocoder_nano/llm_types.py

@@ -30,9 +30,9 @@ class AutoCoderArgs(BaseModel):
     context: Optional[str] = None  #
     human_as_model: Optional[bool] = False  #
     human_model_num: Optional[int] = 1  #
-    include_project_structure: Optional[bool] = False  #
+    include_project_structure: Optional[bool] = False  # 在生成代码的提示中是否包含项目目录结构
     urls: Optional[Union[str, List[str]]] = ""  # 一些文档的URL/路径，可以帮助模型了解你当前的工作
-    model: Optional[str] = ""  # 您要驱动运行的模型
+    # model: Optional[str] = ""  # 您要驱动运行的模型
     model_max_input_length: Optional[int] = 6000  # 模型最大输入长度
     skip_confirm: Optional[bool] = False
     silence: Optional[bool] = False
@@ -66,12 +66,16 @@ class AutoCoderArgs(BaseModel):
     # 模型相关参数
     current_chat_model: Optional[str] = ""
     current_code_model: Optional[str] = ""
-    # chat_model: Optional[str] = ""
-    # code_model: Optional[str] = ""
-    # emb_model: Optional[str] = ""
-    # recall_model: Optional[str] = ""
-    # qa_model: Optional[str] = ""
-    # chunk_model: Optional[str] = ""
+    model: Optional[str] = ""    # 默认模型
+    chat_model: Optional[str] = ""    # AI Chat交互模型
+    index_model: Optional[str] = ""    # 代码索引生成模型
+    code_model: Optional[str] = ""    # 编码模型
+    commit_model: Optional[str] = ""    # Git Commit 模型
+    emb_model: Optional[str] = ""    # RAG Emb 模型
+    recall_model: Optional[str] = ""    # RAG 召回阶段模型
+    chunk_model: Optional[str] = ""    # 段落重排序模型
+    qa_model: Optional[str] = ""    # RAG 提问模型
+    vl_model: Optional[str] = ""    # 多模态模型
 
     class Config:
         protected_namespaces = ()

autocoder_nano/version.py

@@ -1,3 +1,3 @@
-__version__ = "0.1.25"
+__version__ = "0.1.27"
 __author__ = "moofs"
 __license__ = "Apache License 2.0"

{autocoder_nano-0.1.25.dist-info → autocoder_nano-0.1.27.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: autocoder_nano
-Version: 0.1.25
+Version: 0.1.27
 Summary: AutoCoder Nano
 Author: moofs
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence