auto-coder 0.1.346__tar.gz → 0.1.347__tar.gz

{auto_coder-0.1.346 → auto_coder-0.1.347}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: auto-coder
-Version: 0.1.346
+Version: 0.1.347
 Summary: AutoCoder: AutoCoder
 Author: allwefantasy
 Classifier: Programming Language :: Python :: 3.10

{auto_coder-0.1.346 → auto_coder-0.1.347}/src/auto_coder.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: auto-coder
-Version: 0.1.346
+Version: 0.1.347
 Summary: AutoCoder: AutoCoder
 Author: allwefantasy
 Classifier: Programming Language :: Python :: 3.10

{auto_coder-0.1.346 → auto_coder-0.1.347}/src/auto_coder.egg-info/SOURCES.txt

@@ -129,6 +129,7 @@ src/autocoder/common/v2/agent/agentic_edit_tools/base_tool_resolver.py
 src/autocoder/common/v2/agent/agentic_edit_tools/execute_command_tool_resolver.py
 src/autocoder/common/v2/agent/agentic_edit_tools/list_code_definition_names_tool_resolver.py
 src/autocoder/common/v2/agent/agentic_edit_tools/list_files_tool_resolver.py
+src/autocoder/common/v2/agent/agentic_edit_tools/list_package_info_tool_resolver.py
 src/autocoder/common/v2/agent/agentic_edit_tools/plan_mode_respond_tool_resolver.py
 src/autocoder/common/v2/agent/agentic_edit_tools/read_file_tool_resolver.py
 src/autocoder/common/v2/agent/agentic_edit_tools/replace_in_file_tool_resolver.py
@@ -223,6 +224,7 @@ src/autocoder/rag/cache/__init__.py
 src/autocoder/rag/cache/base_cache.py
 src/autocoder/rag/cache/byzer_storage_cache.py
 src/autocoder/rag/cache/cache_result_merge.py
+src/autocoder/rag/cache/failed_files_utils.py
 src/autocoder/rag/cache/file_monitor_cache.py
 src/autocoder/rag/cache/local_byzer_storage_cache.py
 src/autocoder/rag/cache/local_duckdb_storage_cache.py

{auto_coder-0.1.346 → auto_coder-0.1.347}/src/autocoder/common/auto_coder_lang.py

@@ -833,6 +833,14 @@ MESSAGES = {
     "/agent/edit/user_query":{
         "en":"User Query",
         "zh":"用户查询"
+    },
+    "/agent/edit/apply_pre_changes":{
+        "en":"Commit user changes",
+        "zh":"提交用户修改"
+    },
+    "/agent/edit/apply_changes":{
+        "en":"Commit changes",
+        "zh":"提交修改"
     }
 }
 

{auto_coder-0.1.346 → auto_coder-0.1.347}/src/autocoder/common/v2/agent/agentic_edit.py

@@ -52,8 +52,10 @@ from autocoder.common.v2.agent.agentic_edit_tools import (  # Import specific re
     ExecuteCommandToolResolver, ReadFileToolResolver, WriteToFileToolResolver,
     ReplaceInFileToolResolver, SearchFilesToolResolver, ListFilesToolResolver,
     ListCodeDefinitionNamesToolResolver, AskFollowupQuestionToolResolver,
-    AttemptCompletionToolResolver, PlanModeRespondToolResolver, UseMcpToolResolver
+    AttemptCompletionToolResolver, PlanModeRespondToolResolver, UseMcpToolResolver,
+    ListPackageInfoToolResolver
 )
+
 from autocoder.common.v2.agent.agentic_edit_types import (AgenticEditRequest, ToolResult,
                                                           MemoryConfig, CommandConfig, BaseTool,
                                                           ExecuteCommandTool, ReadFileTool,
@@ -63,6 +65,7 @@ from autocoder.common.v2.agent.agentic_edit_types import (AgenticEditRequest, To
                                                           ListFilesTool,
                                                           ListCodeDefinitionNamesTool, AskFollowupQuestionTool,
                                                           AttemptCompletionTool, PlanModeRespondTool, UseMcpTool,
+                                                          ListPackageInfoTool,
                                                           TOOL_MODEL_MAP,
                                                           # Event Types
                                                           LLMOutputEvent, LLMThinkingEvent, ToolCallEvent,
@@ -83,6 +86,7 @@ TOOL_RESOLVER_MAP: Dict[Type[BaseTool], Type[BaseToolResolver]] = {
     SearchFilesTool: SearchFilesToolResolver,
     ListFilesTool: ListFilesToolResolver,
     ListCodeDefinitionNamesTool: ListCodeDefinitionNamesToolResolver,
+    ListPackageInfoTool: ListPackageInfoToolResolver,
     AskFollowupQuestionTool: AskFollowupQuestionToolResolver,
     AttemptCompletionTool: AttemptCompletionToolResolver,  # Will stop the loop anyway
     PlanModeRespondTool: PlanModeRespondToolResolver,
@@ -245,6 +249,7 @@ class AgenticEdit:
 
         Always adhere to this format for the tool use to ensure proper parsing and execution.
 
+
         # Tools
 
         ## execute_command
@@ -258,6 +263,15 @@ class AgenticEdit:
         <requires_approval>true or false</requires_approval>
         </execute_command>
 
+        ## list_package_info
+        Description: Request to retrieve information about a source code package, such as recent changes or documentation summary, to better understand the code context. It accepts a directory path (absolute or relative to the current project).
+        Parameters:
+        - path: (required) The source code package directory path.
+        Usage:
+        <list_package_info>
+        <path>relative/or/absolute/package/path</path>
+        </list_package_info>
+
         ## read_file
         Description: Request to read the contents of a file at the specified path. Use this when you need to examine the contents of an existing file you do not know the contents of, for example to analyze code, review text files, or extract information from configuration files. Automatically extracts raw text from PDF and DOCX files. May not be suitable for other types of binary files, as it returns the raw content as a string.
         Parameters:
@@ -670,7 +684,7 @@ class AgenticEdit:
 
         {% if extra_docs %}
         ====
-        
+
         RULES PROVIDED BY USER
 
         The following rules are provided by the user, and you must follow them strictly.
@@ -683,7 +697,8 @@ class AgenticEdit:
         """
         import os
         extra_docs = {}
-        rules_dir = os.path.join(self.args.source_dir,".auto-coder", "autocoderrules")
+        rules_dir = os.path.join(self.args.source_dir,
+                                 ".auto-coder", "autocoderrules")
         if os.path.isdir(rules_dir):
             for fname in os.listdir(rules_dir):
                 if fname.endswith(".md"):
@@ -1137,6 +1152,7 @@ class AgenticEdit:
         standard event system format and writing them using the event manager.
         """
         event_manager = get_event_manager(self.args.event_file)
+        self.apply_pre_changes()
 
         try:
             event_stream = self.analyze(request)
@@ -1280,6 +1296,25 @@ class AgenticEdit:
             # Re-raise the exception if needed, or handle appropriately
             raise e
 
+    def apply_pre_changes(self):
+        # get the file name
+        file_name = os.path.basename(self.args.file)
+        if not self.args.skip_commit:
+            try:
+                get_event_manager(self.args.event_file).write_result(
+                    EventContentCreator.create_result(
+                        content=self.printer.get_message_from_key("/agent/edit/apply_pre_changes")), metadata=EventMetadata(
+                        action_file=self.args.file,
+                        is_streaming=False,
+                        path="/agent/edit/apply_pre_changes",
+                        stream_out_type="/agent/edit").to_dict())
+                git_utils.commit_changes(
+                    self.args.source_dir, f"auto_coder_pre_{file_name}")
+            except Exception as e:
+                self.printer.print_in_terminal("git_init_required",
+                                               source_dir=self.args.source_dir, error=str(e))
+                return
+
     def apply_changes(self):
         """
         Apply all tracked file changes to the original project directory.
@@ -1297,6 +1332,12 @@ class AgenticEdit:
                         f"{self.args.query}\nauto_coder_{file_name}",
                     )
 
+                    get_event_manager(self.args.event_file).write_result(
+                        EventContentCreator.create_result(
+                            content=self.printer.get_message_from_key("/agent/edit/apply_changes")), metadata=EventMetadata(
+                            action_file=self.args.file,
+                            is_streaming=False,
+                            stream_out_type="/agent/edit").to_dict())
                     action_yml_file_manager = ActionYmlFileManager(
                         self.args.source_dir)
                     action_file_name = os.path.basename(self.args.file)
@@ -1343,6 +1384,7 @@ class AgenticEdit:
             f"[bold]{get_message('/agent/edit/user_query')}:[/bold]\n{request.user_input}", title=get_message("/agent/edit/objective"), border_style="blue"))
 
         try:
+            self.apply_pre_changes()
             event_stream = self.analyze(request)
             for event in event_stream:
                 if isinstance(event, LLMThinkingEvent):
@@ -1434,7 +1476,8 @@ class AgenticEdit:
                             logger.warning(
                                 f"Error formatting tool result content: {e}")
                             panel_content.append(
-                                _format_content(str(result.content)))  # Fallback
+                                # Fallback
+                                _format_content(str(result.content)))
 
                     # Print the base info panel
                     console.print(Panel("\n".join(

{auto_coder-0.1.346 → auto_coder-0.1.347}/src/autocoder/common/v2/agent/agentic_edit_tools/__init__.py

@@ -11,6 +11,7 @@ from .ask_followup_question_tool_resolver import AskFollowupQuestionToolResolver
 from .attempt_completion_tool_resolver import AttemptCompletionToolResolver
 from .plan_mode_respond_tool_resolver import PlanModeRespondToolResolver
 from .use_mcp_tool_resolver import UseMcpToolResolver
+from .list_package_info_tool_resolver import ListPackageInfoToolResolver
 
 __all__ = [
     "BaseToolResolver",
@@ -25,4 +26,5 @@ __all__ = [
     "AttemptCompletionToolResolver",
     "PlanModeRespondToolResolver",
     "UseMcpToolResolver",
+    "ListPackageInfoToolResolver",
 ]

auto_coder-0.1.347/src/autocoder/common/v2/agent/agentic_edit_tools/list_package_info_tool_resolver.py

@@ -0,0 +1,42 @@
+
+import os
+from typing import Optional
+from autocoder.common.v2.agent.agentic_edit_tools.base_tool_resolver import BaseToolResolver
+from autocoder.common.v2.agent.agentic_edit_types import ListPackageInfoTool, ToolResult
+from loguru import logger
+import typing
+
+if typing.TYPE_CHECKING:
+    from autocoder.common.v2.agent.agentic_edit import AgenticEdit
+
+class ListPackageInfoToolResolver(BaseToolResolver):
+    def __init__(self, agent: Optional['AgenticEdit'], tool: ListPackageInfoTool, args):
+        super().__init__(agent, tool, args)
+        self.tool: ListPackageInfoTool = tool
+
+    def resolve(self) -> ToolResult:
+        source_dir = self.args.source_dir or "."
+        abs_source_dir = os.path.abspath(source_dir)
+
+        input_path = self.tool.path.strip()
+        abs_input_path = os.path.abspath(os.path.join(source_dir, input_path)) if not os.path.isabs(input_path) else input_path
+
+        # 校验输入目录是否在项目目录内
+        if not abs_input_path.startswith(abs_source_dir):
+            return ToolResult(success=False, message=f"Error: Access denied. Path outside project: {self.tool.path}")
+
+        rel_package_path = os.path.relpath(abs_input_path, abs_source_dir)
+        active_md_path = os.path.join(abs_source_dir, ".auto-coder", "active-context", rel_package_path, "active.md")
+
+        logger.info(f"Looking for package info at: {active_md_path}")
+
+        if not os.path.exists(active_md_path):
+            return ToolResult(success=True, message="No package info found for this path.", content="没有相关包信息。")
+
+        try:
+            with open(active_md_path, 'r', encoding='utf-8', errors='replace') as f:
+                content = f.read()
+            return ToolResult(success=True, message="Successfully retrieved package info.", content=content)
+        except Exception as e:
+            logger.error(f"Error reading package info file: {e}")
+            return ToolResult(success=False, message=f"Error reading package info file: {e}")

{auto_coder-0.1.346 → auto_coder-0.1.347}/src/autocoder/common/v2/agent/agentic_edit_types.py

@@ -57,6 +57,9 @@ class UseMcpTool(BaseTool):
     tool_name: str
     query:str
 
+class ListPackageInfoTool(BaseTool):
+    path: str  # 源码包目录，相对路径或绝对路径
+
 # Event Types for Rich Output Streaming
 class LLMOutputEvent(BaseModel):
     """Represents plain text output from the LLM."""
@@ -112,6 +115,7 @@ TOOL_MODEL_MAP: Dict[str, Type[BaseTool]] = {
     "attempt_completion": AttemptCompletionTool,
     "plan_mode_respond": PlanModeRespondTool,
     "use_mcp_tool": UseMcpTool,
+    "list_package_info": ListPackageInfoTool,
 }
 
 class FileChangeEntry(BaseModel):

{auto_coder-0.1.346 → auto_coder-0.1.347}/src/autocoder/rag/cache/byzer_storage_cache.py

@@ -29,6 +29,7 @@ import hashlib
 from typing import Union
 from pydantic import BaseModel
 from autocoder.rag.cache.cache_result_merge import CacheResultMerger, MergeStrategy
+from .failed_files_utils import save_failed_files, load_failed_files
 import time
 
 if platform.system() != "Windows":
@@ -65,74 +66,18 @@ class ByzerStorageCache(BaseCacheManager):
         ignore_spec,
         required_exts,
         extra_params: Optional[AutoCoderArgs] = None,
+        args=None,
+        llm=None,
     ):
         """
         初始化基于云端 Byzer Storage 的 RAG 缓存管理器。
-        
-        参数:
-            path: 需要索引的代码库根目录
-            ignore_spec: 指定哪些文件/目录应被忽略的规则
-            required_exts: 需要处理的文件扩展名列表
-            extra_params: 额外的配置参数，包含向量索引相关设置
-            
-        缓存结构 (self.cache):
-            self.cache 是一个字典，键为文件路径，值为 CacheItem 对象:
-            {
-                "file_path1": CacheItem(
-                    file_path: str,              # 文件的绝对路径
-                    relative_path: str,          # 相对于项目根目录的路径
-                    content: List[Dict],         # 文件内容的结构化表示，每个元素是 SourceCode 对象的序列化
-                    modify_time: float,          # 文件最后修改时间的时间戳
-                    md5: str                     # 文件内容的 MD5 哈希值，用于检测变更
-                ),
-                "file_path2": CacheItem(...),
-                ...
-            }
-            
-            这个缓存有两层存储:
-            1. 本地文件缓存: 保存在项目根目录的 .cache/byzer_storage_speedup.jsonl 文件中
-               - 用于跟踪文件变更和快速加载
-               - 使用 JSONL 格式存储，每行是一个 CacheItem 的 JSON 表示
-            
-            2. 云端 Byzer Storage 向量数据库:
-               - 存储文件内容的分块和向量嵌入
-               - 每个文件被分割成大小为 chunk_size 的文本块
-               - 每个块都会生成向量嵌入，用于语义搜索
-               - 存储结构包含: 文件路径、内容块、原始内容、向量嵌入、修改时间
-        
-        源代码处理流程:
-            在缓存更新过程中使用了两个关键函数:
-            
-            1. process_file_in_multi_process: 在多进程环境中处理文件
-               - 参数: file_info (文件信息元组)
-               - 返回值: List[SourceCode] 或 None
-               - 用途: 在初始构建缓存时并行处理多个文件
-            
-            2. process_file_local: 在当前进程中处理单个文件
-               - 参数: file_path (文件路径)
-               - 返回值: List[SourceCode] 或 None
-               - 用途: 在检测到文件更新时处理单个文件
-            
-            文件处理后，会:
-            1. 更新内存中的缓存 (self.cache)
-            2. 将缓存持久化到本地文件
-            3. 将内容分块并更新到 Byzer Storage 向量数据库
-        
-        更新机制:
-            - 通过单独的线程异步处理文件变更
-            - 使用 MD5 哈希值检测文件是否发生变化
-            - 支持文件添加、更新和删除事件
-            - 使用向量数据库进行语义检索，支持相似度搜索
-            
-        与 LocalByzerStorageCache 的区别:
-            - 使用云端 ByzerStorage 而非本地存储
-            - 适用于需要远程访问和共享索引的场景
-            - 支持大规模分布式检索和更高级的查询功能
         """
         self.path = path
         self.ignore_spec = ignore_spec
         self.required_exts = required_exts
         self.extra_params = extra_params
+        self.args = args
+        self.llm = llm
         self.rag_build_name = extra_params.rag_build_name
         self.storage = ByzerStorage("byzerai_store", "rag", self.rag_build_name)
         self.queue = []
@@ -150,16 +95,20 @@ class ByzerStorageCache(BaseCacheManager):
         self.cache_file = os.path.join(self.cache_dir, "byzer_storage_speedup.jsonl")
         self.cache: Dict[str, CacheItem] = {}
 
+        # 创建缓存目录
+        if not os.path.exists(self.cache_dir):
+            os.makedirs(self.cache_dir)
+
+        # failed files support
+        self.failed_files_path = os.path.join(self.cache_dir, "failed_files.json")
+        self.failed_files = load_failed_files(self.failed_files_path)
+
         self.lock = threading.Lock()
         self.stop_event = threading.Event()
         self.thread = threading.Thread(target=self.process_queue)
         self.thread.daemon = True
         self.thread.start()
 
-        # 创建缓存目录
-        if not os.path.exists(self.cache_dir):
-            os.makedirs(self.cache_dir)
-
         # 加载缓存
         self.cache = self._load_cache()
 
@@ -455,6 +404,10 @@ class ByzerStorageCache(BaseCacheManager):
                 for item in file_list.file_paths:
                     logger.info(f"[QUEUE PROCESSING] Processing file deletion: {item}")
                     del self.cache[item]
+                    # remove from failed files if present
+                    if item in self.failed_files:
+                        self.failed_files.remove(item)
+                        save_failed_files(self.failed_files_path, self.failed_files)
                     # Create a temporary FileInfo object
                     file_info = FileInfo(file_path=item, relative_path="", modify_time=0, file_md5="")
                     self.update_storage(file_info, is_delete=True)
@@ -463,16 +416,29 @@ class ByzerStorageCache(BaseCacheManager):
                 logger.info(f"[QUEUE PROCESSING] Processing add/update event, total files: {len(file_list.file_infos)}")
                 for file_info in file_list.file_infos:
                     logger.info(f"[QUEUE PROCESSING] Processing file update: {file_info.file_path}")
-                    # Process file and create CacheItem
-                    content = process_file_local(self.fileinfo_to_tuple(file_info))
-                    self.cache[file_info.file_path] = CacheItem(
-                        file_path=file_info.file_path,
-                        relative_path=file_info.relative_path,
-                        content=[c.model_dump() for c in content],
-                        modify_time=file_info.modify_time,
-                        md5=file_info.file_md5,
-                    )
-                    self.update_storage(file_info, is_delete=False)
+                    try:
+                        content = process_file_local(self.fileinfo_to_tuple(file_info))
+                        if content:
+                            self.cache[file_info.file_path] = CacheItem(
+                                file_path=file_info.file_path,
+                                relative_path=file_info.relative_path,
+                                content=[c.model_dump() for c in content],
+                                modify_time=file_info.modify_time,
+                                md5=file_info.file_md5,
+                            )
+                            self.update_storage(file_info, is_delete=False)
+                            # remove from failed files if present
+                            if file_info.file_path in self.failed_files:
+                                self.failed_files.remove(file_info.file_path)
+                                save_failed_files(self.failed_files_path, self.failed_files)
+                        else:
+                            logger.warning(f"Empty result for file: {file_info.file_path}, treat as parse failed, skipping cache update")
+                            self.failed_files.add(file_info.file_path)
+                            save_failed_files(self.failed_files_path, self.failed_files)
+                    except Exception as e:
+                        logger.error(f"Error in process_queue: {e}")
+                        self.failed_files.add(file_info.file_path)
+                        save_failed_files(self.failed_files_path, self.failed_files)
             self.write_cache()
         
         elapsed = time.time() - start_time
@@ -486,6 +452,10 @@ class ByzerStorageCache(BaseCacheManager):
         current_files = set()
         for file_info in self.get_all_files():            
             current_files.add(file_info.file_path)
+            # skip failed files
+            if file_info.file_path in self.failed_files:
+                logger.info(f"文件 {file_info.file_path} 之前解析失败，跳过此次更新")
+                continue
             if (
                 file_info.file_path not in self.cache
                 or self.cache[file_info.file_path].md5 != file_info.file_md5

auto_coder-0.1.347/src/autocoder/rag/cache/failed_files_utils.py

@@ -0,0 +1,39 @@
+
+import os
+import json
+from loguru import logger
+
+def load_failed_files(failed_files_path: str) -> set:
+    """
+    Load the set of failed file paths from a JSON file.
+    
+    Args:
+        failed_files_path: Path to the JSON file storing failed files.
+    
+    Returns:
+        A set of failed file paths.
+    """
+    directory = os.path.dirname(failed_files_path)
+    if not os.path.exists(directory):
+        os.makedirs(directory, exist_ok=True)
+    if os.path.exists(failed_files_path):
+        try:
+            with open(failed_files_path, "r", encoding="utf-8") as f:
+                return set(json.load(f))
+        except Exception:
+            return set()
+    return set()
+
+def save_failed_files(failed_files_path: str, failed_files: set) -> None:
+    """
+    Save the set of failed file paths to a JSON file.
+    
+    Args:
+        failed_files_path: Path to the JSON file.
+        failed_files: A set of failed file paths.
+    """
+    try:
+        with open(failed_files_path, "w", encoding="utf-8") as f:
+            json.dump(list(failed_files), f, ensure_ascii=False, indent=2)
+    except Exception as e:
+        logger.error(f"Error saving failed files list: {e}")

{auto_coder-0.1.346 → auto_coder-0.1.347}/src/autocoder/rag/cache/file_monitor_cache.py

@@ -48,7 +48,7 @@ class AutoCoderRAGDocListener(BaseCacheManager):
         r"^test.*$",
     ]
 
-    def __init__(self, path: str, ignore_spec, required_exts: List) -> None:
+    def __init__(self, path: str, ignore_spec, required_exts: List, args=None, llm=None) -> None:
         """
         初始化文件监控缓存管理器。
         
@@ -89,6 +89,8 @@ class AutoCoderRAGDocListener(BaseCacheManager):
         self.path = path
         self.ignore_spec = ignore_spec
         self.required_exts = required_exts
+        self.args = args
+        self.llm = llm
         self.stop_event = threading.Event()
 
         # connect list

{auto_coder-0.1.346 → auto_coder-0.1.347}/src/autocoder/rag/cache/local_byzer_storage_cache.py

@@ -30,6 +30,7 @@ from typing import Union
 from byzerllm import SimpleByzerLLM, ByzerLLM
 from autocoder.rag.cache.cache_result_merge import CacheResultMerger, MergeStrategy
 import time
+from .failed_files_utils import save_failed_files, load_failed_files
 
 if platform.system() != "Windows":
     import fcntl
@@ -70,71 +71,17 @@ class LocalByzerStorageCache(BaseCacheManager):
         emb_llm: Union[ByzerLLM, SimpleByzerLLM] = None,
         host: str = "127.0.0.1",
         port: int = 33333,
+        args=None,
+        llm=None,
     ):
         """
         初始化基于 Byzer Storage 的 RAG 缓存管理器。
-        
-        参数:
-            path: 需要索引的代码库根目录
-            ignore_spec: 指定哪些文件/目录应被忽略的规则
-            required_exts: 需要处理的文件扩展名列表
-            extra_params: 额外的配置参数，包含向量索引相关设置
-            emb_llm: 用于生成文本向量嵌入的 ByzerLLM 实例
-            host: Byzer Storage 服务的主机地址
-            port: Byzer Storage 服务的端口
-            
-        缓存结构 (self.cache):
-            self.cache 是一个字典，键为文件路径，值为 CacheItem 对象:
-            {
-                "file_path1": CacheItem(
-                    file_path: str,              # 文件的绝对路径
-                    relative_path: str,          # 相对于项目根目录的路径
-                    content: List[Dict],         # 文件内容的结构化表示，每个元素是 SourceCode 对象的序列化
-                    modify_time: float,          # 文件最后修改时间的时间戳
-                    md5: str                     # 文件内容的 MD5 哈希值，用于检测变更
-                ),
-                "file_path2": CacheItem(...),
-                ...
-            }
-            
-            这个缓存有两层存储:
-            1. 本地文件缓存: 保存在项目根目录的 .cache/byzer_storage_speedup.jsonl 文件中
-               - 用于跟踪文件变更和快速加载
-               - 使用 JSONL 格式存储，每行是一个 CacheItem 的 JSON 表示
-            
-            2. Byzer Storage 向量数据库:
-               - 存储文件内容的分块和向量嵌入
-               - 每个文件被分割成大小为 chunk_size 的文本块
-               - 每个块都会生成向量嵌入，用于语义搜索
-               - 存储结构包含: 文件路径、内容块、原始内容、向量嵌入、修改时间
-        
-        源代码处理流程:
-            在缓存更新过程中使用了两个关键函数:
-            
-            1. process_file_in_multi_process: 在多进程环境中处理文件
-               - 参数: file_info (文件信息元组)
-               - 返回值: List[SourceCode] 或 None
-               - 用途: 在初始构建缓存时并行处理多个文件
-            
-            2. process_file_local: 在当前进程中处理单个文件
-               - 参数: file_path (文件路径)
-               - 返回值: List[SourceCode] 或 None
-               - 用途: 在检测到文件更新时处理单个文件
-            
-            文件处理后，会:
-            1. 更新内存中的缓存 (self.cache)
-            2. 将缓存持久化到本地文件
-            3. 将内容分块并更新到 Byzer Storage 向量数据库
-        
-        更新机制:
-            - 通过单独的线程异步处理文件变更
-            - 使用 MD5 哈希值检测文件是否发生变化
-            - 支持文件添加、更新和删除事件
-            - 使用向量数据库进行语义检索，支持相似度搜索
         """
         self.path = path
         self.ignore_spec = ignore_spec
         self.required_exts = required_exts
+        self.args = args
+        self.llm = llm
         self.rag_build_name = extra_params.rag_build_name
         self.storage = LocalByzerStorage("byzerai_store",
             "rag_test", self.rag_build_name, host=host, port=port,emb_llm=emb_llm)
@@ -153,16 +100,20 @@ class LocalByzerStorageCache(BaseCacheManager):
             self.cache_dir, "byzer_storage_speedup.jsonl")
         self.cache: Dict[str, CacheItem] = {}
 
+        # 创建缓存目录
+        if not os.path.exists(self.cache_dir):
+            os.makedirs(self.cache_dir)
+
+        # failed files support
+        self.failed_files_path = os.path.join(self.cache_dir, "failed_files.json")
+        self.failed_files = load_failed_files(self.failed_files_path)
+
         self.lock = threading.Lock()
         self.stop_event = threading.Event()
         self.thread = threading.Thread(target=self.process_queue)
         self.thread.daemon = True
         self.thread.start()
 
-        # 创建缓存目录
-        if not os.path.exists(self.cache_dir):
-            os.makedirs(self.cache_dir)
-
         # 加载缓存
         self.cache = self._load_cache()
 
@@ -485,6 +436,10 @@ class LocalByzerStorageCache(BaseCacheManager):
                 for item in file_list.file_paths:
                     logger.info(f"[QUEUE PROCESSING] Processing file deletion: {item}")
                     del self.cache[item]
+                    # remove from failed files if present
+                    if item in self.failed_files:
+                        self.failed_files.remove(item)                        
+                        save_failed_files(self.failed_files_path, self.failed_files)
                     # Create a temporary FileInfo object
                     file_info = FileInfo(
                         file_path=item, relative_path="", modify_time=0, file_md5="")
@@ -495,17 +450,30 @@ class LocalByzerStorageCache(BaseCacheManager):
                 for file_info in file_list.file_infos:
                     logger.info(
                         f"[QUEUE PROCESSING] Processing file update: {file_info.file_path}")
-                    # Process file and create CacheItem
-                    content = process_file_local(
-                        self.fileinfo_to_tuple(file_info))
-                    self.cache[file_info.file_path] = CacheItem(
-                        file_path=file_info.file_path,
-                        relative_path=file_info.relative_path,
-                        content=[c.model_dump() for c in content],
-                        modify_time=file_info.modify_time,
-                        md5=file_info.file_md5,
-                    )
-                    self.update_storage(file_info, is_delete=False)
+                    try:
+                        content = process_file_local(
+                            self.fileinfo_to_tuple(file_info))
+                        if content:
+                            self.cache[file_info.file_path] = CacheItem(
+                                file_path=file_info.file_path,
+                                relative_path=file_info.relative_path,
+                                content=[c.model_dump() for c in content],
+                                modify_time=file_info.modify_time,
+                                md5=file_info.file_md5,
+                            )
+                            self.update_storage(file_info, is_delete=False)
+                            # remove from failed files if present
+                            if file_info.file_path in self.failed_files:
+                                self.failed_files.remove(file_info.file_path)                                
+                                save_failed_files(self.failed_files_path, self.failed_files)
+                        else:
+                            logger.warning(f"Empty result for file: {file_info.file_path}, treat as parse failed, skipping cache update")
+                            self.failed_files.add(file_info.file_path)                            
+                            save_failed_files(self.failed_files_path, self.failed_files)
+                    except Exception as e:
+                        logger.error(f"Error in process_queue: {e}")
+                        self.failed_files.add(file_info.file_path)                        
+                        save_failed_files(self.failed_files_path, self.failed_files)
             self.write_cache()
         
         elapsed = time.time() - start_time
@@ -519,6 +487,10 @@ class LocalByzerStorageCache(BaseCacheManager):
         current_files = set()
         for file_info in self.get_all_files():
             current_files.add(file_info.file_path)
+            # skip failed files
+            if file_info.file_path in self.failed_files:
+                logger.info(f"文件 {file_info.file_path} 之前解析失败，跳过此次更新")
+                continue
             if (
                 file_info.file_path not in self.cache
                 or self.cache[file_info.file_path].md5 != file_info.file_md5