auto-coder 0.1.352__py3-none-any.whl → 0.1.354__py3-none-any.whl

autocoder/rag/loaders/filter_utils.py

@@ -1,106 +1,163 @@
-
 import os
 import json
-from typing import Dict, Optional
+import threading
+from typing import Dict, Optional, List
 from loguru import logger
+from functools import lru_cache
 
 class FilterRuleManager:
     '''
+    单例模式的过滤规则管理器。支持按文件类型定义不同的过滤规则。
+    
+    支持的规则格式：
     {
-        "whitelist": [
-            "glob:*.png",
-            "regex:^/tmp/.*hidden.*"
-        ],
-        "blacklist": [
-            "glob:*/private/*",
-            "regex:.*/secret/.*\\.jpg$"
-        ]
+        "image": {
+            "whitelist": ["*.png", "*.jpg"],
+            "blacklist": ["*/private/*"]
+        },
+        "document": {
+            "whitelist": ["*.pdf", "*.docx"],
+            "blacklist": ["*/tmp/*"]
+        },
+        "default": {
+            "whitelist": [],
+            "blacklist": ["*/node_modules/*", "*/.*"]
         }
-    '''    
-    _cache_rules: Optional[Dict] = None
-    _cache_mtime: Optional[float] = None
-
-    def __init__(self, llm, source_dir: str):
-        """
-        初始化过滤规则管理器
-
-        参数:
-            llm: 大模型对象，当前未使用，预留
-            source_dir: 项目根目录路径
-        """
-        self.llm = llm
-        self.source_dir = source_dir
-        self.filter_rules_path = os.path.join(self.source_dir, ".cache", "filterrules")
+    }
+    '''
+    _instance = None
+    _lock = threading.RLock()  # 使用可重入锁避免死锁
+
+    def __new__(cls, *args, **kwargs):
+        if cls._instance is None:
+            with cls._lock:
+                if cls._instance is None:  # 双重检查锁定模式
+                    cls._instance = super(FilterRuleManager, cls).__new__(cls)
+                    cls._instance._initialized = False
+        return cls._instance
+
+    @classmethod
+    def get_instance(cls):
+        return cls()  # 直接调用__new__，不需要重复加锁
+
+    def __init__(self):
+        with self._lock:
+            if hasattr(self, '_initialized') and self._initialized:
+                return
+                
+            self.source_dir = os.getcwd()
+            self.filter_rules_path = os.path.join(self.source_dir, ".cache", "filterrules")
+            self._cache_rules: Optional[Dict] = None
+            self._cache_mtime: Optional[float] = None
+            self._rule_lock = threading.RLock()  # 单独的锁用于规则访问
+            self._initialized = True
 
     def load_filter_rules(self) -> Dict:
-        try:
-            current_mtime = os.path.getmtime(self.filter_rules_path) if os.path.exists(self.filter_rules_path) else None
-        except Exception:
-            current_mtime = None
-
+        # 先检查是否需要重新加载，不持有锁
+        current_mtime = self._get_file_mtime()
         need_reload = False
 
-        # 如果缓存为空，或者文件已更新，触发重新加载
-        if FilterRuleManager._cache_rules is None:
+        if self._cache_rules is None:
             need_reload = True
-        elif current_mtime is not None and FilterRuleManager._cache_mtime != current_mtime:
+        elif current_mtime is not None and self._cache_mtime != current_mtime:
             need_reload = True
 
+        # 只在需要重新加载时获取锁
         if need_reload:
-            FilterRuleManager._cache_rules = {"whitelist": [], "blacklist": []}
-            try:
-                if os.path.exists(self.filter_rules_path):
-                    with open(self.filter_rules_path, "r", encoding="utf-8") as f:
-                        FilterRuleManager._cache_rules = json.load(f)
-                FilterRuleManager._cache_mtime = current_mtime
-            except Exception as e:
-                logger.warning(f"Failed to load filterrules: {e}")
-
-        return FilterRuleManager._cache_rules or {"whitelist": [], "blacklist": []}
+            with self._rule_lock:
+                # 双重检查，避免多线程重复加载
+                current_mtime = self._get_file_mtime()
+                if self._cache_rules is None or (current_mtime is not None and self._cache_mtime != current_mtime):
+                    self._load_rules_from_file(current_mtime)
+
+        # 返回规则副本，避免外部修改影响缓存
+        with self._rule_lock:
+            return self._cache_rules.copy() if self._cache_rules else self._get_default_rules()
+
+    def _get_file_mtime(self) -> Optional[float]:
+        """获取文件修改时间，与IO相关的操作单独提取出来"""
+        try:
+            return os.path.getmtime(self.filter_rules_path) if os.path.exists(self.filter_rules_path) else None
+        except Exception:
+            logger.warning(f"Failed to get mtime for {self.filter_rules_path}")
+            return None
+
+    def _get_default_rules(self) -> Dict:
+        """返回默认的规则结构"""
+        return {
+            "default": {
+                "whitelist": [],
+                "blacklist": []
+            }
+        }
 
-    def should_parse_image(self, file_path: str) -> bool:
+    def _load_rules_from_file(self, current_mtime: Optional[float]) -> None:
+        """从文件加载规则，仅在持有锁时调用"""
+        self._cache_rules = self._get_default_rules()
+        try:
+            if os.path.exists(self.filter_rules_path):
+                with open(self.filter_rules_path, "r", encoding="utf-8") as f:
+                    file_rules = json.load(f)
+                    
+                    # 转换旧格式规则到新格式（如果需要）
+                    if "whitelist" in file_rules or "blacklist" in file_rules:
+                        # 旧格式转换为新格式
+                        self._cache_rules = {
+                            "default": {
+                                "whitelist": file_rules.get("whitelist", []),
+                                "blacklist": file_rules.get("blacklist", [])
+                            }
+                        }
+                        logger.info("Converted old format rules to new format")
+                    else:
+                        # 新格式直接使用
+                        self._cache_rules = file_rules
+            self._cache_mtime = current_mtime
+        except Exception as e:
+            logger.warning(f"Failed to load filterrules: {e}")
+
+    @lru_cache(maxsize=1024)  # 缓存频繁使用的路径判断结果
+    def should_parse_file(self, file_path: str, file_type: str = "default") -> bool:
         """
-        判断某个文件是否需要对图片进行解析。
-
-        支持规则格式：
-        - glob通配符匹配，示例："glob:*.png" 或 "*.png"
-        - 正则表达式匹配，示例："regex:^/tmp/.*hidden.*"
-
+        判断某个文件是否需要进行解析。
+        
+        参数:
+            file_path: 文件路径
+            file_type: 文件类型（如"image"、"document"等），默认为"default"
+            
         返回:
             True 表示应该解析
             False 表示不解析
         """
         import fnmatch
-        import re
-
+        
         rules = self.load_filter_rules()
-        whitelist = rules.get("whitelist", [])
-        blacklist = rules.get("blacklist", [])
-
-        def match_pattern(pattern: str, path: str) -> bool:
-            if pattern.startswith("glob:"):
-                pat = pattern[len("glob:"):]
-                return fnmatch.fnmatch(path, pat)
-            elif pattern.startswith("regex:"):
-                pat = pattern[len("regex:"):]
-                try:
-                    return re.search(pat, path) is not None
-                except re.error:
-                    logger.warning(f"Invalid regex pattern: {pat}")
-                    return False
-            else:
-                # 默认按glob处理
-                return fnmatch.fnmatch(path, pattern)
-
+        
+        # 获取指定类型的规则，如果不存在则使用默认规则
+        type_rules = rules.get(file_type, rules.get("default", {"whitelist": [], "blacklist": []}))
+        whitelist = type_rules.get("whitelist", [])
+        blacklist = type_rules.get("blacklist", [])
+        
         # 优先匹配黑名单
         for pattern in blacklist:
-            if match_pattern(pattern, file_path):
+            if fnmatch.fnmatch(file_path, pattern):
                 return False
-
-        # 再匹配白名单
+        
+        # 如果白名单为空，则默认所有文件都通过（除非被黑名单过滤）
+        if not whitelist:
+            return True
+            
+        # 匹配白名单
         for pattern in whitelist:
-            if match_pattern(pattern, file_path):
+            if fnmatch.fnmatch(file_path, pattern):
                 return True
-
-        # 默认不解析
+                
+        # 有白名单但不匹配，不通过
         return False
+        
+    # 保持向后兼容
+    def should_parse_image(self, file_path: str) -> bool:
+        """
+        判断某个图片文件是否需要解析（兼容旧版API）
+        """
+        return self.should_parse_file(file_path, "image")

autocoder/rag/loaders/image_loader.py

@@ -538,7 +538,7 @@ class ImageLoader:
     def image_to_markdown(
         image_path: str,
         llm,
-        engine: str = "vl",
+        engine: str = "paddle",
         product_mode: str = "lite",
         paddle_kwargs: dict = None
     ) -> str:
@@ -554,6 +554,13 @@ class ImageLoader:
         Returns:
             markdown内容字符串
         """
+        logger.info(f"image_path: {image_path} engine: {engine} product_mode: {product_mode} paddle_kwargs: {paddle_kwargs}")        
+
+        # 新增：如果 engine 为 paddle 且 PaddleOCR 为 None，直接返回空字符串
+        if engine == "paddle" and PaddleOCR is None:
+            logger.warning("PaddleOCR 未安装，无法识别图片内容，直接返回空字符串。")
+            return ""
+
         md_content = ImageLoader.extract_text_from_image(
             image_path,
             llm,
@@ -561,8 +568,13 @@ class ImageLoader:
             product_mode=product_mode,
             paddle_kwargs=paddle_kwargs
         )
-
-        md_path = os.path.splitext(image_path)[0] + ".md"
+            
+        # Get directory and filename separately
+        dir_name = os.path.dirname(image_path)
+        file_name = os.path.basename(image_path)
+        base_name = os.path.splitext(file_name)[0]
+        # Create new path with dot before filename
+        md_path = os.path.join(dir_name, f".{base_name}.md")
         try:
             with open(md_path, "w", encoding="utf-8") as f:
                 f.write(md_content)

autocoder/rag/loaders/pdf_loader.py

@@ -1,6 +1,5 @@
 from io import BytesIO
 from pypdf import PdfReader
-from autocoder.utils._markitdown import MarkItDown
 import traceback
 
 
@@ -15,7 +14,8 @@ def extract_text_from_pdf_old(file_path):
     return text
 
 def extract_text_from_pdf(file_path, llm=None, product_mode="lite"):
-    try:
+    try:        
+        from autocoder.utils._markitdown import MarkItDown
         md_converter = MarkItDown(llm=llm, product_mode=product_mode)
         result = md_converter.convert(file_path)
         return result.text_content

autocoder/rag/long_context_rag.py

@@ -41,6 +41,8 @@ from autocoder.rag.qa_conversation_strategy import get_qa_strategy
 from autocoder.rag.searchable import SearchableResults
 from autocoder.rag.conversation_to_queries import extract_search_queries
 from autocoder.common import openai_content as OpenAIContentProcessor
+from autocoder.common.save_formatted_log import save_formatted_log
+import json, os
 try:
     from autocoder_pro.rag.llm_compute import LLMComputeEngine
     pro_version = version("auto-coder-pro")
@@ -849,6 +851,15 @@ class LongContextRAG:
                     conversations=conversations, local_image_host=self.args.local_image_host
                 )
 
+                # 保存 new_conversations
+                try:                    
+                    logger.info(f"Saving new_conversations log to {self.args.source_dir}/.cache/logs")
+                    project_root = self.args.source_dir
+                    json_text = json.dumps(new_conversations, ensure_ascii=False)
+                    save_formatted_log(project_root, json_text, "rag_conversation")
+                except Exception as e:
+                    logger.warning(f"Failed to save new_conversations log: {e}")
+
                 chunks = target_llm.stream_chat_oai(
                     conversations=new_conversations,
                     model=model,

autocoder/rag/qa_conversation_strategy.py

@@ -2,6 +2,7 @@ from abc import ABC, abstractmethod
 from typing import List, Dict, Any, Generator
 import byzerllm
 from autocoder.common import AutoCoderArgs
+from autocoder.common.rulefiles.autocoderrules_utils import get_rules
 
 class QAConversationStrategy(ABC):
     """
@@ -124,22 +125,8 @@ class MultiRoundStrategy(QAConversationStrategy):
         {% endfor %}        
         {% endif %}
 
-        """
-
-        import os
-        extra_docs = {}
-        rules_dir = os.path.join(self.args.source_dir, ".autocoderrules")
-        if os.path.isdir(rules_dir):
-            for fname in os.listdir(rules_dir):
-                if fname.endswith(".md"):
-                    fpath = os.path.join(rules_dir, fname)
-                    try:
-                        with open(fpath, "r", encoding="utf-8") as f:
-                            content = f.read()
-                            key = os.path.splitext(fname)[0]
-                            extra_docs[key] = content
-                    except Exception:
-                        continue
+        """        
+        extra_docs = get_rules()               
         return {"extra_docs": extra_docs}
 
 class SingleRoundStrategy(QAConversationStrategy):
@@ -253,21 +240,8 @@ class SingleRoundStrategy(QAConversationStrategy):
         {% endfor %}        
         {% endif %}
 
-        """
-        import os
-        extra_docs = {}
-        rules_dir = os.path.join(getattr(self, 'args', None).source_dir if getattr(self, 'args', None) else ".", ".autocoderrules")
-        if os.path.isdir(rules_dir):
-            for fname in os.listdir(rules_dir):
-                if fname.endswith(".md"):
-                    fpath = os.path.join(rules_dir, fname)
-                    try:
-                        with open(fpath, "r", encoding="utf-8") as f:
-                            content = f.read()
-                            key = os.path.splitext(fname)[0]
-                            extra_docs[key] = content
-                    except Exception:
-                        continue
+        """        
+        extra_docs = extra_docs = get_rules()
         return {"extra_docs": extra_docs}
 
 def get_qa_strategy(args: AutoCoderArgs) -> QAConversationStrategy:

autocoder/rag/utils.py

@@ -2,8 +2,9 @@ from autocoder.common import SourceCode
 from autocoder.rag.token_counter import count_tokens_worker, count_tokens
 from autocoder.rag.loaders.pdf_loader import extract_text_from_pdf
 from autocoder.rag.loaders.docx_loader import extract_text_from_docx
-from autocoder.rag.loaders.excel_loader import extract_text_from_excel
+from autocoder.rag.loaders.excel_loader import extract_text_from_excel 
 from autocoder.rag.loaders.ppt_loader import extract_text_from_ppt
+from autocoder.rag.loaders.image_loader import ImageLoader
 from typing import List, Tuple, Optional, Union
 import time
 from loguru import logger
@@ -21,7 +22,7 @@ def process_file_in_multi_process(
         llm = get_single_llm(llm,product_mode)
 
     start_time = time.time()
-    file_path, relative_path, _, _ = file_info
+    file_path, relative_path, _, _ = file_info    
     try:
         if file_path.endswith(".pdf"):
             content = extract_text_from_pdf(file_path, llm, product_mode)
@@ -61,6 +62,15 @@ def process_file_in_multi_process(
                     tokens=count_tokens_worker(content),
                 )
             ]
+        elif file_path.lower().endswith((".png", ".jpg", ".jpeg", ".bmp", ".gif")):
+            content = ImageLoader.image_to_markdown(file_path, llm=llm, product_mode=product_mode)
+            v = [
+                SourceCode(
+                    module_name=f"##File: {file_path}",
+                    source_code=content,
+                    tokens=count_tokens_worker(content),
+                )
+            ]
         else:
             with open(file_path, "r", encoding="utf-8") as f:
                 content = f.read()
@@ -126,6 +136,15 @@ def process_file_local(
                     tokens=count_tokens(content),
                 )
             ]
+        elif file_path.lower().endswith((".png", ".jpg", ".jpeg", ".bmp", ".gif")):
+            content = ImageLoader.image_to_markdown(file_path, llm=llm, product_mode=product_mode)
+            v = [
+                SourceCode(
+                    module_name=f"##File: {file_path}",
+                    source_code=content,
+                    tokens=count_tokens(content),
+                )
+            ]
         else:
             with open(file_path, "r", encoding="utf-8") as f:
                 content = f.read()

autocoder/utils/_markitdown.py

@@ -30,18 +30,20 @@ from pdfminer.pdfpage import PDFPage
 from pdfminer.pdfinterp import PDFResourceManager, PDFPageInterpreter
 import pptx
 from pdfminer.image import ImageWriter
+import time
 
 import numpy as np
 from PIL import Image
 
 # 新增导入
-from autocoder.rag.loaders import filter_utils
+from autocoder.rag.loaders.filter_utils import FilterRuleManager
 from autocoder.rag.loaders.image_loader import ImageLoader
 
 # File-format detection
 import puremagic
 import requests
 from bs4 import BeautifulSoup
+from loguru import logger
 
 # Optional Transcription support
 try:
@@ -503,12 +505,16 @@ class PdfConverter(DocumentConverter):
     Converts PDFs to Markdown with support for extracting and including images.
     """
 
+    def __init__(self, llm=None, product_mode="lite"):
+        super().__init__()
+        self.llm = llm
+        self.product_mode = product_mode
+
     def convert(self, local_path, **kwargs) -> Union[None, DocumentConverterResult]:
         # Bail if not a PDF
         extension = kwargs.get("file_extension", "")
         if extension.lower() != ".pdf":
-            return None
-
+            return None        
         image_output_dir = None
         if kwargs.get("image_output_dir", None):
             image_output_dir = kwargs.get("image_output_dir")
@@ -531,17 +537,18 @@ class PdfConverter(DocumentConverter):
             rsrcmgr = PDFResourceManager()
             laparams = LAParams()
             device = PDFPageAggregator(rsrcmgr, laparams=laparams)
-            interpreter = PDFPageInterpreter(rsrcmgr, device)
+            interpreter = PDFPageInterpreter(rsrcmgr, device)            
 
             # Process each page
             for page in PDFPage.create_pages(document):
                 interpreter.process_page(page)
-                layout = device.get_result()
+                layout = device.get_result()                
 
                 # Extract text and images from the page
                 page_content = self._process_layout(
                     layout, image_output_dir, image_count
                 )
+                
                 text_content.extend(page_content)
                 image_count += len([c for c in page_content if c.startswith("![Image")])
 
@@ -582,13 +589,12 @@ class PdfConverter(DocumentConverter):
                             image_output_dir, f"image_{local_image_count}{suffix}")
                         os.rename(temp_path, image_path)
                         content.append(f"![Image {local_image_count}]({image_path})")
-                        # ===== 新增：根据filter_utils判断是否需要解析图片
-                        if filter_utils.should_parse_image(image_path):
-                            try:
-                                _ = ImageLoader.image_to_markdown(image_path, llm=None, engine="paddle")
-                                # image_to_markdown会自动生成md文件
-                            except Exception:
-                                import traceback; traceback.print_exc()
+                        # ===== 修改：通过FilterRuleManager单例实例判断是否需要解析图片
+                        v = try_parse_image(image_path,self.llm)
+                        if v:
+                            content.append("<image_content>")
+                            content.append(v)
+                            content.append("</image_content>")
                         # =====
                         local_image_count += 1
                         continue
@@ -618,7 +624,11 @@ class PdfConverter(DocumentConverter):
                                 content.append(
                                     f"![Image {local_image_count}]({image_path})\n"
                                 )
-                                try_parse_image(image_path)
+                                v = try_parse_image(image_path,self.llm)
+                                if v:
+                                    content.append("<image_content>")
+                                    content.append(v)
+                                    content.append("</image_content>")
                                 local_image_count += 1
                                 continue
                             elif colorspace == "DeviceGray":
@@ -629,7 +639,11 @@ class PdfConverter(DocumentConverter):
                                 content.append(
                                     f"![Image {local_image_count}]({image_path})\n"
                                 )
-                                try_parse_image(image_path)
+                                v = try_parse_image(image_path,self.llm)
+                                if v:
+                                    content.append("<image_content>")
+                                    content.append(v)
+                                    content.append("</image_content>")
                                 local_image_count += 1
                                 continue
                     except Exception as e:
@@ -641,8 +655,12 @@ class PdfConverter(DocumentConverter):
                         img_file.write(image_data)
 
                     content.append(f"![Image {local_image_count}]({image_path})\n")
-                    # ===== 新增：根据filter_utils判断是否需要解析图片
-                    try_parse_image(image_path)
+                    # ===== 新增：图片解析
+                    v = try_parse_image(image_path,self.llm)
+                    if v:
+                        content.append("<image_content>")
+                        content.append(v)
+                        content.append("</image_content>")
                     local_image_count += 1
 
             # Handle text
@@ -1089,6 +1107,8 @@ class MarkItDown:
         llm: Optional[Any] = None,
         product_mode: Optional[str] = None,
     ):
+        # 初始化FilterRuleManager单例实例
+        self._filter_rule_manager = FilterRuleManager.get_instance()
         if requests_session is None:
             self._requests_session = requests.Session()
         else:
@@ -1117,7 +1137,7 @@ class MarkItDown:
         self.register_page_converter(WavConverter())
         self.register_page_converter(Mp3Converter())
         self.register_page_converter(ImageConverter())
-        self.register_page_converter(PdfConverter())
+        self.register_page_converter(PdfConverter(llm,product_mode))
 
     def convert(
         self, source: Union[str, requests.Response], **kwargs: Any
@@ -1126,8 +1146,7 @@ class MarkItDown:
         Args:
             - source: can be a string representing a path or url, or a requests.response object
             - extension: specifies the file extension to use when interpreting the file. If None, infer from source (path, uri, content-type, etc.)
-        """
-
+        """        
         # Local path or url
         if isinstance(source, str):
             if (
@@ -1343,14 +1362,36 @@ class MarkItDown:
         self._page_converters.insert(0, converter)
 
 
-def try_parse_image(image_path: str):
+def try_parse_image(image_path: str, llm=None):
     """
-    根据filter_utils判断是否需要解析图片，如果需要则调用ImageLoader.image_to_markdown。
+    根据FilterRuleManager单例实例判断是否需要解析图片，如果需要则调用ImageLoader.image_to_markdown。
     解析失败会自动捕获异常。
     """
-    if filter_utils.should_parse_image(image_path):
+    import uuid
+    start_time = time.time()
+    req_id = str(uuid.uuid4())[:8]
+    logger.info(f"\n==== [try_parse_image] START | req_id={req_id} ====")
+    logger.info(f"[try_parse_image][{req_id}] image_path: {image_path}, llm: {llm}")
+    if FilterRuleManager.get_instance().should_parse_image(image_path):
+        logger.info(f"[try_parse_image][{req_id}] should_parse_image=True, start parsing...")
         try:
-            _ = ImageLoader.image_to_markdown(image_path, llm=None, engine="paddle")
-        except Exception:
-            import traceback; traceback.print_exc()
+            v = ImageLoader.image_to_markdown(image_path, llm=llm, engine="paddle")
+            logger.info(f"[try_parse_image][{req_id}] image_to_markdown result: {str(v)[:200]}")
+            if llm:
+                v = ImageLoader.format_table_in_content(v, llm)
+                logger.info(f"[try_parse_image][{req_id}] format_table_in_content result: {str(v)[:200]}")
+            elapsed = time.time() - start_time
+            logger.info(f"[try_parse_image][{req_id}] SUCCESS | execution time: {elapsed:.3f} seconds")
+            logger.info(f"==== [try_parse_image] END | req_id={req_id} ====")
+            return v
+        except Exception as e:
+            elapsed = time.time() - start_time
+            logger.error(f"[try_parse_image][{req_id}] EXCEPTION | execution time: {elapsed:.3f} seconds | image_path: {image_path} | llm: {llm}")
+            logger.exception(e)
+            logger.info(f"==== [try_parse_image] END (EXCEPTION) | req_id={req_id} ====")
+            return ""
+    else:
+        logger.info(f"[try_parse_image][{req_id}] should_parse_image=False, skip parsing.")
+        logger.info(f"==== [try_parse_image] END (SKIP) | req_id={req_id} ====")
+        return ""
 

autocoder/utils/auto_coder_utils/chat_stream_out.py

@@ -292,6 +292,7 @@ def stream_out(
                 get_event_manager(args.event_file).write_stream(content.to_dict(),  
                     metadata=EventMetadata(
                         stream_out_type=extra_meta.get("stream_out_type", ""),
+                        path=extra_meta.get("path", ""),
                         is_streaming=True,
                         output="delta",
                         action_file=args.file

autocoder/version.py

@@ -1 +1 @@
-__version__ = "0.1.352"
+__version__ = "0.1.354"