auto-coder 0.1.353__py3-none-any.whl → 0.1.355__py3-none-any.whl

autocoder/utils/_markitdown.py

@@ -30,18 +30,20 @@ from pdfminer.pdfpage import PDFPage
 from pdfminer.pdfinterp import PDFResourceManager, PDFPageInterpreter
 import pptx
 from pdfminer.image import ImageWriter
+import time
 
 import numpy as np
 from PIL import Image
 
 # 新增导入
-from autocoder.rag.loaders import filter_utils
+from autocoder.rag.loaders.filter_utils import FilterRuleManager
 from autocoder.rag.loaders.image_loader import ImageLoader
 
 # File-format detection
 import puremagic
 import requests
 from bs4 import BeautifulSoup
+from loguru import logger
 
 # Optional Transcription support
 try:
@@ -503,12 +505,16 @@ class PdfConverter(DocumentConverter):
     Converts PDFs to Markdown with support for extracting and including images.
     """
 
+    def __init__(self, llm=None, product_mode="lite"):
+        super().__init__()
+        self.llm = llm
+        self.product_mode = product_mode
+
     def convert(self, local_path, **kwargs) -> Union[None, DocumentConverterResult]:
         # Bail if not a PDF
         extension = kwargs.get("file_extension", "")
         if extension.lower() != ".pdf":
-            return None
-
+            return None        
         image_output_dir = None
         if kwargs.get("image_output_dir", None):
             image_output_dir = kwargs.get("image_output_dir")
@@ -531,17 +537,18 @@ class PdfConverter(DocumentConverter):
             rsrcmgr = PDFResourceManager()
             laparams = LAParams()
             device = PDFPageAggregator(rsrcmgr, laparams=laparams)
-            interpreter = PDFPageInterpreter(rsrcmgr, device)
+            interpreter = PDFPageInterpreter(rsrcmgr, device)            
 
             # Process each page
             for page in PDFPage.create_pages(document):
                 interpreter.process_page(page)
-                layout = device.get_result()
+                layout = device.get_result()                
 
                 # Extract text and images from the page
                 page_content = self._process_layout(
                     layout, image_output_dir, image_count
                 )
+                
                 text_content.extend(page_content)
                 image_count += len([c for c in page_content if c.startswith("![Image")])
 
@@ -582,13 +589,12 @@ class PdfConverter(DocumentConverter):
                             image_output_dir, f"image_{local_image_count}{suffix}")
                         os.rename(temp_path, image_path)
                         content.append(f"![Image {local_image_count}]({image_path})")
-                        # ===== 新增：根据filter_utils判断是否需要解析图片
-                        if filter_utils.should_parse_image(image_path):
-                            try:
-                                _ = ImageLoader.image_to_markdown(image_path, llm=None, engine="paddle")
-                                # image_to_markdown会自动生成md文件
-                            except Exception:
-                                import traceback; traceback.print_exc()
+                        # ===== 修改：通过FilterRuleManager单例实例判断是否需要解析图片
+                        v = try_parse_image(image_path,self.llm)
+                        if v:
+                            content.append("<image_content>")
+                            content.append(v)
+                            content.append("</image_content>")
                         # =====
                         local_image_count += 1
                         continue
@@ -618,7 +624,11 @@ class PdfConverter(DocumentConverter):
                                 content.append(
                                     f"![Image {local_image_count}]({image_path})\n"
                                 )
-                                try_parse_image(image_path)
+                                v = try_parse_image(image_path,self.llm)
+                                if v:
+                                    content.append("<image_content>")
+                                    content.append(v)
+                                    content.append("</image_content>")
                                 local_image_count += 1
                                 continue
                             elif colorspace == "DeviceGray":
@@ -629,7 +639,11 @@ class PdfConverter(DocumentConverter):
                                 content.append(
                                     f"![Image {local_image_count}]({image_path})\n"
                                 )
-                                try_parse_image(image_path)
+                                v = try_parse_image(image_path,self.llm)
+                                if v:
+                                    content.append("<image_content>")
+                                    content.append(v)
+                                    content.append("</image_content>")
                                 local_image_count += 1
                                 continue
                     except Exception as e:
@@ -641,8 +655,12 @@ class PdfConverter(DocumentConverter):
                         img_file.write(image_data)
 
                     content.append(f"![Image {local_image_count}]({image_path})\n")
-                    # ===== 新增：根据filter_utils判断是否需要解析图片
-                    try_parse_image(image_path)
+                    # ===== 新增：图片解析
+                    v = try_parse_image(image_path,self.llm)
+                    if v:
+                        content.append("<image_content>")
+                        content.append(v)
+                        content.append("</image_content>")
                     local_image_count += 1
 
             # Handle text
@@ -1089,6 +1107,8 @@ class MarkItDown:
         llm: Optional[Any] = None,
         product_mode: Optional[str] = None,
     ):
+        # 初始化FilterRuleManager单例实例
+        self._filter_rule_manager = FilterRuleManager.get_instance()
         if requests_session is None:
             self._requests_session = requests.Session()
         else:
@@ -1117,7 +1137,7 @@ class MarkItDown:
         self.register_page_converter(WavConverter())
         self.register_page_converter(Mp3Converter())
         self.register_page_converter(ImageConverter())
-        self.register_page_converter(PdfConverter())
+        self.register_page_converter(PdfConverter(llm,product_mode))
 
     def convert(
         self, source: Union[str, requests.Response], **kwargs: Any
@@ -1126,8 +1146,7 @@ class MarkItDown:
         Args:
             - source: can be a string representing a path or url, or a requests.response object
             - extension: specifies the file extension to use when interpreting the file. If None, infer from source (path, uri, content-type, etc.)
-        """
-
+        """        
         # Local path or url
         if isinstance(source, str):
             if (
@@ -1343,14 +1362,36 @@ class MarkItDown:
         self._page_converters.insert(0, converter)
 
 
-def try_parse_image(image_path: str):
+def try_parse_image(image_path: str, llm=None):
     """
-    根据filter_utils判断是否需要解析图片，如果需要则调用ImageLoader.image_to_markdown。
+    根据FilterRuleManager单例实例判断是否需要解析图片，如果需要则调用ImageLoader.image_to_markdown。
     解析失败会自动捕获异常。
     """
-    if filter_utils.should_parse_image(image_path):
+    import uuid
+    start_time = time.time()
+    req_id = str(uuid.uuid4())[:8]
+    logger.info(f"\n==== [try_parse_image] START | req_id={req_id} ====")
+    logger.info(f"[try_parse_image][{req_id}] image_path: {image_path}, llm: {llm}")
+    if FilterRuleManager.get_instance().should_parse_image(image_path):
+        logger.info(f"[try_parse_image][{req_id}] should_parse_image=True, start parsing...")
         try:
-            _ = ImageLoader.image_to_markdown(image_path, llm=None, engine="paddle")
-        except Exception:
-            import traceback; traceback.print_exc()
+            v = ImageLoader.image_to_markdown(image_path, llm=llm, engine="paddle")
+            logger.info(f"[try_parse_image][{req_id}] image_to_markdown result: {str(v)[:200]}")
+            if llm:
+                v = ImageLoader.format_table_in_content(v, llm)
+                logger.info(f"[try_parse_image][{req_id}] format_table_in_content result: {str(v)[:200]}")
+            elapsed = time.time() - start_time
+            logger.info(f"[try_parse_image][{req_id}] SUCCESS | execution time: {elapsed:.3f} seconds")
+            logger.info(f"==== [try_parse_image] END | req_id={req_id} ====")
+            return v
+        except Exception as e:
+            elapsed = time.time() - start_time
+            logger.error(f"[try_parse_image][{req_id}] EXCEPTION | execution time: {elapsed:.3f} seconds | image_path: {image_path} | llm: {llm}")
+            logger.exception(e)
+            logger.info(f"==== [try_parse_image] END (EXCEPTION) | req_id={req_id} ====")
+            return ""
+    else:
+        logger.info(f"[try_parse_image][{req_id}] should_parse_image=False, skip parsing.")
+        logger.info(f"==== [try_parse_image] END (SKIP) | req_id={req_id} ====")
+        return ""
 

autocoder/utils/auto_coder_utils/chat_stream_out.py

@@ -230,9 +230,8 @@ def stream_out(
             refresh_per_second=4,
             console=console
         ) as live:
-            for res in stream_generator:
-                global_cancel.check_and_raise(cancel_token)
-                    
+            for res in stream_generator:  
+                global_cancel.check_and_raise(args.event_file)                                  
                 last_meta = res[1]                
                 content = res[0]
 
@@ -241,7 +240,7 @@ def stream_out(
                     reasoning_content = last_meta.reasoning_content
 
                 if reasoning_content == "" and content == "":
-                    continue
+                    continue             
                 
                 if first_token_time == 0.0:
                     first_token_time = time.time() - first_token_time_start
@@ -292,6 +291,7 @@ def stream_out(
                 get_event_manager(args.event_file).write_stream(content.to_dict(),  
                     metadata=EventMetadata(
                         stream_out_type=extra_meta.get("stream_out_type", ""),
+                        path=extra_meta.get("path", ""),
                         is_streaming=True,
                         output="delta",
                         action_file=args.file

autocoder/utils/thread_utils.py

@@ -36,9 +36,7 @@ def run_in_raw_thread(token: Optional[str] = None, context: Optional[Dict[str, A
         def wrapper(*args, **kwargs):
             # Store thread results
             result = []
-            exception_raised = [None]  # 存储工作线程中的异常
-            thread_token = token
-            thread_context = context or {}
+            exception_raised = [None]  # 存储工作线程中的异常            
             thread_terminated = threading.Event()  # 用于标记线程是否已终止
             
             def worker():
@@ -53,9 +51,7 @@ def run_in_raw_thread(token: Optional[str] = None, context: Optional[Dict[str, A
                 except Exception as e:
                     # 存储其他异常
                     exception_raised[0] = e
-                finally:
-                    # 无论如何执行完毕后，重置取消标志并标记线程已终止
-                    global_cancel.reset(thread_token)
+                finally:                                                            
                     thread_terminated.set()
             
             # Create and start thread with a meaningful name
@@ -72,18 +68,7 @@ def run_in_raw_thread(token: Optional[str] = None, context: Optional[Dict[str, A
                 
                 while thread.is_alive():
                     # 每次等待较短时间，以便能够及时响应中断
-                    thread.join(0.1)
-                    
-                    # 检查是否已经超过最大等待时间（仅适用于已取消的情况）
-                    elapsed_time = time.time() - wait_start_time
-                    if cancelled_by_keyboard and elapsed_time > max_wait_time:
-                        printer.print_in_terminal("force_terminating_thread")
-                        break
-                    
-                    # 检查线程间的取消请求
-                    if global_cancel.is_requested(thread_token):
-                        # 传播取消请求到工作线程
-                        raise CancelRequestedException(thread_token)
+                    thread.join(0.1)                                                                          
 
                 # 如果工作线程出现了异常，在主线程中重新抛出
                 if exception_raised[0] is not None:
@@ -92,15 +77,12 @@ def run_in_raw_thread(token: Optional[str] = None, context: Optional[Dict[str, A
                 # 返回结果
                 return result[0] if result else None            
                 
-            except KeyboardInterrupt:
-                # 设置取消标志
-                cancel_context = {"message": get_message("task_cancelled_by_user"), "source": "keyboard_interrupt"}
-                cancel_context.update(thread_context)
-                global_cancel.set(thread_token, cancel_context)
-                printer.print_in_terminal("cancellation_requested")
-
-                # 标记为键盘中断取消
-                cancelled_by_keyboard = True
+            except KeyboardInterrupt:            
+                # 取消所有任务    
+                for token in global_cancel.get_active_tokens():
+                    print(f"Cancelling job: {token}")
+                global_cancel.set_active_tokens()
+                printer.print_in_terminal("cancellation_requested")                
                 wait_start_time = time.time()
                 
                 # 等待线程终止或检测到取消

autocoder/version.py

@@ -1 +1 @@
-__version__ = "0.1.353"
+__version__ = "0.1.355"