magic-pdf 0.7.0a1__py3-none-any.whl → 0.7.0b1__py3-none-any.whl

magic_pdf/libs/Constants.py

@@ -8,4 +8,7 @@ CROSS_PAGE = "cross_page"
 block维度自定义字段
 """
 # block中lines是否被删除
-LINES_DELETED = "lines_deleted"
+LINES_DELETED = "lines_deleted"
+
+# table recognition max time default value
+TABLE_MAX_TIME_VALUE = 400

magic_pdf/libs/draw_bbox.py

@@ -1,6 +1,7 @@
 from magic_pdf.libs.Constants import CROSS_PAGE
 from magic_pdf.libs.commons import fitz  # PyMuPDF
-from magic_pdf.libs.ocr_content_type import ContentType, BlockType
+from magic_pdf.libs.ocr_content_type import ContentType, BlockType, CategoryId
+from magic_pdf.model.magic_model import MagicModel
 
 
 def draw_bbox_without_number(i, bbox_list, page, rgb_config, fill_config):
@@ -225,3 +226,67 @@ def draw_span_bbox(pdf_info, pdf_bytes, out_path):
 
     # Save the PDF
     pdf_docs.save(f"{out_path}/spans.pdf")
+
+
+def drow_model_bbox(model_list: list, pdf_bytes, out_path):
+    dropped_bbox_list = []
+    tables_body_list, tables_caption_list, tables_footnote_list = [], [], []
+    imgs_body_list, imgs_caption_list = [], []
+    titles_list = []
+    texts_list = []
+    interequations_list = []
+    pdf_docs = fitz.open("pdf", pdf_bytes)
+    magic_model = MagicModel(model_list, pdf_docs)
+    for i in range(len(model_list)):
+        page_dropped_list = []
+        tables_body, tables_caption, tables_footnote = [], [], []
+        imgs_body, imgs_caption = [], []
+        titles = []
+        texts = []
+        interequations = []
+        page_info = magic_model.get_model_list(i)
+        layout_dets = page_info["layout_dets"]
+        for layout_det in layout_dets:
+            bbox = layout_det["bbox"]
+            if layout_det["category_id"] == CategoryId.Text:
+                texts.append(bbox)
+            elif layout_det["category_id"] == CategoryId.Title:
+                titles.append(bbox)
+            elif layout_det["category_id"] == CategoryId.TableBody:
+                tables_body.append(bbox)
+            elif layout_det["category_id"] == CategoryId.TableCaption:
+                tables_caption.append(bbox)
+            elif layout_det["category_id"] == CategoryId.TableFootnote:
+                tables_footnote.append(bbox)
+            elif layout_det["category_id"] == CategoryId.ImageBody:
+                imgs_body.append(bbox)
+            elif layout_det["category_id"] == CategoryId.ImageCaption:
+                imgs_caption.append(bbox)
+            elif layout_det["category_id"] == CategoryId.InterlineEquation_YOLO:
+                interequations.append(bbox)
+            elif layout_det["category_id"] == CategoryId.Abandon:
+                page_dropped_list.append(bbox)
+
+        tables_body_list.append(tables_body)
+        tables_caption_list.append(tables_caption)
+        tables_footnote_list.append(tables_footnote)
+        imgs_body_list.append(imgs_body)
+        imgs_caption_list.append(imgs_caption)
+        titles_list.append(titles)
+        texts_list.append(texts)
+        interequations_list.append(interequations)
+        dropped_bbox_list.append(page_dropped_list)
+
+    for i, page in enumerate(pdf_docs):
+        draw_bbox_with_number(i, dropped_bbox_list, page, [158, 158, 158], True) # color !
+        draw_bbox_with_number(i, tables_body_list, page, [204, 204, 0], True)
+        draw_bbox_with_number(i, tables_caption_list, page, [255, 255, 102], True)
+        draw_bbox_with_number(i, tables_footnote_list, page, [229, 255, 204], True)
+        draw_bbox_with_number(i, imgs_body_list, page, [153, 255, 51], True)
+        draw_bbox_with_number(i, imgs_caption_list, page, [102, 178, 255], True)
+        draw_bbox_with_number(i, titles_list, page, [102, 102, 255], True)
+        draw_bbox_with_number(i, texts_list, page, [153, 0, 76], True)
+        draw_bbox_with_number(i, interequations_list, page, [0, 255, 0], True)
+
+    # Save the PDF
+    pdf_docs.save(f"{out_path}/model.pdf")

magic_pdf/libs/ocr_content_type.py

@@ -19,3 +19,17 @@ class BlockType:
     Footnote = "footnote"
     Discarded = "discarded"
 
+
+class CategoryId:
+    Title = 0
+    Text = 1
+    Abandon = 2
+    ImageBody = 3
+    ImageCaption = 4
+    TableBody = 5
+    TableCaption = 6
+    TableFootnote = 7
+    InterlineEquation_Layout = 8
+    InlineEquation = 13
+    InterlineEquation_YOLO = 14
+    OcrText = 15

magic_pdf/libs/version.py

@@ -1 +1 @@
-__version__ = "0.7.0a1"
+__version__ = "0.7.0b1"

magic_pdf/model/doc_analyze_by_custom_model.py

@@ -37,8 +37,8 @@ def load_images_from_pdf(pdf_bytes: bytes, dpi=200) -> list:
             mat = fitz.Matrix(dpi / 72, dpi / 72)
             pm = page.get_pixmap(matrix=mat, alpha=False)
 
-            # if width or height > 3000 pixels, don't enlarge the image
-            if pm.width > 3000 or pm.height > 3000:
+            # If the width or height exceeds 9000 after scaling, do not scale further.
+            if pm.width > 9000 or pm.height > 9000:
                 pm = page.get_pixmap(matrix=fitz.Matrix(1, 1), alpha=False)
 
             img = Image.frombytes("RGB", (pm.width, pm.height), pm.samples)

magic_pdf/model/pdf_extract_kit.py

@@ -2,6 +2,7 @@ from loguru import logger
 import os
 import time
 
+from magic_pdf.libs.Constants import TABLE_MAX_TIME_VALUE
 
 os.environ['NO_ALBUMENTATIONS_UPDATE'] = '1'  # 禁止albumentations检查更新
 try:
@@ -26,7 +27,7 @@ except ImportError as e:
     logger.exception(e)
     logger.error(
         'Required dependency not installed, please install by \n'
-        '"pip install magic-pdf[full] detectron2 --extra-index-url https://myhloli.github.io/wheels/"')
+        '"pip install magic-pdf[full] --extra-index-url https://myhloli.github.io/wheels/"')
     exit(1)
 
 from magic_pdf.model.pek_sub_modules.layoutlmv3.model_init import Layoutlmv3_Predictor
@@ -35,7 +36,7 @@ from magic_pdf.model.pek_sub_modules.self_modify import ModifiedPaddleOCR
 from magic_pdf.model.pek_sub_modules.structeqtable.StructTableModel import StructTableModel
 
 
-def table_model_init(model_path, max_time=400, _device_='cpu'):
+def table_model_init(model_path, max_time, _device_='cpu'):
     table_model = StructTableModel(model_path, max_time=max_time, device=_device_)
     return table_model
 
@@ -105,6 +106,7 @@ class CustomPEKModel:
         self.apply_formula = kwargs.get("apply_formula", self.configs["config"]["formula"])
         self.table_config = kwargs.get("table_config", self.configs["config"]["table_config"])
         self.apply_table = self.table_config.get("is_table_recog_enable", False)
+        self.table_max_time = self.table_config.get("max_time", TABLE_MAX_TIME_VALUE)
         self.apply_ocr = ocr
         logger.info(
             "DocAnalysis init, this may take some times. apply_layout: {}, apply_formula: {}, apply_ocr: {}, apply_table: {}".format(
@@ -141,9 +143,8 @@ class CustomPEKModel:
 
         # init structeqtable
         if self.apply_table:
-            max_time = self.table_config.get("max_time", 400)
             self.table_model = table_model_init(str(os.path.join(models_dir, self.configs["weights"]["table"])),
-                                                max_time=max_time, _device_=self.device)
+                                                max_time = self.table_max_time, _device_=self.device)
         logger.info('DocAnalysis init done!')
 
     def __call__(self, image):
@@ -187,50 +188,56 @@ class CustomPEKModel:
             mfr_cost = round(time.time() - mfr_start, 2)
             logger.info(f"formula nums: {len(mf_image_list)}, mfr time: {mfr_cost}")
 
+        # Select regions for OCR / formula regions / table regions
+        ocr_res_list = []
+        table_res_list = []
+        single_page_mfdetrec_res = []
+        for res in layout_res:
+            if int(res['category_id']) in [13, 14]:
+                single_page_mfdetrec_res.append({
+                    "bbox": [int(res['poly'][0]), int(res['poly'][1]),
+                             int(res['poly'][4]), int(res['poly'][5])],
+                })
+            elif int(res['category_id']) in [0, 1, 2, 4, 6, 7]:
+                ocr_res_list.append(res)
+            elif int(res['category_id']) in [5]:
+                table_res_list.append(res)
+
+        #  Unified crop img logic
+        def crop_img(input_res, input_pil_img, crop_paste_x=0, crop_paste_y=0):
+            crop_xmin, crop_ymin = int(input_res['poly'][0]), int(input_res['poly'][1])
+            crop_xmax, crop_ymax = int(input_res['poly'][4]), int(input_res['poly'][5])
+            # Create a white background with an additional width and height of 50
+            crop_new_width = crop_xmax - crop_xmin + crop_paste_x * 2
+            crop_new_height = crop_ymax - crop_ymin + crop_paste_y * 2
+            return_image = Image.new('RGB', (crop_new_width, crop_new_height), 'white')
+
+            # Crop image
+            crop_box = (crop_xmin, crop_ymin, crop_xmax, crop_ymax)
+            cropped_img = input_pil_img.crop(crop_box)
+            return_image.paste(cropped_img, (crop_paste_x, crop_paste_y))
+            return_list = [crop_paste_x, crop_paste_y, crop_xmin, crop_ymin, crop_xmax, crop_ymax, crop_new_width, crop_new_height]
+            return return_image, return_list
+
+        pil_img = Image.fromarray(image)
+
         # ocr识别
         if self.apply_ocr:
             ocr_start = time.time()
-            pil_img = Image.fromarray(image)
-
-            # 筛选出需要OCR的区域和公式区域
-            ocr_res_list = []
-            single_page_mfdetrec_res = []
-            for res in layout_res:
-                if int(res['category_id']) in [13, 14]:
-                    single_page_mfdetrec_res.append({
-                        "bbox": [int(res['poly'][0]), int(res['poly'][1]),
-                                 int(res['poly'][4]), int(res['poly'][5])],
-                    })
-                elif int(res['category_id']) in [0, 1, 2, 4, 6, 7]:
-                    ocr_res_list.append(res)
-
-            # 对每一个需OCR处理的区域进行处理
+            # Process each area that requires OCR processing
             for res in ocr_res_list:
-                xmin, ymin = int(res['poly'][0]), int(res['poly'][1])
-                xmax, ymax = int(res['poly'][4]), int(res['poly'][5])
-
-                paste_x = 50
-                paste_y = 50
-                # 创建一个宽高各多50的白色背景
-                new_width = xmax - xmin + paste_x * 2
-                new_height = ymax - ymin + paste_y * 2
-                new_image = Image.new('RGB', (new_width, new_height), 'white')
-
-                # 裁剪图像
-                crop_box = (xmin, ymin, xmax, ymax)
-                cropped_img = pil_img.crop(crop_box)
-                new_image.paste(cropped_img, (paste_x, paste_y))
-
-                # 调整公式区域坐标
+                new_image, useful_list = crop_img(res, pil_img, crop_paste_x=50, crop_paste_y=50)
+                paste_x, paste_y, xmin, ymin, xmax, ymax, new_width, new_height = useful_list
+                # Adjust the coordinates of the formula area
                 adjusted_mfdetrec_res = []
                 for mf_res in single_page_mfdetrec_res:
                     mf_xmin, mf_ymin, mf_xmax, mf_ymax = mf_res["bbox"]
-                    # 将公式区域坐标调整为相对于裁剪区域的坐标
+                    # Adjust the coordinates of the formula area to the coordinates relative to the cropping area
                     x0 = mf_xmin - xmin + paste_x
                     y0 = mf_ymin - ymin + paste_y
                     x1 = mf_xmax - xmin + paste_x
                     y1 = mf_ymax - ymin + paste_y
-                    # 过滤在图外的公式块
+                    # Filter formula blocks outside the graph
                     if any([x1 < 0, y1 < 0]) or any([x0 > new_width, y0 > new_height]):
                         continue
                     else:
@@ -238,17 +245,17 @@ class CustomPEKModel:
                             "bbox": [x0, y0, x1, y1],
                         })
 
-                # OCR识别
+                # OCR recognition
                 new_image = cv2.cvtColor(np.asarray(new_image), cv2.COLOR_RGB2BGR)
                 ocr_res = self.ocr_model.ocr(new_image, mfd_res=adjusted_mfdetrec_res)[0]
 
-                # 整合结果
+                # Integration results
                 if ocr_res:
                     for box_ocr_res in ocr_res:
                         p1, p2, p3, p4 = box_ocr_res[0]
                         text, score = box_ocr_res[1]
 
-                        # 将坐标转换回原图坐标系
+                        # Convert the coordinates back to the original coordinate system
                         p1 = [p1[0] - paste_x + xmin, p1[1] - paste_y + ymin]
                         p2 = [p2[0] - paste_x + xmin, p2[1] - paste_y + ymin]
                         p3 = [p3[0] - paste_x + xmin, p3[1] - paste_y + ymin]
@@ -266,30 +273,24 @@ class CustomPEKModel:
 
         # 表格识别 table recognition
         if self.apply_table:
-            pil_img = Image.fromarray(image)
-            for layout in layout_res:
-                if layout.get("category_id", -1) == 5:
-                    poly = layout["poly"]
-                    xmin, ymin = int(poly[0]), int(poly[1])
-                    xmax, ymax = int(poly[4]), int(poly[5])
-
-                    paste_x = 50
-                    paste_y = 50
-                    # 创建一个宽高各多50的白色背景 create a whiteboard with 50 larger width and length
-                    new_width = xmax - xmin + paste_x * 2
-                    new_height = ymax - ymin + paste_y * 2
-                    new_image = Image.new('RGB', (new_width, new_height), 'white')
-
-                    # 裁剪图像 crop image
-                    crop_box = (xmin, ymin, xmax, ymax)
-                    cropped_img = pil_img.crop(crop_box)
-                    new_image.paste(cropped_img, (paste_x, paste_y))
-                    start_time = time.time()
-                    logger.info("------------------table recognition processing begins-----------------")
+            table_start = time.time()
+            for res in table_res_list:
+                new_image, _ = crop_img(res, pil_img)
+                single_table_start_time = time.time()
+                logger.info("------------------table recognition processing begins-----------------")
+                with torch.no_grad():
                     latex_code = self.table_model.image2latex(new_image)[0]
-                    end_time = time.time()
-                    run_time = end_time - start_time
-                    logger.info(f"------------table recognition processing ends within {run_time}s-----")
-                    layout["latex"] = latex_code
+                run_time = time.time() - single_table_start_time
+                logger.info(f"------------table recognition processing ends within {run_time}s-----")
+                if run_time > self.table_max_time:
+                    logger.warning(f"------------table recognition processing exceeds max time {self.table_max_time}s----------")
+                # 判断是否返回正常
+                expected_ending = latex_code.strip().endswith('end{tabular}') or latex_code.strip().endswith('end{table}')
+                if latex_code and expected_ending:
+                    res["latex"] = latex_code
+                else:
+                    logger.warning(f"------------table recognition processing fails----------")
+            table_cost = round(time.time() - table_start, 2)
+            logger.info(f"table cost: {table_cost}")
 
         return layout_res

magic_pdf/tools/cli_dev.py

@@ -30,12 +30,10 @@ def read_s3_path(s3path):
         byte_start, byte_end = 0, None
     else:
         byte_start, byte_end = int(may_range_params[0]), int(may_range_params[1])
-        byte_end += byte_start - 1
-    return s3_rw.read_jsonl(
+    return s3_rw.read_offset(
         remove_non_official_s3_args(s3path),
         byte_start,
         byte_end,
-        AbsReaderWriter.MODE_BIN,
     )
 
 
@@ -71,24 +69,23 @@ def cli():
     default="",
 )
 def jsonl(jsonl, method, output_dir):
-    print("haha")
     model_config.__use_inside_model__ = False
-    full_jsonl_path = os.path.realpath(jsonl)
-    if output_dir == "":
-        output_dir = os.path.join(os.path.dirname(full_jsonl_path), "output")
-
     if jsonl.startswith("s3://"):
         jso = json_parse.loads(read_s3_path(jsonl).decode("utf-8"))
+        full_jsonl_path = "."
     else:
+        full_jsonl_path = os.path.realpath(jsonl)
         with open(jsonl) as f:
             jso = json_parse.loads(f.readline())
+
+    if output_dir == "":
+        output_dir = os.path.join(os.path.dirname(full_jsonl_path), "output")
     s3_file_path = jso.get("file_location")
     if s3_file_path is None:
         s3_file_path = jso.get("path")
     pdf_file_name = Path(s3_file_path).stem
     pdf_data = read_s3_path(s3_file_path)
 
-
     print(pdf_file_name, jso, method)
     do_parse(
         output_dir,
@@ -97,6 +94,7 @@ def jsonl(jsonl, method, output_dir):
         jso["doc_layout_result"],
         method,
         f_dump_content_list=True,
+        f_draw_model_bbox=True,
     )
 
 
@@ -149,6 +147,7 @@ def pdf(pdf, json_data, output_dir, method):
         model_json_list,
         method,
         f_dump_content_list=True,
+        f_draw_model_bbox=True,
     )
 
 

magic_pdf/tools/common.py

@@ -4,7 +4,7 @@ import copy
 import click
 from loguru import logger
 from magic_pdf.libs.MakeContentConfig import DropMode, MakeMode
-from magic_pdf.libs.draw_bbox import draw_layout_bbox, draw_span_bbox
+from magic_pdf.libs.draw_bbox import draw_layout_bbox, draw_span_bbox, drow_model_bbox
 from magic_pdf.pipe.UNIPipe import UNIPipe
 from magic_pdf.pipe.OCRPipe import OCRPipe
 from magic_pdf.pipe.TXTPipe import TXTPipe
@@ -37,6 +37,7 @@ def do_parse(
     f_dump_orig_pdf=True,
     f_dump_content_list=False,
     f_make_md_mode=MakeMode.MM_MD,
+    f_draw_model_bbox=False,
 ):
     orig_model_list = copy.deepcopy(model_list)
     local_image_dir, local_md_dir = prepare_env(output_dir, pdf_file_name, parse_method)
@@ -73,6 +74,8 @@ def do_parse(
         draw_layout_bbox(pdf_info, pdf_bytes, local_md_dir)
     if f_draw_span_bbox:
         draw_span_bbox(pdf_info, pdf_bytes, local_md_dir)
+    if f_draw_model_bbox:
+        drow_model_bbox(orig_model_list, pdf_bytes, local_md_dir)
 
     md_content = pipe.pipe_mk_markdown(
         image_dir, drop_mode=DropMode.NONE, md_make_mode=f_make_md_mode

magic_pdf-0.7.0b1.dist-info/METADATA

@@ -0,0 +1,421 @@
+Metadata-Version: 2.1
+Name: magic-pdf
+Version: 0.7.0b1
+Summary: A practical tool for converting PDF to Markdown
+Home-page: https://github.com/opendatalab/MinerU
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE.md
+Requires-Dist: boto3>=1.28.43
+Requires-Dist: Brotli>=1.1.0
+Requires-Dist: click>=8.1.7
+Requires-Dist: PyMuPDF>=1.24.9
+Requires-Dist: loguru>=0.6.0
+Requires-Dist: numpy<2.0.0,>=1.21.6
+Requires-Dist: fast-langdetect==0.2.0
+Requires-Dist: wordninja>=2.0.0
+Requires-Dist: scikit-learn>=1.0.2
+Requires-Dist: pdfminer.six==20231228
+Provides-Extra: full
+Requires-Dist: unimernet==0.1.6; extra == "full"
+Requires-Dist: ultralytics; extra == "full"
+Requires-Dist: paddleocr==2.7.3; extra == "full"
+Requires-Dist: pypandoc; extra == "full"
+Requires-Dist: struct-eqtable==0.1.0; extra == "full"
+Requires-Dist: detectron2; extra == "full"
+Requires-Dist: paddlepaddle==3.0.0b1; platform_system == "Linux" and extra == "full"
+Requires-Dist: matplotlib; (platform_system == "Linux" or platform_system == "Darwin") and extra == "full"
+Requires-Dist: matplotlib<=3.9.0; platform_system == "Windows" and extra == "full"
+Requires-Dist: paddlepaddle==2.6.1; (platform_system == "Windows" or platform_system == "Darwin") and extra == "full"
+Provides-Extra: lite
+Requires-Dist: paddleocr==2.7.3; extra == "lite"
+Requires-Dist: paddlepaddle==3.0.0b1; platform_system == "Linux" and extra == "lite"
+Requires-Dist: paddlepaddle==2.6.1; (platform_system == "Windows" or platform_system == "Darwin") and extra == "lite"
+
+<div align="center" xmlns="http://www.w3.org/1999/html">
+<!-- logo -->
+<p align="center">
+  <img src="docs/images/MinerU-logo.png" width="300px" style="vertical-align:middle;">
+</p>
+
+<!-- icon -->
+[![stars](https://img.shields.io/github/stars/opendatalab/MinerU.svg)](https://github.com/opendatalab/MinerU)
+[![forks](https://img.shields.io/github/forks/opendatalab/MinerU.svg)](https://github.com/opendatalab/MinerU)
+[![open issues](https://img.shields.io/github/issues-raw/opendatalab/MinerU)](https://github.com/opendatalab/MinerU/issues)
+[![issue resolution](https://img.shields.io/github/issues-closed-raw/opendatalab/MinerU)](https://github.com/opendatalab/MinerU/issues)
+[![PyPI version](https://badge.fury.io/py/magic-pdf.svg)](https://badge.fury.io/py/magic-pdf)
+[![Downloads](https://static.pepy.tech/badge/magic-pdf)](https://pepy.tech/project/magic-pdf)
+[![Downloads](https://static.pepy.tech/badge/magic-pdf/month)](https://pepy.tech/project/magic-pdf)
+<a href="https://trendshift.io/repositories/11174" target="_blank"><img src="https://trendshift.io/api/badge/repositories/11174" alt="opendatalab%2FMinerU | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
+
+<!-- language -->
+[English](README.md) | [简体中文](README_zh-CN.md)
+
+<!-- hot link -->
+<p align="center">
+<a href="https://github.com/opendatalab/PDF-Extract-Kit">PDF-Extract-Kit: High-Quality PDF Extraction Toolkit</a>🔥🔥🔥
+</p>
+
+<!-- join us -->
+<p align="center">
+    👋 join us on <a href="https://discord.gg/Tdedn9GTXq" target="_blank">Discord</a> and <a href="https://cdn.vansin.top/internlm/mineru.jpg" target="_blank">WeChat</a>
+</p>
+
+</div>
+
+# Changelog
+- 2024/08/09: Version 0.7.0b1 released, simplified installation process, added table recognition functionality
+- 2024/08/01: Version 0.6.2b1 released, optimized dependency conflict issues and installation documentation
+- 2024/07/05: Initial open-source release
+
+<!-- TABLE OF CONTENT -->
+<details open="open">
+  <summary><h2 style="display: inline-block">Table of Contents</h2></summary>
+  <ol>
+    <li>
+      <a href="#mineru">MinerU</a>
+      <ul>
+        <li><a href="#project-introduction">Project Introduction</a></li>
+        <li><a href="#key-features">Key Features</a></li>
+        <li><a href="#quick-start">Quick Start</a>
+            <ul>
+            <li><a href="#online-demo">Online Demo</a></li>
+            <li><a href="#quick-cpu-demo">Quick CPU Demo</a></li>
+            <li><a href="#using-gpu">Using GPU</a></li>
+            </ul>
+        </li>
+        <li><a href="#usage">Usage</a>
+            <ul>
+            <li><a href="#command-line">Command Line</a></li>
+            <li><a href="#api">API</a></li>
+            <li><a href="#development-guide">Development Guide</a></li>
+            </ul>
+        </li>
+      </ul>
+    </li>
+    <li><a href="#todo">TODO</a></li>
+    <li><a href="#known-issues">Known Issues</a></li>
+    <li><a href="#faq">FAQ</a></li>
+    <li><a href="#all-thanks-to-our-contributors">All Thanks To Our Contributors</a></li>
+    <li><a href="#license-information">License Information</a></li>
+    <li><a href="#acknowledgments">Acknowledgments</a></li>
+    <li><a href="#citation">Citation</a></li>
+    <li><a href="#star-history">Star History</a></li>
+    <li><a href="#magic-doc">Magic-doc</a></li>
+    <li><a href="#magic-html">Magic-html</a></li>
+    <li><a href="#links">Links</a></li>
+  </ol>
+</details>
+
+
+
+# MinerU
+## Project Introduction
+MinerU is a tool that converts PDFs into machine-readable formats (e.g., markdown, JSON), allowing for easy extraction into any format.
+MinerU was born during the pre-training process of [InternLM](https://github.com/InternLM/InternLM). We focus on solving symbol conversion issues in scientific literature and hope to contribute to technological development in the era of large models.
+Compared to well-known commercial products, MinerU is still young. If you encounter any issues or if the results are not as expected, please submit an issue on [issue](https://github.com/opendatalab/MinerU/issues) and **attach the relevant PDF**.
+
+https://github.com/user-attachments/assets/4bea02c9-6d54-4cd6-97ed-dff14340982c
+
+## Key Features
+
+- Removes elements such as headers, footers, footnotes, and page numbers while maintaining semantic continuity
+- Outputs text in a human-readable order from multi-column documents
+- Retains the original structure of the document, including titles, paragraphs, and lists
+- Extracts images, image captions, tables, and table captions
+- Automatically recognizes formulas in the document and converts them to LaTeX
+- Automatically recognizes tables in the document and converts them to LaTeX
+- Automatically detects and enables OCR for corrupted PDFs
+- Supports both CPU and GPU environments
+- Supports Windows, Linux, and Mac platforms
+
+## Quick Start
+
+If you encounter any installation issues, please first consult the <a href="#faq">FAQ</a>. </br>
+If the parsing results are not as expected, refer to the <a href="#known-issues">Known Issues</a>. </br>
+There are three different ways to experience MinerU:
+- [Online Demo (No Installation Required)](#online-demo)
+- [Quick CPU Demo (Windows, Linux, Mac)](#quick-cpu-demo)
+- [Linux/Windows + CUDA](#Using-GPU)
+
+**⚠️ Pre-installation Notice—Hardware and Software Environment Support**
+
+To ensure the stability and reliability of the project, we only optimize and test for specific hardware and software environments during development. This ensures that users deploying and running the project on recommended system configurations will get the best performance with the fewest compatibility issues.
+
+By focusing resources on the mainline environment, our team can more efficiently resolve potential bugs and develop new features.
+
+In non-mainline environments, due to the diversity of hardware and software configurations, as well as third-party dependency compatibility issues, we cannot guarantee 100% project availability. Therefore, for users who wish to use this project in non-recommended environments, we suggest carefully reading the documentation and FAQ first. Most issues already have corresponding solutions in the FAQ. We also encourage community feedback to help us gradually expand support.
+
+<table>
+    <tr>
+        <td colspan="3" rowspan="2">Operating System</td>
+    </tr>
+    <tr>
+        <td>Ubuntu 22.04 LTS</td>
+        <td>Windows 10 / 11</td>
+        <td>macOS 11+</td>
+    </tr>
+    <tr>
+        <td colspan="3">CPU</td>
+        <td>x86_64</td>
+        <td>x86_64</td>
+        <td>x86_64 / arm64</td>
+    </tr>
+    <tr>
+        <td colspan="3">Memory</td>
+        <td colspan="3">16GB or more, recommended 32GB+</td>
+    </tr>
+    <tr>
+        <td colspan="3">Python Version</td>
+        <td colspan="3">3.10</td>
+    </tr>
+    <tr>
+        <td colspan="3">Nvidia Driver Version</td>
+        <td>latest (Proprietary Driver)</td>
+        <td>latest</td>
+        <td>None</td>
+    </tr>
+    <tr>
+        <td colspan="3">CUDA Environment</td>
+        <td>Automatic installation [12.1 (pytorch) + 11.8 (paddle)]</td>
+        <td>11.8 (manual installation) + cuDNN v8.7.0 (manual installation)</td>
+        <td>None</td>
+    </tr>
+    <tr>
+        <td rowspan="2">GPU Hardware Support List</td>
+        <td colspan="2">Minimum Requirement 8G+ VRAM</td>
+        <td colspan="2">3060ti/3070/3080/3080ti/4060/4070/4070ti<br>
+        8G VRAM only enables layout and formula recognition acceleration</td>
+        <td rowspan="2">None</td>
+    </tr>
+    <tr>
+        <td colspan="2">Recommended Configuration 16G+ VRAM</td>
+        <td colspan="2">3090/3090ti/4070ti super/4080/4090<br>
+        16G or more can enable layout, formula recognition, and OCR acceleration simultaneously</td>
+    </tr>
+</table>
+
+### Online Demo
+
+[Click here for the online demo](https://opendatalab.com/OpenSourceTools/Extractor/PDF)
+
+### Quick CPU Demo
+
+#### 1. Install magic-pdf
+```bash
+conda create -n MinerU python=3.10
+conda activate MinerU
+pip install magic-pdf[full]==0.7.0b1 --extra-index-url https://wheels.myhloli.com
+```
+#### 2. Download model weight files
+
+Refer to [How to Download Model Files](docs/how_to_download_models_en.md) for detailed instructions.
+> ❗️After downloading the models, please make sure to verify the completeness of the model files.
+> 
+> Check if the model file sizes match the description on the webpage. If possible, use sha256 to verify the integrity of the files.
+
+#### 3. Copy and configure the template file
+You can find the `magic-pdf.template.json` template configuration file in the root directory of the repository.
+> ❗️Make sure to execute the following command to copy the configuration file to your **user directory**; otherwise, the program will not run.
+> 
+> The user directory for Windows is `C:\Users\YourUsername`, for Linux it is `/home/YourUsername`, and for macOS it is `/Users/YourUsername`.
+```bash
+cp magic-pdf.template.json ~/magic-pdf.json
+```
+
+Find the `magic-pdf.json` file in your user directory and configure the "models-dir" path to point to the directory where the model weight files were downloaded in [Step 2](#2-download-model-weight-files).
+> ❗️Make sure to correctly configure the **absolute path** to the model weight files directory, otherwise the program will not run because it can't find the model files.
+>
+> On Windows, this path should include the drive letter and all backslashes (`\`) in the path should be replaced with forward slashes (`/`) to avoid syntax errors in the JSON file due to escape sequences.
+> 
+> For example: If the models are stored in the "models" directory at the root of the D drive, the "model-dir" value should be `D:/models`.
+```json
+{
+  // other config
+  "models-dir": "D:/models",
+  "table-config": {
+        "is_table_recog_enable": false, // Table recognition is disabled by default, modify this value to enable it
+        "max_time": 400
+    }
+}
+```
+
+
+### Using GPU
+If your device supports CUDA and meets the GPU requirements of the mainline environment, you can use GPU acceleration. Please select the appropriate guide based on your system:
+
+- [Ubuntu 22.04 LTS + GPU](docs/README_Ubuntu_CUDA_Acceleration_en_US.md)
+- [Windows 10/11 + GPU](docs/README_Windows_CUDA_Acceleration_en_US.md)
+
+
+## Usage
+
+### Command Line
+
+```bash
+magic-pdf --help
+Usage: magic-pdf [OPTIONS]
+
+Options:
+  -v, --version                display the version and exit
+  -p, --path PATH              local pdf filepath or directory  [required]
+  -o, --output-dir TEXT        output local directory
+  -m, --method [ocr|txt|auto]  the method for parsing pdf.  
+                               ocr: using ocr technique to extract information from pdf,
+                               txt: suitable for the text-based pdf only and outperform ocr,
+                               auto: automatically choose the best method for parsing pdf
+                                  from ocr and txt.
+                               without method specified, auto will be used by default. 
+  --help                       Show this message and exit.
+
+
+## show version
+magic-pdf -v
+
+## command line example
+magic-pdf -p {some_pdf} -o {some_output_dir} -m auto
+```
+
+`{some_pdf}` can be a single PDF file or a directory containing multiple PDFs.
+The results will be saved in the `{some_output_dir}` directory. The output file list is as follows:
+
+```text
+├── some_pdf.md                 # markdown file
+├── images                      # directory for storing images
+├── layout.pdf                  # layout diagram
+├── middle.json                 # MinerU intermediate processing result
+├── model.json                  # model inference result
+├── origin.pdf                  # original PDF file
+└── spans.pdf                   # smallest granularity bbox position information diagram
+```
+
+For more information about the output files, please refer to the [Output File Description](docs/output_file_en_us.md).
+
+### API
+
+Processing files from local disk
+```python
+image_writer = DiskReaderWriter(local_image_dir)
+image_dir = str(os.path.basename(local_image_dir))
+jso_useful_key = {"_pdf_type": "", "model_list": []}
+pipe = UNIPipe(pdf_bytes, jso_useful_key, image_writer)
+pipe.pipe_classify()
+pipe.pipe_analyze()
+pipe.pipe_parse()
+md_content = pipe.pipe_mk_markdown(image_dir, drop_mode="none")
+```
+
+Processing files from object storage
+```python
+s3pdf_cli = S3ReaderWriter(pdf_ak, pdf_sk, pdf_endpoint)
+image_dir = "s3://img_bucket/"
+s3image_cli = S3ReaderWriter(img_ak, img_sk, img_endpoint, parent_path=image_dir)
+pdf_bytes = s3pdf_cli.read(s3_pdf_path, mode=s3pdf_cli.MODE_BIN)
+jso_useful_key = {"_pdf_type": "", "model_list": []}
+pipe = UNIPipe(pdf_bytes, jso_useful_key, s3image_cli)
+pipe.pipe_classify()
+pipe.pipe_analyze()
+pipe.pipe_parse()
+md_content = pipe.pipe_mk_markdown(image_dir, drop_mode="none")
+```
+
+For detailed implementation, refer to:
+- [demo.py Simplest Processing Method](demo/demo.py)
+- [magic_pdf_parse_main.py More Detailed Processing Workflow](demo/magic_pdf_parse_main.py)
+
+
+### Development Guide
+
+TODO
+
+# TODO
+
+- [ ] Semantic-based reading order
+- [ ] List recognition within the text
+- [ ] Code block recognition within the text
+- [ ] Table of contents recognition
+- [x] Table recognition
+- [ ] Chemical formula recognition
+- [ ] Geometric shape recognition
+
+# Known Issues
+- Reading order is segmented based on rules, which can cause disordered sequences in some cases
+- Vertical text is not supported
+- Lists, code blocks, and table of contents are not yet supported in the layout model
+- Comic books, art books, elementary school textbooks, and exercise books are not well-parsed yet
+- Enabling OCR may produce better results in PDFs with a high density of formulas
+- If you are processing PDFs with a large number of formulas, it is strongly recommended to enable the OCR function. When using PyMuPDF to extract text, overlapping text lines can occur, leading to inaccurate formula insertion positions.
+- **Table Recognition** is currently in the testing phase; recognition speed is slow, and accuracy needs improvement. Below are some performance test results in an Ubuntu 22.04 LTS + Intel(R) Xeon(R) Platinum 8352V CPU @ 2.10GHz + NVIDIA GeForce RTX 4090 environment for reference.
+
+| Table Size     | Parsing Time        | 
+|---------------|----------------------------| 
+| 6\*5 55kb     | 37s                   | 
+| 16\*12 284kb  | 3m18s                 | 
+| 44\*7 559kb   | 4m12s                 | 
+
+# FAQ
+[FAQ in Chinese](docs/FAQ_zh_cn.md)
+
+[FAQ in English](docs/FAQ_en_us.md)
+
+
+# All Thanks To Our Contributors
+
+<a href="https://github.com/opendatalab/MinerU/graphs/contributors">
+  <img src="https://contrib.rocks/image?repo=opendatalab/MinerU" />
+</a>
+
+# License Information
+
+[LICENSE.md](LICENSE.md)
+
+This project currently uses PyMuPDF to achieve advanced functionality. However, since it adheres to the AGPL license, it may impose restrictions on certain usage scenarios. In future iterations, we plan to explore and replace it with a more permissive PDF processing library to enhance user-friendliness and flexibility.
+
+
+# Acknowledgments
+- [PDF-Extract-Kit](https://github.com/opendatalab/PDF-Extract-Kit)
+- [StructEqTable](https://github.com/UniModal4Reasoning/StructEqTable-Deploy)
+- [PaddleOCR](https://github.com/PaddlePaddle/PaddleOCR)
+- [PyMuPDF](https://github.com/pymupdf/PyMuPDF)
+- [fast-langdetect](https://github.com/LlmKira/fast-langdetect)
+- [pdfminer.six](https://github.com/pdfminer/pdfminer.six)
+
+# Citation
+
+```bibtex
+@article{he2024opendatalab,
+  title={Opendatalab: Empowering general artificial intelligence with open datasets},
+  author={He, Conghui and Li, Wei and Jin, Zhenjiang and Xu, Chao and Wang, Bin and Lin, Dahua},
+  journal={arXiv preprint arXiv:2407.13773},
+  year={2024}
+}
+
+@misc{2024mineru,
+    title={MinerU: A One-stop, Open-source, High-quality Data Extraction Tool},
+    author={MinerU Contributors},
+    howpublished = {\url{https://github.com/opendatalab/MinerU}},
+    year={2024}
+}
+```
+
+# Star History
+
+<a>
+ <picture>
+   <source media="(prefers-color-scheme: dark)" srcset="https://api.star-history.com/svg?repos=opendatalab/MinerU&type=Date&theme=dark" />
+   <source media="(prefers-color-scheme: light)" srcset="https://api.star-history.com/svg?repos=opendatalab/MinerU&type=Date" />
+   <img alt="Star History Chart" src="https://api.star-history.com/svg?repos=opendatalab/MinerU&type=Date" />
+ </picture>
+</a>
+
+# Magic-doc
+[Magic-Doc](https://github.com/InternLM/magic-doc) Fast speed ppt/pptx/doc/docx/pdf extraction tool
+
+# Magic-html
+[Magic-HTML](https://github.com/opendatalab/magic-html) Mixed web page extraction tool
+
+# Links
+
+- [LabelU (A Lightweight Multi-modal Data Annotation Tool)](https://github.com/opendatalab/labelU)
+- [LabelLLM (An Open-source LLM Dialogue Annotation Platform)](https://github.com/opendatalab/LabelLLM)
+- [PDF-Extract-Kit (A Comprehensive Toolkit for High-Quality PDF Content Extraction)](https://github.com/opendatalab/PDF-Extract-Kit)

{magic_pdf-0.7.0a1.dist-info → magic_pdf-0.7.0b1.dist-info}/RECORD

@@ -15,7 +15,7 @@ magic_pdf/layout/layout_det_utils.py,sha256=NCYBTvsrULE3Cue53aMD1MfXTmOL9Xy0nivl
 magic_pdf/layout/layout_sort.py,sha256=ovqRX1xcRA7E7s8VvsI7ZNbaNSElJe07bApCh5hxwIE,33533
 magic_pdf/layout/layout_spiler_recog.py,sha256=QjBSgB-a7J2yjUR1eaCs9ZD7URtiRnV6W934hpAeuC4,3067
 magic_pdf/layout/mcol_sort.py,sha256=ADnLisBJBHXDKYChcf2lzTb_TC_vZ4q89_CSN8mwEJc,11331
-magic_pdf/libs/Constants.py,sha256=AwQw5aK7JkWjerEyq5vxxMTHH1Gvku8K9NS8xjHKimI,189
+magic_pdf/libs/Constants.py,sha256=aKdTHeK75qkVvxvE_2EA5LYis6Z6HLmiuk9o8ESOnNg,260
 magic_pdf/libs/MakeContentConfig.py,sha256=UDZPpsv8q4DqTy8h0vRtrT2kHqWiVI205VnVhlUEQc0,206
 magic_pdf/libs/ModelBlockTypeEnum.py,sha256=kalXPbo5ya6hKhhBHPGlHl1yjWOURoXZWQM3rVUyPsY,164
 magic_pdf/libs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -26,7 +26,7 @@ magic_pdf/libs/config_reader.py,sha256=dPx6JJJuCw9AzNgKtrTG1elmfdeN6gDhgFK9r15-N
 magic_pdf/libs/convert_utils.py,sha256=Ov-lsfCLBPz_15iSJXIslBNmrSf_E_1g_XDWJy8NgO8,143
 magic_pdf/libs/coordinate_transform.py,sha256=Bbop2cP2uz2ZG0U0gwd7J6EKkgABq5Rv03qf2LMPw80,429
 magic_pdf/libs/detect_language_from_model.py,sha256=Uln8F9qs8EJOw4EgI7KRlaU3lD_mK8KMTlADLFtz8fk,816
-magic_pdf/libs/draw_bbox.py,sha256=90FDAYN3dxgN07_xRzdUgnDAyEswpl9VCXaDo_SMZkA,9449
+magic_pdf/libs/draw_bbox.py,sha256=wPoaxXteZDCL20pjVEQ4kcu0u6Ea-9balUaxrkMsYks,12531
 magic_pdf/libs/drop_reason.py,sha256=IfjPSrPLMmVziqjOXPep7r_ioQKFRahDgbOW1SD-Tuw,2148
 magic_pdf/libs/drop_tag.py,sha256=bZDg3bIVWvBT1Ec1icwj5WLOkt5-hI6eRYZ2tX9_a74,673
 magic_pdf/libs/hash_utils.py,sha256=VEKK9WfFoZgrPfi8kfITjLpr8Ahufs8tXh9R1Y5lAL8,404
@@ -35,19 +35,19 @@ magic_pdf/libs/language.py,sha256=Hj5-lrGoNExxdHLbkcNG-c27U4AjJ9AZPdZblaNSehU,10
 magic_pdf/libs/local_math.py,sha256=tqljQOgqh3fZc146HYhO88JXJaiXMVwArBkk_CSGICc,177
 magic_pdf/libs/markdown_utils.py,sha256=cLxLXjRhrNp_wCHvtglrGA_FVdrvfd1KULeTtj1p18w,944
 magic_pdf/libs/nlp_utils.py,sha256=-X9W3-Ns5ZdDYFvyyEq6i6P2b5hCATaFEZeOjwNOH9M,6901
-magic_pdf/libs/ocr_content_type.py,sha256=DiGTYppd6WlibwCAeVpIy3NHCQkglfIAQsJ_ffu5BPw,526
+magic_pdf/libs/ocr_content_type.py,sha256=9c12CoJ8xvdFa4Rk81J_S238yuQl0bDQeapvqb-JkEk,794
 magic_pdf/libs/path_utils.py,sha256=Hykw_l5CU736b2egHV9P7B-qh3QNKO4nZSGCbsi0Z8E,1043
 magic_pdf/libs/pdf_check.py,sha256=MAe8wzwT0qvPf_I72wEZG7k1g4haNHS7oUtLqkB5rlE,2145
 magic_pdf/libs/pdf_image_tools.py,sha256=CAd01giTKr_UJz1_QtDOARG9G9z69GFpzRZwcWSfLtE,1282
 magic_pdf/libs/safe_filename.py,sha256=ckwcM_eqoysTb5id8czp-tXq2G9da0-l3pshZDCHQtE,236
 magic_pdf/libs/textbase.py,sha256=SC1Frhz3Fb7V7n2SFRBsl7Bmg0JZdlvZskq0lfW1vIk,732
-magic_pdf/libs/version.py,sha256=iEqOsQ5JUsdTQDAPqryKypkJMTOXXBl71cd4Drh5pDs,24
+magic_pdf/libs/version.py,sha256=95eHzU5LYX2l3ASu7OvUb95xo-2kfuwh1uUYnY54K90,24
 magic_pdf/libs/vis_utils.py,sha256=hTOTEakKV0pGMbk0tbRkVI_tku7A3dGc96ynObZ4kwI,10207
 magic_pdf/model/__init__.py,sha256=1QcfMKET0xQhSaZMjNQHi_TjzSSDR6PI5mjkmaXHPe8,52
-magic_pdf/model/doc_analyze_by_custom_model.py,sha256=cSmh27RG1cvY0pd98T82rq0pANBwkYN0siZIN6oPNI8,4332
+magic_pdf/model/doc_analyze_by_custom_model.py,sha256=SoT21VHy6ICXoRfC9V3XS6BMiX8EZI6zaqSNgoE17oo,4347
 magic_pdf/model/magic_model.py,sha256=xwKV9BrdjOJecJSzbErT54N6qeJu0tvFuJg2S1z_2kU,25413
 magic_pdf/model/model_list.py,sha256=AqxAtKGLDn7VVXWYwk0l9LnACxDLyU2jwOJ7vjPZj04,72
-magic_pdf/model/pdf_extract_kit.py,sha256=I3pZBWQu8y5YVjURTUJnsIySjeSGK-Yefit9PiqN9VE,12952
+magic_pdf/model/pdf_extract_kit.py,sha256=21vBy8p6pI5a0b6V45ul52yE8zD1R0xrjv4Tx8r9gaw,13620
 magic_pdf/model/pp_structure_v2.py,sha256=1sn8IJK0d5ZmqJ2XFt9FdaSdI0RQf-iwNAWBrVrIeuc,2872
 magic_pdf/model/pek_sub_modules/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 magic_pdf/model/pek_sub_modules/post_process.py,sha256=HzRxV2sVR3Qo8XKYEHhT6tae-bYTb6dnAfGP6gfVNaM,1135
@@ -138,11 +138,11 @@ magic_pdf/spark/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 magic_pdf/spark/spark_api.py,sha256=eSLXTjMYW5Ya41VMIApRVfji1ZxEZXdH9ZdsL6fy5Kw,1131
 magic_pdf/tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 magic_pdf/tools/cli.py,sha256=aVmurGAEyWT-MOv0MOaCRrfef1-jkRTpeVVWUsEVyeY,2157
-magic_pdf/tools/cli_dev.py,sha256=uDc4fDxVuOIrkaKRdjNAqyh9htyLd-fYDEfJBNFUYao,4149
-magic_pdf/tools/common.py,sha256=x4W-Tyo0A-TGsOjzlUGAhxiU2AisU3nBE3_2H_RLUO4,3801
-magic_pdf-0.7.0a1.dist-info/LICENSE.md,sha256=hIahDEOTzuHCU5J2nd07LWwkLW7Hko4UFO__ffsvB-8,34523
-magic_pdf-0.7.0a1.dist-info/METADATA,sha256=NBLsixinI-5iHwdweKr13SM5qg6Jf-fWCwg5ihavlpY,12455
-magic_pdf-0.7.0a1.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-magic_pdf-0.7.0a1.dist-info/entry_points.txt,sha256=wXwYke3j8fqDQTocUspL-CqDUEv3Tfcwp09fM8dZAhA,98
-magic_pdf-0.7.0a1.dist-info/top_level.txt,sha256=J9I0AzmHWGkp9c6DL8Oe4mEx3yYphLzkRn4H25Lg1rE,10
-magic_pdf-0.7.0a1.dist-info/RECORD,,
+magic_pdf/tools/cli_dev.py,sha256=w-J4OixDzHjknnUuRW44PXsUlUqyiD4nPbBSSk9WkXM,4160
+magic_pdf/tools/common.py,sha256=XoSs19DD-4ubbjrDFQer83T9O6O_MmgEO61NbjlP_2M,3939
+magic_pdf-0.7.0b1.dist-info/LICENSE.md,sha256=hIahDEOTzuHCU5J2nd07LWwkLW7Hko4UFO__ffsvB-8,34523
+magic_pdf-0.7.0b1.dist-info/METADATA,sha256=47QGAd2iGc0i1osA_jbBS1QT_Jrfmofoyetsrh9KRy8,18571
+magic_pdf-0.7.0b1.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+magic_pdf-0.7.0b1.dist-info/entry_points.txt,sha256=wXwYke3j8fqDQTocUspL-CqDUEv3Tfcwp09fM8dZAhA,98
+magic_pdf-0.7.0b1.dist-info/top_level.txt,sha256=J9I0AzmHWGkp9c6DL8Oe4mEx3yYphLzkRn4H25Lg1rE,10
+magic_pdf-0.7.0b1.dist-info/RECORD,,

magic_pdf-0.7.0a1.dist-info/METADATA

@@ -1,362 +0,0 @@
-Metadata-Version: 2.1
-Name: magic-pdf
-Version: 0.7.0a1
-Summary: A practical tool for converting PDF to Markdown
-Home-page: https://github.com/opendatalab/MinerU
-Requires-Python: >=3.9
-Description-Content-Type: text/markdown
-License-File: LICENSE.md
-Requires-Dist: boto3>=1.28.43
-Requires-Dist: Brotli>=1.1.0
-Requires-Dist: click>=8.1.7
-Requires-Dist: PyMuPDF>=1.24.9
-Requires-Dist: loguru>=0.6.0
-Requires-Dist: numpy<2.0.0,>=1.21.6
-Requires-Dist: fast-langdetect==0.2.0
-Requires-Dist: wordninja>=2.0.0
-Requires-Dist: scikit-learn>=1.0.2
-Requires-Dist: pdfminer.six==20231228
-Provides-Extra: full
-Requires-Dist: unimernet==0.1.6; extra == "full"
-Requires-Dist: ultralytics; extra == "full"
-Requires-Dist: paddleocr==2.7.3; extra == "full"
-Requires-Dist: pypandoc; extra == "full"
-Requires-Dist: struct-eqtable==0.1.0; extra == "full"
-Requires-Dist: detectron2; extra == "full"
-Requires-Dist: paddlepaddle==3.0.0b1; platform_system == "Linux" and extra == "full"
-Requires-Dist: matplotlib; (platform_system == "Linux" or platform_system == "Darwin") and extra == "full"
-Requires-Dist: matplotlib<=3.9.0; platform_system == "Windows" and extra == "full"
-Requires-Dist: paddlepaddle==2.6.1; (platform_system == "Windows" or platform_system == "Darwin") and extra == "full"
-Provides-Extra: lite
-Requires-Dist: paddleocr==2.7.3; extra == "lite"
-Requires-Dist: paddlepaddle==3.0.0b1; platform_system == "Linux" and extra == "lite"
-Requires-Dist: paddlepaddle==2.6.1; (platform_system == "Windows" or platform_system == "Darwin") and extra == "lite"
-
-<div id="top">
-
-<p align="center">
-  <img src="docs/images/MinerU-logo.png" width="300px" style="vertical-align:middle;">
-</p>
-
-</div>
-<div align="center">
-
-[![stars](https://img.shields.io/github/stars/opendatalab/MinerU.svg)](https://github.com/opendatalab/MinerU)
-[![forks](https://img.shields.io/github/forks/opendatalab/MinerU.svg)](https://github.com/opendatalab/MinerU)
-[![open issues](https://img.shields.io/github/issues-raw/opendatalab/MinerU)](https://github.com/opendatalab/MinerU/issues)
-[![issue resolution](https://img.shields.io/github/issues-closed-raw/opendatalab/MinerU)](https://github.com/opendatalab/MinerU/issues)
-[![PyPI version](https://badge.fury.io/py/magic-pdf.svg)](https://badge.fury.io/py/magic-pdf)
-[![Downloads](https://static.pepy.tech/badge/magic-pdf)](https://pepy.tech/project/magic-pdf)
-[![Downloads](https://static.pepy.tech/badge/magic-pdf/month)](https://pepy.tech/project/magic-pdf)
-
-<a href="https://trendshift.io/repositories/11174" target="_blank"><img src="https://trendshift.io/api/badge/repositories/11174" alt="opendatalab%2FMinerU | Trendshift" style="width: 200px; height: 55px;"/></a>
-
-
-
-
-[English](README.md) | [简体中文](README_zh-CN.md) | [日本語](README_ja-JP.md)
-
-</div>
-
-<div align="center">
-<p align="center">
-<a href="https://github.com/opendatalab/MinerU">MinerU: An end-to-end PDF parsing tool based on PDF-Extract-Kit, supporting conversion from PDF to Markdown.</a>🚀🚀🚀<br>
-<a href="https://github.com/opendatalab/PDF-Extract-Kit">PDF-Extract-Kit: A Comprehensive Toolkit for High-Quality PDF Content Extraction</a>🔥🔥🔥
-</p>
-
-<p align="center">
-    👋 join us on <a href="https://discord.gg/gPxmVeGC" target="_blank">Discord</a> and <a href="https://cdn.vansin.top/internlm/mineru.jpg" target="_blank">WeChat</a>
-</p>
-</div>
-
-# MinerU 
-
-
-## Introduction
-
-MinerU is a one-stop, open-source, high-quality data extraction tool, includes the following primary features:
-
-- [Magic-PDF](#Magic-PDF)  PDF Document Extraction  
-- [Magic-Doc](#Magic-Doc)  Webpage & E-book Extraction
-
-
-# Magic-PDF
-
-
-## Introduction
-
-Magic-PDF is a tool designed to convert PDF documents into Markdown format, capable of processing files stored locally or on object storage supporting S3 protocol.
-
-Key features include:
-
-- Support for multiple front-end model inputs
-- Removal of headers, footers, footnotes, and page numbers
-- Human-readable layout formatting
-- Retains the original document's structure and formatting, including headings, paragraphs, lists, and more
-- Extraction and display of images and tables within markdown
-- Conversion of equations into LaTeX format
-- Automatic detection and conversion of garbled PDFs
-- Compatibility with CPU and GPU environments
-- Available for Windows, Linux, and macOS platforms
-
-
-https://github.com/user-attachments/assets/4bea02c9-6d54-4cd6-97ed-dff14340982c
-
-
-
-## Project Panorama
-
-![Project Panorama](docs/images/project_panorama_en.png)
-
-
-## Flowchart
-
-![Flowchart](docs/images/flowchart_en.png)
-
-### Dependency repositorys
-
-- [PDF-Extract-Kit : A Comprehensive Toolkit for High-Quality PDF Content Extraction](https://github.com/opendatalab/PDF-Extract-Kit) 🚀🚀🚀
-
-## Getting Started
-
-### Requirements
-
-- Python >= 3.9
-
-Using a virtual environment is recommended to avoid potential dependency conflicts; both venv and conda are suitable. 
-For example:
-```bash
-conda create -n MinerU python=3.10
-conda activate MinerU
-```
-
-### Installation and Configuration
-
-#### 1. Install Magic-PDF
-
-**1.Install dependencies**
-
-The full-feature package depends on detectron2, which requires a compilation installation.   
-If you need to compile it yourself, please refer to https://github.com/facebookresearch/detectron2/issues/5114  
-Alternatively, you can directly use our precompiled whl package (limited to Python 3.10):
-
-```bash
-pip install detectron2 --extra-index-url https://wheels.myhloli.com
-```
-
-**2.Install the full-feature package with pip**
->Note: The pip-installed package supports CPU-only and is ideal for quick tests.
->
->For CUDA/MPS acceleration in production, see [Acceleration Using CUDA or MPS](#4-Acceleration-Using-CUDA-or-MPS).
-
-```bash
-pip install magic-pdf[full]==0.6.2b1
-```
-> ❗️❗️❗️
-> We have pre-released the 0.6.2 beta version, addressing numerous issues mentioned in our logs. However, this build has not undergone full QA testing and does not represent the final release quality. Should you encounter any problems, please promptly report them to us via issues or revert to using version 0.6.1.
-> ```bash
-> pip install magic-pdf[full-cpu]==0.6.1
-> ```
-
-
-
-#### 2. Downloading model weights files
-
-For detailed references, please see below [how_to_download_models](docs/how_to_download_models_en.md)
-
-After downloading the model weights, move the 'models' directory to a directory on a larger disk space, preferably an SSD.
-
-
-#### 3. Copy the Configuration File and Make Configurations
-You can get the [magic-pdf.template.json](magic-pdf.template.json) file in the repository root directory.
-```bash
-cp magic-pdf.template.json ~/magic-pdf.json
-```
-In magic-pdf.json, configure "models-dir" to point to the directory where the model weights files are located.
-
-```json
-{
-  "models-dir": "/tmp/models"
-}
-```
-
-
-#### 4. Acceleration Using CUDA or MPS
-If you have an available Nvidia GPU or are using a Mac with Apple Silicon, you can leverage acceleration with CUDA or MPS respectively.
-##### CUDA
-
-You need to install the corresponding PyTorch version according to your CUDA version.  
-This example installs the CUDA 11.8 version.More information https://pytorch.org/get-started/locally/
-```bash
-pip install --force-reinstall torch==2.3.1 torchvision==0.18.1 --index-url https://download.pytorch.org/whl/cu118
-```
-> ❗ ️Make sure to specify version
-> ```bash
-> torch==2.3.1 torchvision==0.18.1
-> ```
->  in the command, as these are the highest versions we support. Failing to specify the versions may result in automatically installing higher versions which can cause the program to fail.
-
-Also, you need to modify the value of "device-mode" in the configuration file magic-pdf.json.  
-```json
-{
-  "device-mode":"cuda"
-}
-```
-
-##### MPS
-
-For macOS users with M-series chip devices, you can use MPS for inference acceleration.  
-You also need to modify the value of "device-mode" in the configuration file magic-pdf.json.  
-```json
-{
-  "device-mode":"mps"
-}
-```
-
-
-### Usage
-
-#### 1.Usage via Command Line
-
-###### simple
-
-```bash
-magic-pdf pdf-command --pdf "pdf_path" --inside_model true
-```
-After the program has finished, you can find the generated markdown files under the directory "/tmp/magic-pdf".  
-You can find the corresponding xxx_model.json file in the markdown directory.   
-If you intend to do secondary development on the post-processing pipeline, you can use the command:  
-```bash
-magic-pdf pdf-command --pdf "pdf_path" --model "model_json_path"
-```
-In this way, you won't need to re-run the model data, making debugging more convenient.
-
-
-###### more 
-
-```bash
-magic-pdf --help
-```
-
-
-#### 2. Usage via Api
-
-###### Local
-```python
-image_writer = DiskReaderWriter(local_image_dir)
-image_dir = str(os.path.basename(local_image_dir))
-jso_useful_key = {"_pdf_type": "", "model_list": []}
-pipe = UNIPipe(pdf_bytes, jso_useful_key, image_writer)
-pipe.pipe_classify()
-pipe.pipe_parse()
-md_content = pipe.pipe_mk_markdown(image_dir, drop_mode="none")
-```
-
-###### Object Storage
-```python
-s3pdf_cli = S3ReaderWriter(pdf_ak, pdf_sk, pdf_endpoint)
-image_dir = "s3://img_bucket/"
-s3image_cli = S3ReaderWriter(img_ak, img_sk, img_endpoint, parent_path=image_dir)
-pdf_bytes = s3pdf_cli.read(s3_pdf_path, mode=s3pdf_cli.MODE_BIN)
-jso_useful_key = {"_pdf_type": "", "model_list": []}
-pipe = UNIPipe(pdf_bytes, jso_useful_key, s3image_cli)
-pipe.pipe_classify()
-pipe.pipe_parse()
-md_content = pipe.pipe_mk_markdown(image_dir, drop_mode="none")
-```
-
-Demo can be referred to [demo.py](demo/demo.py)
-
-
-# Magic-Doc
-
-
-## Introduction
-
-Magic-Doc is a tool designed to convert web pages or multi-format e-books into markdown format.
-
-Key Features Include:
-
-- Web Page Extraction
-  - Cross-modal precise parsing of text, images, tables, and formula information.
-
-- E-Book Document Extraction
-  - Supports various document formats including epub, mobi, with full adaptation for text and images.
-
-- Language Type Identification
-  - Accurate recognition of 176 languages.
-
-https://github.com/opendatalab/MinerU/assets/11393164/a5a650e9-f4c0-463e-acc3-960967f1a1ca
-
-
-
-https://github.com/opendatalab/MinerU/assets/11393164/0f4a6fe9-6cca-4113-9fdc-a537749d764d
-
-
-
-https://github.com/opendatalab/MinerU/assets/11393164/20438a02-ce6c-4af8-9dde-d722a4e825b2
-
-
-
-
-## Project Repository
-
-- [Magic-Doc](https://github.com/InternLM/magic-doc)
-  Outstanding Webpage and E-book Extraction Tool
-
-
-# All Thanks To Our Contributors
-
-<a href="https://github.com/opendatalab/MinerU/graphs/contributors">
-  <img src="https://contrib.rocks/image?repo=opendatalab/MinerU" />
-</a>
-
-
-# License Information
-
-[LICENSE.md](LICENSE.md)
-
-The project currently leverages PyMuPDF to deliver advanced functionalities; however, its adherence to the AGPL license may impose limitations on certain use cases. In upcoming iterations, we intend to explore and transition to a more permissively licensed PDF processing library to enhance user-friendliness and flexibility.
-
-
-# Acknowledgments
-
-- [PaddleOCR](https://github.com/PaddlePaddle/PaddleOCR)
-- [PyMuPDF](https://github.com/pymupdf/PyMuPDF)
-- [fast-langdetect](https://github.com/LlmKira/fast-langdetect)
-- [pdfminer.six](https://github.com/pdfminer/pdfminer.six)
-
-
-# Citation
-
-```bibtex
-@article{he2024opendatalab,
-  title={Opendatalab: Empowering general artificial intelligence with open datasets},
-  author={He, Conghui and Li, Wei and Jin, Zhenjiang and Xu, Chao and Wang, Bin and Lin, Dahua},
-  journal={arXiv preprint arXiv:2407.13773},
-  year={2024}
-}
-
-@misc{2024mineru,
-    title={MinerU: A One-stop, Open-source, High-quality Data Extraction Tool},
-    author={MinerU Contributors},
-    howpublished = {\url{https://github.com/opendatalab/MinerU}},
-    year={2024}
-}
-```
-
-
-# Star History
-
-<a>
- <picture>
-   <source media="(prefers-color-scheme: dark)" srcset="https://api.star-history.com/svg?repos=opendatalab/MinerU&type=Date&theme=dark" />
-   <source media="(prefers-color-scheme: light)" srcset="https://api.star-history.com/svg?repos=opendatalab/MinerU&type=Date" />
-   <img alt="Star History Chart" src="https://api.star-history.com/svg?repos=opendatalab/MinerU&type=Date" />
- </picture>
-</a>
-
-# Links
-- [LabelU (A Lightweight Multi-modal Data Annotation Tool)](https://github.com/opendatalab/labelU)
-- [LabelLLM (An Open-source LLM Dialogue Annotation Platform)](https://github.com/opendatalab/LabelLLM)
-- [PDF-Extract-Kit (A Comprehensive Toolkit for High-Quality PDF Content Extraction)](https://github.com/opendatalab/PDF-Extract-Kit)