magic-pdf 0.5.10__py3-none-any.whl → 0.5.11__py3-none-any.whl

magic_pdf/cli/magicpdf.py

@@ -23,7 +23,6 @@ python magicpdf.py pdf-command --pdf  /home/llm/Downloads/xxxx.pdf --model /home
 
 import os
 import json as json_parse
-import sys
 import click
 from loguru import logger
 from pathlib import Path
@@ -46,6 +45,7 @@ from magic_pdf.rw.DiskReaderWriter import DiskReaderWriter
 from magic_pdf.rw.AbsReaderWriter import AbsReaderWriter
 import csv
 import copy
+import magic_pdf.model as model_config
 
 parse_pdf_methods = click.Choice(["ocr", "txt", "auto"])
 
@@ -66,21 +66,21 @@ def write_to_csv(csv_file_path, csv_data):
         csv_writer = csv.writer(csvfile)
         # 写入数据
         csv_writer.writerow(csv_data)
-    print(f"数据已成功追加到 '{csv_file_path}'")
+    logger.info(f"数据已成功追加到 '{csv_file_path}'")
 
 
 def do_parse(
-    pdf_file_name,
-    pdf_bytes,
-    model_list,
-    parse_method,
-    f_draw_span_bbox=True,
-    f_draw_layout_bbox=True,
-    f_dump_md=True,
-    f_dump_middle_json=True,
-    f_dump_model_json=True,
-    f_dump_orig_pdf=True,
-    f_dump_content_list=True,
+        pdf_file_name,
+        pdf_bytes,
+        model_list,
+        parse_method,
+        f_draw_span_bbox=True,
+        f_draw_layout_bbox=True,
+        f_dump_md=True,
+        f_dump_middle_json=True,
+        f_dump_model_json=True,
+        f_dump_orig_pdf=True,
+        f_dump_content_list=True,
 ):
     orig_model_list = copy.deepcopy(model_list)
 
@@ -96,14 +96,18 @@ def do_parse(
     elif parse_method == "ocr":
         pipe = OCRPipe(pdf_bytes, model_list, image_writer, is_debug=True)
     else:
-        print("unknown parse method")
-        sys.exit(1)
+        logger.error("unknown parse method")
+        exit(1)
 
     pipe.pipe_classify()
 
-    """如果没有传入有效的模型数据，则使用内置paddle解析"""
+    """如果没有传入有效的模型数据，则使用内置model解析"""
     if len(model_list) == 0:
-        pipe.pipe_analyze()
+        if model_config.__use_inside_model__:
+            pipe.pipe_analyze()
+        else:
+            logger.error("need model list input")
+            exit(1)
 
     pipe.pipe_parse()
     pdf_info = pipe.pdf_mid_data["pdf_info"]
@@ -170,10 +174,13 @@ def cli():
     help="指定解析方法。txt: 文本型 pdf 解析方法， ocr: 光学识别解析 pdf, auto: 程序智能选择解析方法",
     default="auto",
 )
-def json_command(json, method):
+@click.option("--inside_model", type=click.BOOL, default=False, help="使用内置模型测试")
+def json_command(json, method, inside_model):
+    model_config.__use_inside_model__ = inside_model
+
     if not json.startswith("s3://"):
-        print("usage: python magipdf.py --json s3://some_bucket/some_path")
-        sys.exit(1)
+        logger.error("usage: magic-pdf json-command --json s3://some_bucket/some_path")
+        exit(1)
 
     def read_s3_path(s3path):
         bucket, key = parse_s3path(s3path)
@@ -218,7 +225,10 @@ def json_command(json, method):
     help="指定解析方法。txt: 文本型 pdf 解析方法， ocr: 光学识别解析 pdf, auto: 程序智能选择解析方法",
     default="auto",
 )
-def local_json_command(local_json, method):
+@click.option("--inside_model", type=click.BOOL, default=False, help="使用内置模型测试")
+def local_json_command(local_json, method, inside_model):
+    model_config.__use_inside_model__ = inside_model
+
     def read_s3_path(s3path):
         bucket, key = parse_s3path(s3path)
 
@@ -267,7 +277,10 @@ def local_json_command(local_json, method):
     help="指定解析方法。txt: 文本型 pdf 解析方法， ocr: 光学识别解析 pdf, auto: 程序智能选择解析方法",
     default="auto",
 )
-def pdf_command(pdf, model, method):
+@click.option("--inside_model", type=click.BOOL, default=False, help="使用内置模型测试")
+def pdf_command(pdf, model, method, inside_model):
+    model_config.__use_inside_model__ = inside_model
+
     def read_fn(path):
         disk_rw = DiskReaderWriter(os.path.dirname(path))
         return disk_rw.read(os.path.basename(path), AbsReaderWriter.MODE_BIN)

magic_pdf/libs/version.py

@@ -1 +1 @@
-__version__ = "0.5.10"
+__version__ = "0.5.11"

magic_pdf/model/__init__.py

@@ -0,0 +1 @@
+__use_inside_model__ = False

magic_pdf/model/doc_analyze_by_custom_model.py

@@ -2,9 +2,10 @@ import fitz
 import cv2
 from PIL import Image
 import numpy as np
+from loguru import logger
 
 from magic_pdf.model.model_list import MODEL
-from magic_pdf.model.pp_structure_v2 import CustomPaddleModel
+import magic_pdf.model as model_config
 
 
 def dict_compare(d1, d2):
@@ -41,6 +42,13 @@ def load_images_from_pdf(pdf_bytes: bytes, dpi=200) -> list:
 
 
 def doc_analyze(pdf_bytes: bytes, ocr: bool = False, show_log: bool = False, model=MODEL.Paddle):
+
+    if model_config.__use_inside_model__:
+        from magic_pdf.model.pp_structure_v2 import CustomPaddleModel
+    else:
+        logger.error("use_inside_model is False, not allow to use inside model")
+        exit(1)
+
     images = load_images_from_pdf(pdf_bytes)
     custom_model = None
     if model == MODEL.Paddle:

magic_pdf/model/pp_structure_v2.py

@@ -1,7 +1,12 @@
 import random
 
 from loguru import logger
-from paddleocr import PPStructure
+
+try:
+    from paddleocr import PPStructure
+except ImportError:
+    logger.error('paddleocr not installed, please install by "pip install magic-pdf[cpu]" or "pip install magic-pdf[gpu]"')
+    exit(1)
 
 
 def region_to_bbox(region):

{magic_pdf-0.5.10.dist-info → magic_pdf-0.5.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: magic-pdf
-Version: 0.5.10
+Version: 0.5.11
 Summary: A practical tool for converting PDF to Markdown
 Home-page: https://github.com/magicpdf/Magic-PDF
 Requires-Python: >=3.9
@@ -22,11 +22,26 @@ Requires-Dist: wordninja >=2.0.0
 Requires-Dist: scikit-learn >=1.0.2
 Requires-Dist: nltk ==3.8.1
 Requires-Dist: s3pathlib >=2.1.1
-Requires-Dist: paddleocr
 Requires-Dist: pdfminer.six >=20231228
+Requires-Dist: Levenshtein
+Requires-Dist: nltk
+Requires-Dist: rapidfuzz
+Requires-Dist: statistics
+Requires-Dist: openxlab
+Requires-Dist: pandas
+Requires-Dist: numpy
+Requires-Dist: matplotlib
+Requires-Dist: seaborn
+Requires-Dist: scipy
+Requires-Dist: scikit-learn
+Requires-Dist: tqdm
+Requires-Dist: htmltabletomd
+Requires-Dist: pypandoc
 Provides-Extra: cpu
+Requires-Dist: paddleocr ; extra == 'cpu'
 Requires-Dist: paddlepaddle ; extra == 'cpu'
 Provides-Extra: gpu
+Requires-Dist: paddleocr ; extra == 'gpu'
 Requires-Dist: paddlepaddle-gpu ; extra == 'gpu'
 
 <div id="top"></div>
@@ -64,6 +79,10 @@ Key features include:
 - Compatibility with CPU and GPU environments
 - Available for Windows, Linux, and macOS platforms
 
+## Project Panorama
+
+![Project Panorama](docs/images/project_panorama_en.png)
+
 ## Getting Started
 
 ### Requirements

{magic_pdf-0.5.10.dist-info → magic_pdf-0.5.11.dist-info}/RECORD

@@ -5,7 +5,7 @@ magic_pdf/pdf_parse_for_train.py,sha256=Oby61DMjJ716Jj_ri7lwXfv2Chus0pbBR2RPXrmB
 magic_pdf/pdf_parse_union_core.py,sha256=a67iQuEfuslAEF-wQplGZKXUuz5mT3HiCyvuR52E6Gw,10584
 magic_pdf/user_api.py,sha256=CVQH-VSiZpz0bSkyMT4czk1epZriIPSJsLsPbluPa9Q,3054
 magic_pdf/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-magic_pdf/cli/magicpdf.py,sha256=FF6flO6wUcKG9Qx_FG6-xhHfmQzQWLjwrkMa5kowDgs,10937
+magic_pdf/cli/magicpdf.py,sha256=BA6lPjBhurXAnfmvdbQiS_atEFq3PVdNYDPkvxs1N0M,11654
 magic_pdf/dict2md/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 magic_pdf/dict2md/mkcontent.py,sha256=rWUY-2opd0jeowEUEVOV_uWcKum1Q7ng4nOoT6-ka_s,17459
 magic_pdf/dict2md/ocr_mkcontent.py,sha256=RyxebPtvFfNce_HCa-_YGxwFx_srzL-BfMKc85V9JG0,15442
@@ -44,14 +44,14 @@ magic_pdf/libs/pdf_check.py,sha256=MAe8wzwT0qvPf_I72wEZG7k1g4haNHS7oUtLqkB5rlE,2
 magic_pdf/libs/pdf_image_tools.py,sha256=CAd01giTKr_UJz1_QtDOARG9G9z69GFpzRZwcWSfLtE,1282
 magic_pdf/libs/safe_filename.py,sha256=ckwcM_eqoysTb5id8czp-tXq2G9da0-l3pshZDCHQtE,236
 magic_pdf/libs/textbase.py,sha256=SC1Frhz3Fb7V7n2SFRBsl7Bmg0JZdlvZskq0lfW1vIk,732
-magic_pdf/libs/version.py,sha256=1nlPInsRzDbcDPveZ3ghSJ6v6KveN9n6gnj-twW4DkI,23
+magic_pdf/libs/version.py,sha256=xFez9dUQrcuZqZRWuEIsCbMskoR-Ke1_uUZ51Kyt1tw,23
 magic_pdf/libs/vis_utils.py,sha256=hTOTEakKV0pGMbk0tbRkVI_tku7A3dGc96ynObZ4kwI,10207
 magic_pdf/model/360_layout_analysis.py,sha256=GbchKPJRVcrxvwNXMnR4vt8lOLPauTWMl-43ayyhX7U,221
-magic_pdf/model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-magic_pdf/model/doc_analyze_by_custom_model.py,sha256=QD4NWEIz8UXdIG4V_3P8EaYesxk6PvC1SOtTWEy2GEY,2007
+magic_pdf/model/__init__.py,sha256=X6t9kPDqM8hDCbq8fQc_8jILtG6mepDjN_kadUo39Sk,29
+magic_pdf/model/doc_analyze_by_custom_model.py,sha256=N3DqbVT1hc4s9KhppWDmZWkCj2ExKltoLrQl2IWGk7c,2231
 magic_pdf/model/magic_model.py,sha256=2H6Gz1mg0f0YCvz-TLIWrAWXCQLgZftBXJNRPlSIjwc,25077
 magic_pdf/model/model_list.py,sha256=dNfnDodnbkgIW0PFDjn_KsQMA8DODOzo4Z4jxfOilaA,44
-magic_pdf/model/pp_structure_v2.py,sha256=qsyt9vFDGaVizBMiSaeFVHTDsJTrIHx46Ec2J8SOj1A,2469
+magic_pdf/model/pp_structure_v2.py,sha256=fFbAOYEcLXlkCjqZ3yxZXR7nqtp6V8yowyjSibW3lhY,2635
 magic_pdf/para/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 magic_pdf/para/block_continuation_processor.py,sha256=IkReB5hirjm0OAirNzQQpnWe2f2bdP3Hug3Ef8qTRDA,22749
 magic_pdf/para/block_termination_processor.py,sha256=YU3ZYqJy9e3OQmOuQYZrR6AUpmAlQ0mhj0PgZZPZ_fM,17957
@@ -115,9 +115,9 @@ magic_pdf/train_utils/convert_to_train_format.py,sha256=ifo2FAoBMa_etCvz0O4v03xO
 magic_pdf/train_utils/extract_caption.py,sha256=gommEqIEWLplSDEJWD7_66daqlOBsWhpRBW1DHpkny4,1825
 magic_pdf/train_utils/remove_footer_header.py,sha256=pyeNNdJ-th3wl5Xwb10ZLYNaFN4-6BmahoMFE8VTNNs,5978
 magic_pdf/train_utils/vis_utils.py,sha256=MV9N9cT3ifJ35u7LFKGF9I_bOIQrtU1zcsxu2hj3aqM,10111
-magic_pdf-0.5.10.dist-info/LICENSE.md,sha256=hIahDEOTzuHCU5J2nd07LWwkLW7Hko4UFO__ffsvB-8,34523
-magic_pdf-0.5.10.dist-info/METADATA,sha256=B3e0sVOyFhk47EfHPuLFRUNxzdasWYx3XuYR53LSJX8,4175
-magic_pdf-0.5.10.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-magic_pdf-0.5.10.dist-info/entry_points.txt,sha256=NbSkSmE08UuTwdoJD8Uofq8iyufySA4x7jmIIk4YCzI,57
-magic_pdf-0.5.10.dist-info/top_level.txt,sha256=J9I0AzmHWGkp9c6DL8Oe4mEx3yYphLzkRn4H25Lg1rE,10
-magic_pdf-0.5.10.dist-info/RECORD,,
+magic_pdf-0.5.11.dist-info/LICENSE.md,sha256=hIahDEOTzuHCU5J2nd07LWwkLW7Hko4UFO__ffsvB-8,34523
+magic_pdf-0.5.11.dist-info/METADATA,sha256=wMDXFCmnlXQKkUdp891cG46MrbDn92TlPPD8T7AT3tE,4649
+magic_pdf-0.5.11.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+magic_pdf-0.5.11.dist-info/entry_points.txt,sha256=NbSkSmE08UuTwdoJD8Uofq8iyufySA4x7jmIIk4YCzI,57
+magic_pdf-0.5.11.dist-info/top_level.txt,sha256=J9I0AzmHWGkp9c6DL8Oe4mEx3yYphLzkRn4H25Lg1rE,10
+magic_pdf-0.5.11.dist-info/RECORD,,