magic-pdf 0.5.9__py3-none-any.whl → 0.5.11__py3-none-any.whl

magic_pdf/user_api.py

@@ -18,8 +18,8 @@ from loguru import logger
 from magic_pdf.libs.version import __version__
 from magic_pdf.model.doc_analyze_by_custom_model import doc_analyze
 from magic_pdf.rw import AbsReaderWriter
-from magic_pdf.pdf_parse_by_ocr_v2 import parse_pdf_by_ocr
-from magic_pdf.pdf_parse_by_txt_v2 import parse_pdf_by_txt
+from magic_pdf.pdf_parse_by_ocr import parse_pdf_by_ocr
+from magic_pdf.pdf_parse_by_txt import parse_pdf_by_txt
 
 PARSE_TYPE_TXT = "txt"
 PARSE_TYPE_OCR = "ocr"
@@ -86,45 +86,7 @@ def parse_union_pdf(pdf_bytes: bytes, pdf_models: list, imageWriter: AbsReaderWr
             return None
 
     pdf_info_dict = parse_pdf(parse_pdf_by_txt)
-    # text_all = ""
-    # for page_dict in pdf_info_dict['pdf_info']:
-    #     for para_block in page_dict['para_blocks']:
-    #         if para_block['type'] in ['title', 'text']:
-    #             for line in para_block['lines']:
-    #                 for span in line['spans']:
-    #                     text_all += span['content']
-
-    # def calculate_not_common_character_rate(text):
-    #     garbage_regex = re.compile(r'[^\u4e00-\u9fa5\u0030-\u0039\u0041-\u005a\u0061-\u007a\u3000-\u303f\uff00-\uffef]')
-    #     # 计算乱码字符的数量
-    #     garbage_count = len(garbage_regex.findall(text))
-    #     total = len(text)
-    #     if total == 0:
-    #         return 0  # 避免除以零的错误
-    #     return garbage_count / total
-    #
-    # def calculate_not_printable_rate(text):
-    #     printable_text = ""
-    #     for c in text:
-    #         if c.isprintable():
-    #             printable_text += c
-    #     printable_total = len(printable_text)
-    #     total = len(text)
-    #     if total == 0:
-    #         return 0  # 避免除以零的错误
-    #     return (total - printable_total) / total
-    #
-    # not_common_character_rate = calculate_not_common_character_rate(text_all)
-    # not_printable_rate = calculate_not_printable_rate(text_all)
-    # pdf_info_dict["_not_common_character_rate"] = not_common_character_rate
-    # pdf_info_dict["_not_printable_rate"] = not_printable_rate
-    # logger.info(f"not_common_character_rate: {not_common_character_rate}, not_printable_rate: {not_printable_rate}")
-    '''新逻辑使用pdfminer识别乱码pdf,准确率高且不会误伤,已在解析流程之前进行处理'''
-    # not_common_character_rate对小语种可能会有误伤，not_printable_rate对小语种较为友好
-    if (pdf_info_dict is None
-            or pdf_info_dict.get("_need_drop", False)
-            # or not_printable_rate > 0.02  # 参考一些正常的pdf，这个值没有超过0.01的，阈值设为0.02
-    ):
+    if pdf_info_dict is None or pdf_info_dict.get("_need_drop", False):
         logger.warning(f"parse_pdf_by_txt drop or error, switch to parse_pdf_by_ocr")
         if input_model_is_empty:
             pdf_models = doc_analyze(pdf_bytes, ocr=True)

{magic_pdf-0.5.9.dist-info → magic_pdf-0.5.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: magic-pdf
-Version: 0.5.9
+Version: 0.5.11
 Summary: A practical tool for converting PDF to Markdown
 Home-page: https://github.com/magicpdf/Magic-PDF
 Requires-Python: >=3.9
@@ -22,11 +22,26 @@ Requires-Dist: wordninja >=2.0.0
 Requires-Dist: scikit-learn >=1.0.2
 Requires-Dist: nltk ==3.8.1
 Requires-Dist: s3pathlib >=2.1.1
-Requires-Dist: paddleocr
 Requires-Dist: pdfminer.six >=20231228
+Requires-Dist: Levenshtein
+Requires-Dist: nltk
+Requires-Dist: rapidfuzz
+Requires-Dist: statistics
+Requires-Dist: openxlab
+Requires-Dist: pandas
+Requires-Dist: numpy
+Requires-Dist: matplotlib
+Requires-Dist: seaborn
+Requires-Dist: scipy
+Requires-Dist: scikit-learn
+Requires-Dist: tqdm
+Requires-Dist: htmltabletomd
+Requires-Dist: pypandoc
 Provides-Extra: cpu
+Requires-Dist: paddleocr ; extra == 'cpu'
 Requires-Dist: paddlepaddle ; extra == 'cpu'
 Provides-Extra: gpu
+Requires-Dist: paddleocr ; extra == 'gpu'
 Requires-Dist: paddlepaddle-gpu ; extra == 'gpu'
 
 <div id="top"></div>
@@ -64,6 +79,10 @@ Key features include:
 - Compatibility with CPU and GPU environments
 - Available for Windows, Linux, and macOS platforms
 
+## Project Panorama
+
+![Project Panorama](docs/images/project_panorama_en.png)
+
 ## Getting Started
 
 ### Requirements
@@ -72,20 +91,57 @@ Key features include:
 
 ### Usage Instructions
 
-1. **Install Magic-PDF**
-
+#### 1. Install Magic-PDF
 ```bash
-pip install magic-pdf[cpu] # Install the CPU version 
-or
-pip install magic-pdf[gpu] # Install the GPU version
+pip install magic-pdf
 ```
 
-2. **Usage via Command Line**
+#### 2. Usage via Command Line
 
+###### simple
+```bash
+cp magic-pdf.template.json to ~/magic-pdf.json
+magic-pdf pdf-command --pdf "pdf_path" --model "model_json_path"
+```
+###### more 
 ```bash
 magic-pdf --help
 ```
 
+#### 3. Usage via Api
+
+###### Local
+```python
+image_writer = DiskReaderWriter(local_image_dir)
+image_dir = str(os.path.basename(local_image_dir))
+jso_useful_key = {"_pdf_type": "", "model_list": model_json}
+pipe = UNIPipe(pdf_bytes, jso_useful_key, image_writer)
+pipe.pipe_classify()
+pipe.pipe_parse()
+md_content = pipe.pipe_mk_markdown(image_dir, drop_mode="none")
+```
+
+###### Object Storage
+```python
+s3pdf_cli = S3ReaderWriter(pdf_ak, pdf_sk, pdf_endpoint)
+image_dir = "s3://img_bucket/"
+s3image_cli = S3ReaderWriter(img_ak, img_sk, img_endpoint, parent_path=image_dir)
+pdf_bytes = s3pdf_cli.read(s3_pdf_path, mode=s3pdf_cli.MODE_BIN)
+jso_useful_key = {"_pdf_type": "", "model_list": model_json}
+pipe = UNIPipe(pdf_bytes, jso_useful_key, s3image_cli)
+pipe.pipe_classify()
+pipe.pipe_parse()
+md_content = pipe.pipe_mk_markdown(image_dir, drop_mode="none")
+```
+
+Demo can be referred to [demo.py](https://github.com/magicpdf/Magic-PDF/blob/master/demo/demo.py)
+
+## All Thanks To Our Contributors
+
+<a href="https://github.com/magicpdf/Magic-PDF/graphs/contributors">
+  <img src="https://contrib.rocks/image?repo=magicpdf/Magic-PDF" />
+</a>
+
 ## License Information
 
 See [LICENSE.md](https://github.com/magicpdf/Magic-PDF/blob/master/LICENSE.md) for details.

{magic_pdf-0.5.9.dist-info → magic_pdf-0.5.11.dist-info}/RECORD

@@ -1,13 +1,11 @@
 magic_pdf/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-magic_pdf/pdf_parse_by_ocr.py,sha256=jYEfc4P6o4f7mZqfPFITs3NWlBxe4v1gwLh1yu2VDIQ,8950
-magic_pdf/pdf_parse_by_ocr_v2.py,sha256=1XVRwmcGRtxMMRYgqkJcctD4tLjCprgpGQsHpeVQktM,637
-magic_pdf/pdf_parse_by_txt.py,sha256=5_kdfvDkv_XwDove2AW7SopGysYLJ1-tsOQy2yuII1Y,21932
-magic_pdf/pdf_parse_by_txt_v2.py,sha256=mGadyYamoCNGNsKOQM1uXQR65zMUKyL24yURGHADmVs,1908
+magic_pdf/pdf_parse_by_ocr.py,sha256=IWnSWt1Z-d35xRqspzdLR2iUtma_SAu4W7K4kEk8SHc,638
+magic_pdf/pdf_parse_by_txt.py,sha256=KUSH7Gh83CZmdyWw59pqDskwyJ2Kg-jU-9fnQGJQEs4,537
 magic_pdf/pdf_parse_for_train.py,sha256=Oby61DMjJ716Jj_ri7lwXfv2Chus0pbBR2RPXrmBW08,28661
 magic_pdf/pdf_parse_union_core.py,sha256=a67iQuEfuslAEF-wQplGZKXUuz5mT3HiCyvuR52E6Gw,10584
-magic_pdf/user_api.py,sha256=R4onPBXlA8GARwlqBh5wmxUtTxwQb-PUeFK7eTAWzoU,4971
+magic_pdf/user_api.py,sha256=CVQH-VSiZpz0bSkyMT4czk1epZriIPSJsLsPbluPa9Q,3054
 magic_pdf/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-magic_pdf/cli/magicpdf.py,sha256=aMmY_J83_8IdkkxyZUHg8WzIbCA_oW4cyjlUUZE0Wvc,11117
+magic_pdf/cli/magicpdf.py,sha256=BA6lPjBhurXAnfmvdbQiS_atEFq3PVdNYDPkvxs1N0M,11654
 magic_pdf/dict2md/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 magic_pdf/dict2md/mkcontent.py,sha256=rWUY-2opd0jeowEUEVOV_uWcKum1Q7ng4nOoT6-ka_s,17459
 magic_pdf/dict2md/ocr_mkcontent.py,sha256=RyxebPtvFfNce_HCa-_YGxwFx_srzL-BfMKc85V9JG0,15442
@@ -46,14 +44,14 @@ magic_pdf/libs/pdf_check.py,sha256=MAe8wzwT0qvPf_I72wEZG7k1g4haNHS7oUtLqkB5rlE,2
 magic_pdf/libs/pdf_image_tools.py,sha256=CAd01giTKr_UJz1_QtDOARG9G9z69GFpzRZwcWSfLtE,1282
 magic_pdf/libs/safe_filename.py,sha256=ckwcM_eqoysTb5id8czp-tXq2G9da0-l3pshZDCHQtE,236
 magic_pdf/libs/textbase.py,sha256=SC1Frhz3Fb7V7n2SFRBsl7Bmg0JZdlvZskq0lfW1vIk,732
-magic_pdf/libs/version.py,sha256=JXLyhF5WmLgRZBfWGz9zWe2g5ISKSLpn2jp8yLaC-s4,22
+magic_pdf/libs/version.py,sha256=xFez9dUQrcuZqZRWuEIsCbMskoR-Ke1_uUZ51Kyt1tw,23
 magic_pdf/libs/vis_utils.py,sha256=hTOTEakKV0pGMbk0tbRkVI_tku7A3dGc96ynObZ4kwI,10207
 magic_pdf/model/360_layout_analysis.py,sha256=GbchKPJRVcrxvwNXMnR4vt8lOLPauTWMl-43ayyhX7U,221
-magic_pdf/model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-magic_pdf/model/doc_analyze_by_custom_model.py,sha256=QD4NWEIz8UXdIG4V_3P8EaYesxk6PvC1SOtTWEy2GEY,2007
+magic_pdf/model/__init__.py,sha256=X6t9kPDqM8hDCbq8fQc_8jILtG6mepDjN_kadUo39Sk,29
+magic_pdf/model/doc_analyze_by_custom_model.py,sha256=N3DqbVT1hc4s9KhppWDmZWkCj2ExKltoLrQl2IWGk7c,2231
 magic_pdf/model/magic_model.py,sha256=2H6Gz1mg0f0YCvz-TLIWrAWXCQLgZftBXJNRPlSIjwc,25077
 magic_pdf/model/model_list.py,sha256=dNfnDodnbkgIW0PFDjn_KsQMA8DODOzo4Z4jxfOilaA,44
-magic_pdf/model/pp_structure_v2.py,sha256=qsyt9vFDGaVizBMiSaeFVHTDsJTrIHx46Ec2J8SOj1A,2469
+magic_pdf/model/pp_structure_v2.py,sha256=fFbAOYEcLXlkCjqZ3yxZXR7nqtp6V8yowyjSibW3lhY,2635
 magic_pdf/para/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 magic_pdf/para/block_continuation_processor.py,sha256=IkReB5hirjm0OAirNzQQpnWe2f2bdP3Hug3Ef8qTRDA,22749
 magic_pdf/para/block_termination_processor.py,sha256=YU3ZYqJy9e3OQmOuQYZrR6AUpmAlQ0mhj0PgZZPZ_fM,17957
@@ -64,7 +62,7 @@ magic_pdf/para/exceptions.py,sha256=kpjGxrSZ-drNmoKlmuQ0asTjI8cKKKWsdDDBoDHQP9M,
 magic_pdf/para/layout_match_processor.py,sha256=yr4FEO7GJ502udShqGRqIJQ_FQxoa0aG_mhmWd8nLwI,1554
 magic_pdf/para/para_pipeline.py,sha256=zLaCHI9jLi1UPzh0lHP44mUjpKVTHS0gE_5YrkjVqEY,11796
 magic_pdf/para/para_split.py,sha256=-UJM2jREW_2h3ZlJAU7dRD8bK3CMGKuhJrfgqv3Auvk,31310
-magic_pdf/para/para_split_v2.py,sha256=a04dsUFE3JD4DA9e2DULJgbKrcqWuCfK58de1p-T3Io,36610
+magic_pdf/para/para_split_v2.py,sha256=jGOhsubdh_CEgSv9WMNmp1loq1YNlpcAj3yh3g0gPhw,37027
 magic_pdf/para/raw_processor.py,sha256=mHxD9FrdOSXH7NqM41s55URyCyuyACvm9kKtowkIb3k,6317
 magic_pdf/para/stats.py,sha256=-6Pf9Y8jkP1uJOYWiHUjw9Lb-Fb9GY7MHr_ok7x2GX0,9731
 magic_pdf/para/title_processor.py,sha256=pYZv9vEkIjAtCz8jIUtl9AVUy_ib5SdAZmMVoZtsMRI,38593
@@ -89,7 +87,7 @@ magic_pdf/pre_proc/detect_header.py,sha256=KOmRehgKMuMqNa_2weXkdNSiRVWMFgLMQE4e1
 magic_pdf/pre_proc/detect_images.py,sha256=8DwGGTb5IjxqADZDTc_ngwJrTYXxK2qpRqI2FBoPr00,30432
 magic_pdf/pre_proc/detect_page_number.py,sha256=qvYrBbCtBbREvw-MySL_p7byCRvcm1fkLJ5ZB4TP8OM,2848
 magic_pdf/pre_proc/detect_tables.py,sha256=srJzgLVeVuOsqnESqfdJfVukTF84K8qmI5mgFX_BZGs,2800
-magic_pdf/pre_proc/equations_replace.py,sha256=rpEHD5IvavRgUSus0tLjKcIv2faStp_Ii9vf09HVUtI,20384
+magic_pdf/pre_proc/equations_replace.py,sha256=fXj7ZV7F3YtkDYrAhE9g5tHk4_3pVUyLbhDtMjbxjWU,20386
 magic_pdf/pre_proc/fix_image.py,sha256=5MOfkXc8abfIp49g-68vll40wwTUZ5tcQ2gtsJuFmvs,11486
 magic_pdf/pre_proc/fix_table.py,sha256=20sqJe27fAXcL7_C0qQ9mpsggmH37WuX-wPYWyRgACA,13227
 magic_pdf/pre_proc/main_text_font.py,sha256=1gkjvPuBdKC4oVFkLvnRm2zghsLtVlfAEMKXouyVonM,1048
@@ -117,9 +115,9 @@ magic_pdf/train_utils/convert_to_train_format.py,sha256=ifo2FAoBMa_etCvz0O4v03xO
 magic_pdf/train_utils/extract_caption.py,sha256=gommEqIEWLplSDEJWD7_66daqlOBsWhpRBW1DHpkny4,1825
 magic_pdf/train_utils/remove_footer_header.py,sha256=pyeNNdJ-th3wl5Xwb10ZLYNaFN4-6BmahoMFE8VTNNs,5978
 magic_pdf/train_utils/vis_utils.py,sha256=MV9N9cT3ifJ35u7LFKGF9I_bOIQrtU1zcsxu2hj3aqM,10111
-magic_pdf-0.5.9.dist-info/LICENSE.md,sha256=hIahDEOTzuHCU5J2nd07LWwkLW7Hko4UFO__ffsvB-8,34523
-magic_pdf-0.5.9.dist-info/METADATA,sha256=6Y0tWpKEWrjYaNVrBWddqU9mn4EKR8cSbka47hUSmog,2971
-magic_pdf-0.5.9.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-magic_pdf-0.5.9.dist-info/entry_points.txt,sha256=NbSkSmE08UuTwdoJD8Uofq8iyufySA4x7jmIIk4YCzI,57
-magic_pdf-0.5.9.dist-info/top_level.txt,sha256=J9I0AzmHWGkp9c6DL8Oe4mEx3yYphLzkRn4H25Lg1rE,10
-magic_pdf-0.5.9.dist-info/RECORD,,
+magic_pdf-0.5.11.dist-info/LICENSE.md,sha256=hIahDEOTzuHCU5J2nd07LWwkLW7Hko4UFO__ffsvB-8,34523
+magic_pdf-0.5.11.dist-info/METADATA,sha256=wMDXFCmnlXQKkUdp891cG46MrbDn92TlPPD8T7AT3tE,4649
+magic_pdf-0.5.11.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+magic_pdf-0.5.11.dist-info/entry_points.txt,sha256=NbSkSmE08UuTwdoJD8Uofq8iyufySA4x7jmIIk4YCzI,57
+magic_pdf-0.5.11.dist-info/top_level.txt,sha256=J9I0AzmHWGkp9c6DL8Oe4mEx3yYphLzkRn4H25Lg1rE,10
+magic_pdf-0.5.11.dist-info/RECORD,,

magic_pdf/pdf_parse_by_ocr_v2.py

@@ -1,17 +0,0 @@
-from magic_pdf.pdf_parse_union_core import pdf_parse_union
-
-def parse_pdf_by_ocr(pdf_bytes,
-                     model_list,
-                     imageWriter,
-                     start_page_id=0,
-                     end_page_id=None,
-                     debug_mode=False,
-                     ):
-    return pdf_parse_union(pdf_bytes,
-                           model_list,
-                           imageWriter,
-                           "ocr",
-                           start_page_id=start_page_id,
-                           end_page_id=end_page_id,
-                           debug_mode=debug_mode,
-                           )

magic_pdf/pdf_parse_by_txt_v2.py

@@ -1,56 +0,0 @@
-from magic_pdf.pdf_parse_union_core import pdf_parse_union
-
-
-def parse_pdf_by_txt(
-    pdf_bytes,
-    model_list,
-    imageWriter,
-    start_page_id=0,
-    end_page_id=None,
-    debug_mode=False,
-):
-    return pdf_parse_union(pdf_bytes,
-                           model_list,
-                           imageWriter,
-                           "txt",
-                           start_page_id=start_page_id,
-                           end_page_id=end_page_id,
-                           debug_mode=debug_mode,
-                           )
-
-
-if __name__ == "__main__":
-    pass
-    # if 1:
-    #     import fitz
-    #     import json
-    #
-    #     with open("/opt/data/pdf/20240418/25536-00.pdf", "rb") as f:
-    #         pdf_bytes = f.read()
-    #     pdf_docs = fitz.open("pdf", pdf_bytes)
-    #
-    #     with open("/opt/data/pdf/20240418/25536-00.json") as f:
-    #         model_list = json.loads(f.readline())
-    #
-    #     magic_model = MagicModel(model_list, pdf_docs)
-    #     for i in range(7):
-    #         print(magic_model.get_imgs(i))
-    #
-    #     for page_no, page in enumerate(pdf_docs):
-    #         inline_equations, interline_equations, interline_equation_blocks = (
-    #             magic_model.get_equations(page_no)
-    #         )
-    #
-    #         text_raw_blocks = page.get_text("dict", flags=fitz.TEXTFLAGS_TEXT)["blocks"]
-    #         char_level_text_blocks = page.get_text(
-    #             "rawdict", flags=fitz.TEXTFLAGS_TEXT
-    #         )["blocks"]
-    #         text_blocks = combine_chars_to_pymudict(
-    #             text_raw_blocks, char_level_text_blocks
-    #         )
-    #         text_blocks = replace_equations_in_textblock(
-    #             text_blocks, inline_equations, interline_equations
-    #         )
-    #         text_blocks = remove_citation_marker(text_blocks)
-    #
-    #         text_blocks = remove_chars_in_text_blocks(text_blocks)