magic-pdf 0.8.0__py3-none-any.whl → 0.9.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. magic_pdf/config/__init__.py +0 -0
  2. magic_pdf/config/enums.py +7 -0
  3. magic_pdf/config/exceptions.py +32 -0
  4. magic_pdf/data/__init__.py +0 -0
  5. magic_pdf/data/data_reader_writer/__init__.py +12 -0
  6. magic_pdf/data/data_reader_writer/base.py +51 -0
  7. magic_pdf/data/data_reader_writer/filebase.py +59 -0
  8. magic_pdf/data/data_reader_writer/multi_bucket_s3.py +137 -0
  9. magic_pdf/data/data_reader_writer/s3.py +69 -0
  10. magic_pdf/data/dataset.py +194 -0
  11. magic_pdf/data/io/__init__.py +0 -0
  12. magic_pdf/data/io/base.py +42 -0
  13. magic_pdf/data/io/http.py +37 -0
  14. magic_pdf/data/io/s3.py +114 -0
  15. magic_pdf/data/read_api.py +95 -0
  16. magic_pdf/data/schemas.py +15 -0
  17. magic_pdf/data/utils.py +32 -0
  18. magic_pdf/dict2md/ocr_mkcontent.py +74 -234
  19. magic_pdf/libs/Constants.py +21 -8
  20. magic_pdf/libs/MakeContentConfig.py +1 -0
  21. magic_pdf/libs/boxbase.py +54 -0
  22. magic_pdf/libs/clean_memory.py +10 -0
  23. magic_pdf/libs/config_reader.py +53 -23
  24. magic_pdf/libs/draw_bbox.py +150 -65
  25. magic_pdf/libs/ocr_content_type.py +2 -0
  26. magic_pdf/libs/version.py +1 -1
  27. magic_pdf/model/doc_analyze_by_custom_model.py +77 -32
  28. magic_pdf/model/magic_model.py +418 -51
  29. magic_pdf/model/pdf_extract_kit.py +164 -80
  30. magic_pdf/model/pek_sub_modules/structeqtable/StructTableModel.py +8 -1
  31. magic_pdf/model/ppTableModel.py +2 -2
  32. magic_pdf/model/pp_structure_v2.py +5 -2
  33. magic_pdf/model/v3/__init__.py +0 -0
  34. magic_pdf/model/v3/helpers.py +125 -0
  35. magic_pdf/para/para_split_v3.py +296 -0
  36. magic_pdf/pdf_parse_by_ocr.py +6 -3
  37. magic_pdf/pdf_parse_by_txt.py +6 -3
  38. magic_pdf/pdf_parse_union_core_v2.py +644 -0
  39. magic_pdf/pipe/AbsPipe.py +5 -1
  40. magic_pdf/pipe/OCRPipe.py +10 -4
  41. magic_pdf/pipe/TXTPipe.py +10 -4
  42. magic_pdf/pipe/UNIPipe.py +16 -7
  43. magic_pdf/pre_proc/ocr_detect_all_bboxes.py +83 -1
  44. magic_pdf/pre_proc/ocr_dict_merge.py +27 -2
  45. magic_pdf/resources/model_config/UniMERNet/demo.yaml +7 -7
  46. magic_pdf/resources/model_config/model_configs.yaml +5 -13
  47. magic_pdf/tools/cli.py +14 -1
  48. magic_pdf/tools/common.py +19 -9
  49. magic_pdf/user_api.py +25 -6
  50. magic_pdf/utils/__init__.py +0 -0
  51. magic_pdf/utils/annotations.py +11 -0
  52. {magic_pdf-0.8.0.dist-info → magic_pdf-0.9.0.dist-info}/LICENSE.md +1 -0
  53. magic_pdf-0.9.0.dist-info/METADATA +507 -0
  54. {magic_pdf-0.8.0.dist-info → magic_pdf-0.9.0.dist-info}/RECORD +57 -33
  55. magic_pdf-0.8.0.dist-info/METADATA +0 -459
  56. {magic_pdf-0.8.0.dist-info → magic_pdf-0.9.0.dist-info}/WHEEL +0 -0
  57. {magic_pdf-0.8.0.dist-info → magic_pdf-0.9.0.dist-info}/entry_points.txt +0 -0
  58. {magic_pdf-0.8.0.dist-info → magic_pdf-0.9.0.dist-info}/top_level.txt +0 -0
@@ -1,11 +1,29 @@
1
1
  magic_pdf/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
2
- magic_pdf/pdf_parse_by_ocr.py,sha256=IWnSWt1Z-d35xRqspzdLR2iUtma_SAu4W7K4kEk8SHc,638
3
- magic_pdf/pdf_parse_by_txt.py,sha256=KUSH7Gh83CZmdyWw59pqDskwyJ2Kg-jU-9fnQGJQEs4,537
2
+ magic_pdf/pdf_parse_by_ocr.py,sha256=E-AYHUXjzorFli0CEtmnAi09SI2STJ7FX58yjU0c9PI,810
3
+ magic_pdf/pdf_parse_by_txt.py,sha256=YeFYVAdfwF1CXOHq0LVE5131nqPHA14nt5t_sb-CMMk,709
4
4
  magic_pdf/pdf_parse_union_core.py,sha256=AGIrP7ahc6Ycku0PxAlbjZhwqsdJ8iuRPIn-PFASKWY,11772
5
- magic_pdf/user_api.py,sha256=gzmUHOYlmqPtuw1eZ3Qe2VZCG29v403oN0DURbEsyS8,3417
5
+ magic_pdf/pdf_parse_union_core_v2.py,sha256=18V8aYLz0gLQSMxGJDgUCWSltr15L8s8ClNqB17DNvw,23647
6
+ magic_pdf/user_api.py,sha256=gM-3RQYc6pMEsVvEPFXfWf5RBjAvHcUccugL6fXpP_U,3991
7
+ magic_pdf/config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
8
+ magic_pdf/config/enums.py,sha256=CImYuw4sbKpq9zrj6zrrEvtdoGkjxDt8S8ByNVDpypU,89
9
+ magic_pdf/config/exceptions.py,sha256=87UX7gyUpj4HqjPcz2hLqdnYeImtDQAxOxj8oXZ_zkE,622
10
+ magic_pdf/data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
11
+ magic_pdf/data/dataset.py,sha256=n8rGw1-wizABR8giSk_XWPCXzx3478u5DK2Z0wOCOeI,5089
12
+ magic_pdf/data/read_api.py,sha256=3fKLsEYAow5RwAmGFMMgvcCh0-_WEEHem2uewukjXOA,3570
13
+ magic_pdf/data/schemas.py,sha256=XSFNxyYbIWgU_Z4U0695elpGQP3J5dpq4Rlyr3S0O_s,595
14
+ magic_pdf/data/utils.py,sha256=dJZiqygwNier0UG5tbt5jAPjgwcnfsAN6-m-G1kVPLQ,917
15
+ magic_pdf/data/data_reader_writer/__init__.py,sha256=QtevUaeSivv9dQKi3Tomfn4Z0E4To0cB8qXTnglxaHc,705
16
+ magic_pdf/data/data_reader_writer/base.py,sha256=gUrHCMTHYBrWpqgHdIc-hN7HHwUC2ApK_VXrDUrnfdg,1320
17
+ magic_pdf/data/data_reader_writer/filebase.py,sha256=21RYy4m9MqJGqwd2HWICQJHM-PZXp7UYETCQQK390Kk,1988
18
+ magic_pdf/data/data_reader_writer/multi_bucket_s3.py,sha256=BY3faRfZTg27kfkaI4iXPjgFW_RecN0am9r9z2RuYgY,5582
19
+ magic_pdf/data/data_reader_writer/s3.py,sha256=4tT_hcb5I1m-qojNP2CAUKGOoWBH2ripKQmBa9_dAfg,2096
20
+ magic_pdf/data/io/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
21
+ magic_pdf/data/io/base.py,sha256=So3G_Kndunfs0f9nn3l9dRJG_7N09CX0JbFqYEvyaRI,1113
22
+ magic_pdf/data/io/http.py,sha256=XlKB0DNf4a_uUnfgcclvaaOtmE7lmddx0DnK8A-emAM,958
23
+ magic_pdf/data/io/s3.py,sha256=hyA7sbNriQy64xd_uyJ7acN_oneQ1Pdmoc7_xcvkue8,3606
6
24
  magic_pdf/dict2md/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
7
25
  magic_pdf/dict2md/mkcontent.py,sha256=rWUY-2opd0jeowEUEVOV_uWcKum1Q7ng4nOoT6-ka_s,17459
8
- magic_pdf/dict2md/ocr_mkcontent.py,sha256=YILNQm8Nq1VMpgKTGF_fRNEzkrEg1aQKHxWI0lVQucI,18565
26
+ magic_pdf/dict2md/ocr_mkcontent.py,sha256=eMd3qOIh21cZgTl-LMLGh42uxDMpHU2nwE6iA6b_qrA,11915
9
27
  magic_pdf/filter/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
10
28
  magic_pdf/filter/pdf_classify_by_type.py,sha256=spmDO-f2ihAl1d6-EP-j271Yi50oyu6mw4X2kRd_m0s,42320
11
29
  magic_pdf/filter/pdf_meta_scan.py,sha256=5R2XDiBZw0xd4ugbDxuyk6fztGlT5jFsGN85hLvo-hQ,17390
@@ -20,18 +38,19 @@ magic_pdf/layout/layout_det_utils.py,sha256=NCYBTvsrULE3Cue53aMD1MfXTmOL9Xy0nivl
20
38
  magic_pdf/layout/layout_sort.py,sha256=jtacQVcxnuYAksvEqtS0DH-v6U8qyjX-jmyZgDJ-egA,37005
21
39
  magic_pdf/layout/layout_spiler_recog.py,sha256=QjBSgB-a7J2yjUR1eaCs9ZD7URtiRnV6W934hpAeuC4,3067
22
40
  magic_pdf/layout/mcol_sort.py,sha256=ADnLisBJBHXDKYChcf2lzTb_TC_vZ4q89_CSN8mwEJc,11331
23
- magic_pdf/libs/Constants.py,sha256=rdJVadmgN0UlIB-xcMQ9j7Qk9q1Qahxt3KEY-vL7hSU,774
24
- magic_pdf/libs/MakeContentConfig.py,sha256=UDZPpsv8q4DqTy8h0vRtrT2kHqWiVI205VnVhlUEQc0,206
41
+ magic_pdf/libs/Constants.py,sha256=u9i-ivxxcGp7Hu_zvrLA1jdPcEnYhPb7wXOSUaOSwGQ,1140
42
+ magic_pdf/libs/MakeContentConfig.py,sha256=Do5VKNQp3gfUKyhrZStfzfBj7l-vbsYpsJFF1SsmEc0,248
25
43
  magic_pdf/libs/ModelBlockTypeEnum.py,sha256=kalXPbo5ya6hKhhBHPGlHl1yjWOURoXZWQM3rVUyPsY,164
26
44
  magic_pdf/libs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
27
- magic_pdf/libs/boxbase.py,sha256=h6TUcVZPvfyMm3nfnSNIb9cdAZhgfhAXCHz4SxrI2L4,15308
45
+ magic_pdf/libs/boxbase.py,sha256=ELMHWolgWROxOAQDgwmL7VS5kveZp4ifvEzRmPul2Ws,16925
28
46
  magic_pdf/libs/calc_span_stats.py,sha256=5vnU27DcbkFDRSAoLqAmX0KQ3I9ehWkEgh_t9hxg_zI,10147
47
+ magic_pdf/libs/clean_memory.py,sha256=BIOmEWuwR7c_p4OwTSW2muE3PRaGhmOplS-wTXt_EXk,211
29
48
  magic_pdf/libs/commons.py,sha256=6Zu9-OyamyCNDY7qj0SxR-rux-ggj9im3CVPtC4ubB8,7108
30
- magic_pdf/libs/config_reader.py,sha256=dPx6JJJuCw9AzNgKtrTG1elmfdeN6gDhgFK9r15-NsE,2505
49
+ magic_pdf/libs/config_reader.py,sha256=0uMOTSZE2crvqdGj2j21tlwV0nNRZha44Y9X98gQ3AQ,4067
31
50
  magic_pdf/libs/convert_utils.py,sha256=Ov-lsfCLBPz_15iSJXIslBNmrSf_E_1g_XDWJy8NgO8,143
32
51
  magic_pdf/libs/coordinate_transform.py,sha256=Bbop2cP2uz2ZG0U0gwd7J6EKkgABq5Rv03qf2LMPw80,429
33
52
  magic_pdf/libs/detect_language_from_model.py,sha256=Uln8F9qs8EJOw4EgI7KRlaU3lD_mK8KMTlADLFtz8fk,816
34
- magic_pdf/libs/draw_bbox.py,sha256=sbpN4LJ19k5Fh4NsY79mrLEc1-9XIoyoZJb9hL9A5b4,13703
53
+ magic_pdf/libs/draw_bbox.py,sha256=J_RaG6sRkqNyUd75NiWbDk-3LIE8es-zNIbcwR9myvo,17196
35
54
  magic_pdf/libs/drop_reason.py,sha256=IfjPSrPLMmVziqjOXPep7r_ioQKFRahDgbOW1SD-Tuw,2148
36
55
  magic_pdf/libs/drop_tag.py,sha256=bZDg3bIVWvBT1Ec1icwj5WLOkt5-hI6eRYZ2tX9_a74,673
37
56
  magic_pdf/libs/hash_utils.py,sha256=VEKK9WfFoZgrPfi8kfITjLpr8Ahufs8tXh9R1Y5lAL8,404
@@ -40,21 +59,21 @@ magic_pdf/libs/language.py,sha256=Hj5-lrGoNExxdHLbkcNG-c27U4AjJ9AZPdZblaNSehU,10
40
59
  magic_pdf/libs/local_math.py,sha256=tqljQOgqh3fZc146HYhO88JXJaiXMVwArBkk_CSGICc,177
41
60
  magic_pdf/libs/markdown_utils.py,sha256=cLxLXjRhrNp_wCHvtglrGA_FVdrvfd1KULeTtj1p18w,944
42
61
  magic_pdf/libs/nlp_utils.py,sha256=-X9W3-Ns5ZdDYFvyyEq6i6P2b5hCATaFEZeOjwNOH9M,6901
43
- magic_pdf/libs/ocr_content_type.py,sha256=xpPF5Unhiw9hPVlgsvPtFPpi7ReJo5dvZevluE3lfPc,852
62
+ magic_pdf/libs/ocr_content_type.py,sha256=e_7RBTdShaWvWhMO2SFou7GM521elMH_Jtn5usbHWdY,890
44
63
  magic_pdf/libs/path_utils.py,sha256=Hykw_l5CU736b2egHV9P7B-qh3QNKO4nZSGCbsi0Z8E,1043
45
64
  magic_pdf/libs/pdf_check.py,sha256=MAe8wzwT0qvPf_I72wEZG7k1g4haNHS7oUtLqkB5rlE,2145
46
65
  magic_pdf/libs/pdf_image_tools.py,sha256=CAd01giTKr_UJz1_QtDOARG9G9z69GFpzRZwcWSfLtE,1282
47
66
  magic_pdf/libs/safe_filename.py,sha256=ckwcM_eqoysTb5id8czp-tXq2G9da0-l3pshZDCHQtE,236
48
67
  magic_pdf/libs/textbase.py,sha256=SC1Frhz3Fb7V7n2SFRBsl7Bmg0JZdlvZskq0lfW1vIk,732
49
- magic_pdf/libs/version.py,sha256=iPlYCcIzuzW7T2HKDkmYlMkRI51dBLfNRxPPiWrfw9U,22
68
+ magic_pdf/libs/version.py,sha256=H9NWRZb7NbeRRPLP_V1fARmLNXranorVM-OOY-8_2ug,22
50
69
  magic_pdf/libs/vis_utils.py,sha256=hTOTEakKV0pGMbk0tbRkVI_tku7A3dGc96ynObZ4kwI,10207
51
70
  magic_pdf/model/__init__.py,sha256=1QcfMKET0xQhSaZMjNQHi_TjzSSDR6PI5mjkmaXHPe8,52
52
- magic_pdf/model/doc_analyze_by_custom_model.py,sha256=Ht1zZAB4WV3b-AWdRV5m5fuidlD6ZhNkvJM0j9i1l_E,4809
53
- magic_pdf/model/magic_model.py,sha256=Ze_rb4TQBI53_9-n37F6q6f965CT6r9qemi6rreicLU,28926
71
+ magic_pdf/model/doc_analyze_by_custom_model.py,sha256=eYrtOIlFqw8O95ShoCTaAhLBHk7TXc5DGif93VikW4s,6977
72
+ magic_pdf/model/magic_model.py,sha256=RKJOruUGAV1lHcGqSlCDbkJn5kutb3fphDreOHASPQg,43505
54
73
  magic_pdf/model/model_list.py,sha256=tJ9jtMB93HGx8Rmt8wmQSDFXZBUIPQrwaaYsep4luTM,183
55
- magic_pdf/model/pdf_extract_kit.py,sha256=Bdxqo3AGXs0VByFVj6ZEOm4T6wXTZwkZsRRFtxasNQM,17901
56
- magic_pdf/model/ppTableModel.py,sha256=wWiui9VOjkKYlNX-viPqsWpzgkNJ-9_S2Se-j4oyLqU,2687
57
- magic_pdf/model/pp_structure_v2.py,sha256=1sn8IJK0d5ZmqJ2XFt9FdaSdI0RQf-iwNAWBrVrIeuc,2872
74
+ magic_pdf/model/pdf_extract_kit.py,sha256=qlZANx8DErfSyaPHlOYNYW_Qp50dAEX_4jG8N3coDmM,21317
75
+ magic_pdf/model/ppTableModel.py,sha256=Qm5vy6v5aw2wwO5aZTyVr-r1sr3Pi9ManG86WZvfvEo,2697
76
+ magic_pdf/model/pp_structure_v2.py,sha256=BKPN7W4BjG0eWPAPjPEac1RMnb5eIzmAz4E4Rq-9b1U,3019
58
77
  magic_pdf/model/pek_sub_modules/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
59
78
  magic_pdf/model/pek_sub_modules/post_process.py,sha256=HzRxV2sVR3Qo8XKYEHhT6tae-bYTb6dnAfGP6gfVNaM,1135
60
79
  magic_pdf/model/pek_sub_modules/self_modify.py,sha256=NGUr8t4bXSeh38hwrfs6qxhf0IW-f3J96bnrg1xw8BA,14281
@@ -78,8 +97,10 @@ magic_pdf/model/pek_sub_modules/layoutlmv3/layoutlmft/models/layoutlmv3/configur
78
97
  magic_pdf/model/pek_sub_modules/layoutlmv3/layoutlmft/models/layoutlmv3/modeling_layoutlmv3.py,sha256=mdo8tO-DrJcv0Lbk9Pp98n3NQXYOnFFyXQWjU7t35kA,54633
79
98
  magic_pdf/model/pek_sub_modules/layoutlmv3/layoutlmft/models/layoutlmv3/tokenization_layoutlmv3.py,sha256=diKlrfxYjKAmYrUgjYdx-FXLh-swShC3tl-EBX1b3oI,1197
80
99
  magic_pdf/model/pek_sub_modules/layoutlmv3/layoutlmft/models/layoutlmv3/tokenization_layoutlmv3_fast.py,sha256=0lxiG69_fGpSSBYA9CBLnDa_qqa1rInZ0pJpqBwZ0Yw,1372
81
- magic_pdf/model/pek_sub_modules/structeqtable/StructTableModel.py,sha256=C9WluPhwaqsFg154WsNxN2HlhFXVkAAw0prR7t8r5J4,918
100
+ magic_pdf/model/pek_sub_modules/structeqtable/StructTableModel.py,sha256=qtAkShYlXBrrkRWHvgAy3y9SEBtMRYVIvI3CASTuLHU,1069
82
101
  magic_pdf/model/pek_sub_modules/structeqtable/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
102
+ magic_pdf/model/v3/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
103
+ magic_pdf/model/v3/helpers.py,sha256=IVUFcNMDF3-kio-BIxjppHnWS3eHPqvvNihIw2fbIFM,4372
83
104
  magic_pdf/para/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
84
105
  magic_pdf/para/block_continuation_processor.py,sha256=IkReB5hirjm0OAirNzQQpnWe2f2bdP3Hug3Ef8qTRDA,22749
85
106
  magic_pdf/para/block_termination_processor.py,sha256=YU3ZYqJy9e3OQmOuQYZrR6AUpmAlQ0mhj0PgZZPZ_fM,17957
@@ -91,13 +112,14 @@ magic_pdf/para/layout_match_processor.py,sha256=yr4FEO7GJ502udShqGRqIJQ_FQxoa0aG
91
112
  magic_pdf/para/para_pipeline.py,sha256=zLaCHI9jLi1UPzh0lHP44mUjpKVTHS0gE_5YrkjVqEY,11796
92
113
  magic_pdf/para/para_split.py,sha256=-UJM2jREW_2h3ZlJAU7dRD8bK3CMGKuhJrfgqv3Auvk,31310
93
114
  magic_pdf/para/para_split_v2.py,sha256=ZIiLzpvVL364x1zcEG9IbT6ARJ-6JnWLIVrsDmf4w1M,36878
115
+ magic_pdf/para/para_split_v3.py,sha256=vHHswSAcTpXqnaEAbGEbt2g96YLh9eh839HdRNilDT8,13378
94
116
  magic_pdf/para/raw_processor.py,sha256=mHxD9FrdOSXH7NqM41s55URyCyuyACvm9kKtowkIb3k,6317
95
117
  magic_pdf/para/stats.py,sha256=-6Pf9Y8jkP1uJOYWiHUjw9Lb-Fb9GY7MHr_ok7x2GX0,9731
96
118
  magic_pdf/para/title_processor.py,sha256=pYZv9vEkIjAtCz8jIUtl9AVUy_ib5SdAZmMVoZtsMRI,38593
97
- magic_pdf/pipe/AbsPipe.py,sha256=btbCoH30x5ECzOwoYdMn-MQQPhMOB6IaoDriC0M5oe8,4265
98
- magic_pdf/pipe/OCRPipe.py,sha256=rCff7CPMLDGDzF4SfaLKeUk-6PNXVkzWRo96hS7XqSM,1605
99
- magic_pdf/pipe/TXTPipe.py,sha256=AiO6yAq8XpT66jP3bMaNH2L97y3jRr10xQFbx5VzrxE,1664
100
- magic_pdf/pipe/UNIPipe.py,sha256=X8paSZTvxgKHxUcVLB2LFMoPhdk215zHKXSNOSGOjeg,4101
119
+ magic_pdf/pipe/AbsPipe.py,sha256=jpJD-8S5K89bCxI4FY2_tMKGV1lghlbtmpOeJXqEhz0,4487
120
+ magic_pdf/pipe/OCRPipe.py,sha256=TswwfMkAYP-fpfsBoNIArtcCoNEqzCQCVWfwR4n8G-E,2159
121
+ magic_pdf/pipe/TXTPipe.py,sha256=zcsiOreOHJBtaeYgEc5yGKMSiNzBED8HcsD7NGu5RaY,2218
122
+ magic_pdf/pipe/UNIPipe.py,sha256=3ytYHoyTboHUGlRQpfyITiAP_mYXTOBjqy7wYHoUHpE,4944
101
123
  magic_pdf/pipe/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
102
124
  magic_pdf/post_proc/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
103
125
  magic_pdf/post_proc/detect_para.py,sha256=5LX86ueHQGOV9CNimAxqZH4R3KTi78leum1de_Na0pw,126181
@@ -119,9 +141,9 @@ magic_pdf/pre_proc/equations_replace.py,sha256=fXj7ZV7F3YtkDYrAhE9g5tHk4_3pVUyLb
119
141
  magic_pdf/pre_proc/fix_image.py,sha256=5MOfkXc8abfIp49g-68vll40wwTUZ5tcQ2gtsJuFmvs,11486
120
142
  magic_pdf/pre_proc/fix_table.py,sha256=20sqJe27fAXcL7_C0qQ9mpsggmH37WuX-wPYWyRgACA,13227
121
143
  magic_pdf/pre_proc/main_text_font.py,sha256=1gkjvPuBdKC4oVFkLvnRm2zghsLtVlfAEMKXouyVonM,1048
122
- magic_pdf/pre_proc/ocr_detect_all_bboxes.py,sha256=vFT3bsqQ2_GvcYeFS4J4UQCxxwgEUFk5x3TxId6m8BE,7110
144
+ magic_pdf/pre_proc/ocr_detect_all_bboxes.py,sha256=omR2aOQNerKmKUF13CFP5-Z6Hv4GZztmsAQgLpHiugc,11341
123
145
  magic_pdf/pre_proc/ocr_detect_layout.py,sha256=DW0_HXzmcbW22cXKIYFsyZNFh8mEjSHXIFVjXndJsvQ,5878
124
- magic_pdf/pre_proc/ocr_dict_merge.py,sha256=8t81fXGHdVuJ5-AEKdyx3Q7JH3OMSNZ-GgsYWHl_VjE,13118
146
+ magic_pdf/pre_proc/ocr_dict_merge.py,sha256=KyXSW36lJ1PsK816J55vVQQtXTWODX1xG1_DvA7pOSo,14248
125
147
  magic_pdf/pre_proc/ocr_span_list_modify.py,sha256=jqBheXF8EuYCfS9tn6typr-aE57nfMoeBC36J5GjpbQ,11519
126
148
  magic_pdf/pre_proc/pdf_pre_filter.py,sha256=FIMwe8Lei9LI2RmkqiaSyTHV5b7ViADbpyBwgVwZH-c,2687
127
149
  magic_pdf/pre_proc/post_layout_split.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -133,8 +155,8 @@ magic_pdf/pre_proc/resolve_bbox_conflict.py,sha256=bJiegofPUeDyi--oZjfipQ5Q5RLm6
133
155
  magic_pdf/pre_proc/solve_line_alien.py,sha256=aNoQptPcC38Sm1I2ABhgw8jeH_5kjsRHx3VYlFFtm1g,853
134
156
  magic_pdf/pre_proc/statistics.py,sha256=_9jGlXq0iXd03UMxB92ZqCiu7cjNkG5vHvFlTF_9ytA,220
135
157
  magic_pdf/resources/fasttext-langdetect/lid.176.ftz,sha256=jzRyz-hzintgmejpmcPL-uDc0VaWqsfXc4qAOdtgPoM,938013
136
- magic_pdf/resources/model_config/model_configs.yaml,sha256=_gOSxK9jxe1bFwtH_uwovsyZnRi1sEVNYb1OAexDmF4,301
137
- magic_pdf/resources/model_config/UniMERNet/demo.yaml,sha256=al9_--m3n2j9zEn9OjlmmpfQbqVBAYFakXc_hY4vDXo,807
158
+ magic_pdf/resources/model_config/model_configs.yaml,sha256=muwPuO6C8Z5oEStE-wWEt7vmkenmJiqTbkRfUfle_vU,257
159
+ magic_pdf/resources/model_config/UniMERNet/demo.yaml,sha256=Jdaim2D2lAYrV9rhc1X5Sy2_IacGOrfysJhxEUgSElo,827
138
160
  magic_pdf/resources/model_config/layoutlmv3/layoutlmv3_base_inference.yaml,sha256=9aNAEYgpHTAWpcUrDvuPG2y4V-Qw8QdcJefi96y8yDU,6109
139
161
  magic_pdf/rw/AbsReaderWriter.py,sha256=2H5SDJfAAOX9kPfel06a8VRCHxD1Y8aPbWEkQDdn9JM,452
140
162
  magic_pdf/rw/DiskReaderWriter.py,sha256=7ZAekH8V6xlBo_1WeSZ6sNwAj2WGPtjNl50zq1CoMDY,2614
@@ -143,12 +165,14 @@ magic_pdf/rw/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
143
165
  magic_pdf/spark/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
144
166
  magic_pdf/spark/spark_api.py,sha256=eSLXTjMYW5Ya41VMIApRVfji1ZxEZXdH9ZdsL6fy5Kw,1131
145
167
  magic_pdf/tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
146
- magic_pdf/tools/cli.py,sha256=tUeJhGudJIrCDMNQDRTWtNGE_4E0TWpKTuEkf5y_2uk,2734
168
+ magic_pdf/tools/cli.py,sha256=yl2E-DYxBN3XF7bWOBseYxptbmeE7tXWpwV-sp2aGIE,3140
147
169
  magic_pdf/tools/cli_dev.py,sha256=3e5eyCQEt_EujXZu5fUAWr_W-YQQVqS9pB0Qgw7t1D8,4122
148
- magic_pdf/tools/common.py,sha256=uJkRb2T6pouALitPCXVD2FVEeHwq2NWTsGGeFigdb74,4859
149
- magic_pdf-0.8.0.dist-info/LICENSE.md,sha256=hIahDEOTzuHCU5J2nd07LWwkLW7Hko4UFO__ffsvB-8,34523
150
- magic_pdf-0.8.0.dist-info/METADATA,sha256=od0Sz7-Uq3vCjbkVGMiJem6pMWVW_S_1SCzgkeWTyFU,23839
151
- magic_pdf-0.8.0.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
152
- magic_pdf-0.8.0.dist-info/entry_points.txt,sha256=wXwYke3j8fqDQTocUspL-CqDUEv3Tfcwp09fM8dZAhA,98
153
- magic_pdf-0.8.0.dist-info/top_level.txt,sha256=J9I0AzmHWGkp9c6DL8Oe4mEx3yYphLzkRn4H25Lg1rE,10
154
- magic_pdf-0.8.0.dist-info/RECORD,,
170
+ magic_pdf/tools/common.py,sha256=2S8N60pcA6bFqAmdchoEmn22l9ntQxEfyaKpxfCKJ-Y,5465
171
+ magic_pdf/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
172
+ magic_pdf/utils/annotations.py,sha256=82ou3uELNbQWa9hOFFkVt0gsIskAKf5msCv5J2IJ5V0,211
173
+ magic_pdf-0.9.0.dist-info/LICENSE.md,sha256=jVa0BUaKrRH4erV2P5AeJ24I2WRv9chIGxditreJ6e0,34524
174
+ magic_pdf-0.9.0.dist-info/METADATA,sha256=DPanG2IP5v1TNR6Qyto-UqZ53IOA09lNCQpMyjguJ_k,39420
175
+ magic_pdf-0.9.0.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
176
+ magic_pdf-0.9.0.dist-info/entry_points.txt,sha256=wXwYke3j8fqDQTocUspL-CqDUEv3Tfcwp09fM8dZAhA,98
177
+ magic_pdf-0.9.0.dist-info/top_level.txt,sha256=J9I0AzmHWGkp9c6DL8Oe4mEx3yYphLzkRn4H25Lg1rE,10
178
+ magic_pdf-0.9.0.dist-info/RECORD,,
@@ -1,459 +0,0 @@
1
- Metadata-Version: 2.1
2
- Name: magic-pdf
3
- Version: 0.8.0
4
- Summary: A practical tool for converting PDF to Markdown
5
- Home-page: https://github.com/opendatalab/MinerU
6
- Requires-Python: >=3.9
7
- Description-Content-Type: text/markdown
8
- License-File: LICENSE.md
9
- Requires-Dist: boto3>=1.28.43
10
- Requires-Dist: Brotli>=1.1.0
11
- Requires-Dist: click>=8.1.7
12
- Requires-Dist: fast-langdetect==0.2.0
13
- Requires-Dist: loguru>=0.6.0
14
- Requires-Dist: numpy<2.0.0,>=1.21.6
15
- Requires-Dist: pdfminer.six==20231228
16
- Requires-Dist: pydantic<2.8.0,>=2.7.2
17
- Requires-Dist: PyMuPDF>=1.24.9
18
- Requires-Dist: scikit-learn>=1.0.2
19
- Requires-Dist: wordninja>=2.0.0
20
- Provides-Extra: full
21
- Requires-Dist: unimernet==0.1.6; extra == "full"
22
- Requires-Dist: ultralytics; extra == "full"
23
- Requires-Dist: paddleocr==2.7.3; extra == "full"
24
- Requires-Dist: pypandoc; extra == "full"
25
- Requires-Dist: struct-eqtable==0.1.0; extra == "full"
26
- Requires-Dist: detectron2; extra == "full"
27
- Requires-Dist: paddlepaddle==3.0.0b1; platform_system == "Linux" and extra == "full"
28
- Requires-Dist: matplotlib; (platform_system == "Linux" or platform_system == "Darwin") and extra == "full"
29
- Requires-Dist: matplotlib<=3.9.0; platform_system == "Windows" and extra == "full"
30
- Requires-Dist: paddlepaddle==2.6.1; (platform_system == "Windows" or platform_system == "Darwin") and extra == "full"
31
- Provides-Extra: lite
32
- Requires-Dist: paddleocr==2.7.3; extra == "lite"
33
- Requires-Dist: paddlepaddle==3.0.0b1; platform_system == "Linux" and extra == "lite"
34
- Requires-Dist: paddlepaddle==2.6.1; (platform_system == "Windows" or platform_system == "Darwin") and extra == "lite"
35
-
36
- <div align="center" xmlns="http://www.w3.org/1999/html">
37
- <!-- logo -->
38
- <p align="center">
39
- <img src="docs/images/MinerU-logo.png" width="300px" style="vertical-align:middle;">
40
- </p>
41
-
42
- <!-- icon -->
43
-
44
- [![stars](https://img.shields.io/github/stars/opendatalab/MinerU.svg)](https://github.com/opendatalab/MinerU)
45
- [![forks](https://img.shields.io/github/forks/opendatalab/MinerU.svg)](https://github.com/opendatalab/MinerU)
46
- [![open issues](https://img.shields.io/github/issues-raw/opendatalab/MinerU)](https://github.com/opendatalab/MinerU/issues)
47
- [![issue resolution](https://img.shields.io/github/issues-closed-raw/opendatalab/MinerU)](https://github.com/opendatalab/MinerU/issues)
48
- [![PyPI version](https://badge.fury.io/py/magic-pdf.svg)](https://badge.fury.io/py/magic-pdf)
49
- [![Downloads](https://static.pepy.tech/badge/magic-pdf)](https://pepy.tech/project/magic-pdf)
50
- [![Downloads](https://static.pepy.tech/badge/magic-pdf/month)](https://pepy.tech/project/magic-pdf)
51
-
52
- [![HuggingFace](https://img.shields.io/badge/HuggingFace-Demo-yellow.svg?logo=data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAF8AAABYCAMAAACkl9t/AAAAk1BMVEVHcEz/nQv/nQv/nQr/nQv/nQr/nQv/nQv/nQr/wRf/txT/pg7/yRr/rBD/zRz/ngv/oAz/zhz/nwv/txT/ngv/0B3+zBz/nQv/0h7/wxn/vRb/thXkuiT/rxH/pxD/ogzcqyf/nQvTlSz/czCxky7/SjifdjT/Mj3+Mj3wMj15aTnDNz+DSD9RTUBsP0FRO0Q6O0WyIxEIAAAAGHRSTlMADB8zSWF3krDDw8TJ1NbX5efv8ff9/fxKDJ9uAAAGKklEQVR42u2Z63qjOAyGC4RwCOfB2JAGqrSb2WnTw/1f3UaWcSGYNKTdf/P+mOkTrE+yJBulvfvLT2A5ruenaVHyIks33npl/6C4s/ZLAM45SOi/1FtZPyFur1OYofBX3w7d54Bxm+E8db+nDr12ttmESZ4zludJEG5S7TO72YPlKZFyE+YCYUJTBZsMiNS5Sd7NlDmKM2Eg2JQg8awbglfqgbhArjxkS7dgp2RH6hc9AMLdZYUtZN5DJr4molC8BfKrEkPKEnEVjLbgW1fLy77ZVOJagoIcLIl+IxaQZGjiX597HopF5CkaXVMDO9Pyix3AFV3kw4lQLCbHuMovz8FallbcQIJ5Ta0vks9RnolbCK84BtjKRS5uA43hYoZcOBGIG2Epbv6CvFVQ8m8loh66WNySsnN7htL58LNp+NXT8/PhXiBXPMjLSxtwp8W9f/1AngRierBkA+kk/IpUSOeKByzn8y3kAAAfh//0oXgV4roHm/kz4E2z//zRc3/lgwBzbM2mJxQEa5pqgX7d1L0htrhx7LKxOZlKbwcAWyEOWqYSI8YPtgDQVjpB5nvaHaSnBaQSD6hweDi8PosxD6/PT09YY3xQA7LTCTKfYX+QHpA0GCcqmEHvr/cyfKQTEuwgbs2kPxJEB0iNjfJcCTPyocx+A0griHSmADiC91oNGVwJ69RudYe65vJmoqfpul0lrqXadW0jFKH5BKwAeCq+Den7s+3zfRJzA61/Uj/9H/VzLKTx9jFPPdXeeP+L7WEvDLAKAIoF8bPTKT0+TM7W8ePj3Rz/Yn3kOAp2f1Kf0Weony7pn/cPydvhQYV+eFOfmOu7VB/ViPe34/EN3RFHY/yRuT8ddCtMPH/McBAT5s+vRde/gf2c/sPsjLK+m5IBQF5tO+h2tTlBGnP6693JdsvofjOPnnEHkh2TnV/X1fBl9S5zrwuwF8NFrAVJVwCAPTe8gaJlomqlp0pv4Pjn98tJ/t/fL++6unpR1YGC2n/KCoa0tTLoKiEeUPDl94nj+5/Tv3/eT5vBQ60X1S0oZr+IWRR8Ldhu7AlLjPISlJcO9vrFotky9SpzDequlwEir5beYAc0R7D9KS1DXva0jhYRDXoExPdc6yw5GShkZXe9QdO/uOvHofxjrV/TNS6iMJS+4TcSTgk9n5agJdBQbB//IfF/HpvPt3Tbi7b6I6K0R72p6ajryEJrENW2bbeVUGjfgoals4L443c7BEE4mJO2SpbRngxQrAKRudRzGQ8jVOL2qDVjjI8K1gc3TIJ5KiFZ1q+gdsARPB4NQS4AjwVSt72DSoXNyOWUrU5mQ9nRYyjp89Xo7oRI6Bga9QNT1mQ/ptaJq5T/7WcgAZywR/XlPGAUDdet3LE+qS0TI+g+aJU8MIqjo0Kx8Ly+maxLjJmjQ18rA0YCkxLQbUZP1WqdmyQGJLUm7VnQFqodmXSqmRrdVpqdzk5LvmvgtEcW8PMGdaS23EOWyDVbACZzUJPaqMbjDxpA3Qrgl0AikimGDbqmyT8P8NOYiqrldF8rX+YN7TopX4UoHuSCYY7cgX4gHwclQKl1zhx0THf+tCAUValzjI7Wg9EhptrkIcfIJjA94evOn8B2eHaVzvBrnl2ig0So6hvPaz0IGcOvTHvUIlE2+prqAxLSQxZlU2stql1NqCCLdIiIN/i1DBEHUoElM9dBravbiAnKqgpi4IBkw+utSPIoBijDXJipSVV7MpOEJUAc5Qmm3BnUN+w3hteEieYKfRZSIUcXKMVf0u5wD4EwsUNVvZOtUT7A2GkffHjByWpHqvRBYrTV72a6j8zZ6W0DTE86Hn04bmyWX3Ri9WH7ZU6Q7h+ZHo0nHUAcsQvVhXRDZHChwiyi/hnPuOsSEF6Exk3o6Y9DT1eZ+6cASXk2Y9k+6EOQMDGm6WBK10wOQJCBwren86cPPWUcRAnTVjGcU1LBgs9FURiX/e6479yZcLwCBmTxiawEwrOcleuu12t3tbLv/N4RLYIBhYexm7Fcn4OJcn0+zc+s8/VfPeddZHAGN6TT8eGczHdR/Gts1/MzDkThr23zqrVfAMFT33Nx1RJsx1k5zuWILLnG/vsH+Fv5D4NTVcp1Gzo8AAAAAElFTkSuQmCC)](https://huggingface.co/spaces/opendatalab/MinerU)
53
- [![ModelScope](https://img.shields.io/badge/ModelScope-Demo-purple?logo=data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iMjIzIiBoZWlnaHQ9IjIwMCIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj4KCiA8Zz4KICA8dGl0bGU+TGF5ZXIgMTwvdGl0bGU+CiAgPHBhdGggaWQ9InN2Z18xNCIgZmlsbD0iIzYyNGFmZiIgZD0ibTAsODkuODRsMjUuNjUsMGwwLDI1LjY0OTk5bC0yNS42NSwwbDAsLTI1LjY0OTk5eiIvPgogIDxwYXRoIGlkPSJzdmdfMTUiIGZpbGw9IiM2MjRhZmYiIGQ9Im05OS4xNCwxMTUuNDlsMjUuNjUsMGwwLDI1LjY1bC0yNS42NSwwbDAsLTI1LjY1eiIvPgogIDxwYXRoIGlkPSJzdmdfMTYiIGZpbGw9IiM2MjRhZmYiIGQ9Im0xNzYuMDksMTQxLjE0bC0yNS42NDk5OSwwbDAsMjIuMTlsNDcuODQsMGwwLC00Ny44NGwtMjIuMTksMGwwLDI1LjY1eiIvPgogIDxwYXRoIGlkPSJzdmdfMTciIGZpbGw9IiMzNmNmZDEiIGQ9Im0xMjQuNzksODkuODRsMjUuNjUsMGwwLDI1LjY0OTk5bC0yNS42NSwwbDAsLTI1LjY0OTk5eiIvPgogIDxwYXRoIGlkPSJzdmdfMTgiIGZpbGw9IiMzNmNmZDEiIGQ9Im0wLDY0LjE5bDI1LjY1LDBsMCwyNS42NWwtMjUuNjUsMGwwLC0yNS42NXoiLz4KICA8cGF0aCBpZD0ic3ZnXzE5IiBmaWxsPSIjNjI0YWZmIiBkPSJtMTk4LjI4LDg5Ljg0bDI1LjY0OTk5LDBsMCwyNS42NDk5OWwtMjUuNjQ5OTksMGwwLC0yNS42NDk5OXoiLz4KICA8cGF0aCBpZD0ic3ZnXzIwIiBmaWxsPSIjMzZjZmQxIiBkPSJtMTk4LjI4LDY0LjE5bDI1LjY0OTk5LDBsMCwyNS42NWwtMjUuNjQ5OTksMGwwLC0yNS42NXoiLz4KICA8cGF0aCBpZD0ic3ZnXzIxIiBmaWxsPSIjNjI0YWZmIiBkPSJtMTUwLjQ0LDQybDAsMjIuMTlsMjUuNjQ5OTksMGwwLDI1LjY1bDIyLjE5LDBsMCwtNDcuODRsLTQ3Ljg0LDB6Ii8+CiAgPHBhdGggaWQ9InN2Z18yMiIgZmlsbD0iIzM2Y2ZkMSIgZD0ibTczLjQ5LDg5Ljg0bDI1LjY1LDBsMCwyNS42NDk5OWwtMjUuNjUsMGwwLC0yNS42NDk5OXoiLz4KICA8cGF0aCBpZD0ic3ZnXzIzIiBmaWxsPSIjNjI0YWZmIiBkPSJtNDcuODQsNjQuMTlsMjUuNjUsMGwwLC0yMi4xOWwtNDcuODQsMGwwLDQ3Ljg0bDIyLjE5LDBsMCwtMjUuNjV6Ii8+CiAgPHBhdGggaWQ9InN2Z18yNCIgZmlsbD0iIzYyNGFmZiIgZD0ibTQ3Ljg0LDExNS40OWwtMjIuMTksMGwwLDQ3Ljg0bDQ3Ljg0LDBsMCwtMjIuMTlsLTI1LjY1LDBsMCwtMjUuNjV6Ii8+CiA8L2c+Cjwvc3ZnPg==&labelColor=white)](https://www.modelscope.cn/studios/OpenDataLab/MinerU)
54
- [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/gist/papayalove/b5f4913389e7ff9883c6b687de156e78/mineru_demo.ipynb)
55
- [![Paper](https://img.shields.io/badge/Paper-arXiv-green)](#)
56
-
57
- <a href="https://trendshift.io/repositories/11174" target="_blank"><img src="https://trendshift.io/api/badge/repositories/11174" alt="opendatalab%2FMinerU | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
58
-
59
- <!-- language -->
60
-
61
- [English](README.md) | [简体中文](README_zh-CN.md)
62
-
63
- <!-- hot link -->
64
-
65
- <p align="center">
66
- <a href="https://github.com/opendatalab/PDF-Extract-Kit">PDF-Extract-Kit: High-Quality PDF Extraction Toolkit</a>🔥🔥🔥
67
- </p>
68
-
69
- <!-- join us -->
70
-
71
- <p align="center">
72
- 👋 join us on <a href="https://discord.gg/Tdedn9GTXq" target="_blank">Discord</a> and <a href="https://cdn.vansin.top/internlm/mineru.jpg" target="_blank">WeChat</a>
73
- </p>
74
-
75
- </div>
76
-
77
- # Changelog
78
- - 2024/09/09: Version 0.8.0 released, supporting fast deployment with Dockerfile, and launching demos on Huggingface and Modelscope.
79
- - 2024/08/30: Version 0.7.1 released, add paddle tablemaster table recognition option
80
- - 2024/08/09: Version 0.7.0b1 released, simplified installation process, added table recognition functionality
81
- - 2024/08/01: Version 0.6.2b1 released, optimized dependency conflict issues and installation documentation
82
- - 2024/07/05: Initial open-source release
83
-
84
- <!-- TABLE OF CONTENT -->
85
-
86
- <details open="open">
87
- <summary><h2 style="display: inline-block">Table of Contents</h2></summary>
88
- <ol>
89
- <li>
90
- <a href="#mineru">MinerU</a>
91
- <ul>
92
- <li><a href="#project-introduction">Project Introduction</a></li>
93
- <li><a href="#key-features">Key Features</a></li>
94
- <li><a href="#quick-start">Quick Start</a>
95
- <ul>
96
- <li><a href="#online-demo">Online Demo</a></li>
97
- <li><a href="#quick-cpu-demo">Quick CPU Demo</a></li>
98
- <li><a href="#using-gpu">Using GPU</a></li>
99
- </ul>
100
- </li>
101
- <li><a href="#usage">Usage</a>
102
- <ul>
103
- <li><a href="#command-line">Command Line</a></li>
104
- <li><a href="#api">API</a></li>
105
- <li><a href="#development-guide">Development Guide</a></li>
106
- </ul>
107
- </li>
108
- </ul>
109
- </li>
110
- <li><a href="#todo">TODO</a></li>
111
- <li><a href="#known-issues">Known Issues</a></li>
112
- <li><a href="#faq">FAQ</a></li>
113
- <li><a href="#all-thanks-to-our-contributors">All Thanks To Our Contributors</a></li>
114
- <li><a href="#license-information">License Information</a></li>
115
- <li><a href="#acknowledgments">Acknowledgments</a></li>
116
- <li><a href="#citation">Citation</a></li>
117
- <li><a href="#star-history">Star History</a></li>
118
- <li><a href="#magic-doc">Magic-doc</a></li>
119
- <li><a href="#magic-html">Magic-html</a></li>
120
- <li><a href="#links">Links</a></li>
121
- </ol>
122
- </details>
123
-
124
- # MinerU
125
-
126
- ## Project Introduction
127
-
128
- MinerU is a tool that converts PDFs into machine-readable formats (e.g., markdown, JSON), allowing for easy extraction into any format.
129
- MinerU was born during the pre-training process of [InternLM](https://github.com/InternLM/InternLM). We focus on solving symbol conversion issues in scientific literature and hope to contribute to technological development in the era of large models.
130
- Compared to well-known commercial products, MinerU is still young. If you encounter any issues or if the results are not as expected, please submit an issue on [issue](https://github.com/opendatalab/MinerU/issues) and **attach the relevant PDF**.
131
-
132
- https://github.com/user-attachments/assets/4bea02c9-6d54-4cd6-97ed-dff14340982c
133
-
134
- ## Key Features
135
-
136
- - Removes elements such as headers, footers, footnotes, and page numbers while maintaining semantic continuity
137
- - Outputs text in a human-readable order from multi-column documents
138
- - Retains the original structure of the document, including titles, paragraphs, and lists
139
- - Extracts images, image captions, tables, and table captions
140
- - Automatically recognizes formulas in the document and converts them to LaTeX
141
- - Automatically recognizes tables in the document and converts them to LaTeX
142
- - Automatically detects and enables OCR for corrupted PDFs
143
- - Supports both CPU and GPU environments
144
- - Supports Windows, Linux, and Mac platforms
145
-
146
- ## Quick Start
147
-
148
- If you encounter any installation issues, please first consult the <a href="#faq">FAQ</a>. </br>
149
- If the parsing results are not as expected, refer to the <a href="#known-issues">Known Issues</a>. </br>
150
- There are three different ways to experience MinerU:
151
-
152
- - [Online Demo (No Installation Required)](#online-demo)
153
- - [Quick CPU Demo (Windows, Linux, Mac)](#quick-cpu-demo)
154
- - [Linux/Windows + CUDA](#Using-GPU)
155
-
156
- **⚠️ Pre-installation Notice—Hardware and Software Environment Support**
157
-
158
- To ensure the stability and reliability of the project, we only optimize and test for specific hardware and software environments during development. This ensures that users deploying and running the project on recommended system configurations will get the best performance with the fewest compatibility issues.
159
-
160
- By focusing resources on the mainline environment, our team can more efficiently resolve potential bugs and develop new features.
161
-
162
- In non-mainline environments, due to the diversity of hardware and software configurations, as well as third-party dependency compatibility issues, we cannot guarantee 100% project availability. Therefore, for users who wish to use this project in non-recommended environments, we suggest carefully reading the documentation and FAQ first. Most issues already have corresponding solutions in the FAQ. We also encourage community feedback to help us gradually expand support.
163
-
164
- <table>
165
- <tr>
166
- <td colspan="3" rowspan="2">Operating System</td>
167
- </tr>
168
- <tr>
169
- <td>Ubuntu 22.04 LTS</td>
170
- <td>Windows 10 / 11</td>
171
- <td>macOS 11+</td>
172
- </tr>
173
- <tr>
174
- <td colspan="3">CPU</td>
175
- <td>x86_64</td>
176
- <td>x86_64</td>
177
- <td>x86_64 / arm64</td>
178
- </tr>
179
- <tr>
180
- <td colspan="3">Memory</td>
181
- <td colspan="3">16GB or more, recommended 32GB+</td>
182
- </tr>
183
- <tr>
184
- <td colspan="3">Python Version</td>
185
- <td colspan="3">3.10</td>
186
- </tr>
187
- <tr>
188
- <td colspan="3">Nvidia Driver Version</td>
189
- <td>latest (Proprietary Driver)</td>
190
- <td>latest</td>
191
- <td>None</td>
192
- </tr>
193
- <tr>
194
- <td colspan="3">CUDA Environment</td>
195
- <td>Automatic installation [12.1 (pytorch) + 11.8 (paddle)]</td>
196
- <td>11.8 (manual installation) + cuDNN v8.7.0 (manual installation)</td>
197
- <td>None</td>
198
- </tr>
199
- <tr>
200
- <td rowspan="2">GPU Hardware Support List</td>
201
- <td colspan="2">Minimum Requirement 8G+ VRAM</td>
202
- <td colspan="2">3060ti/3070/3080/3080ti/4060/4070/4070ti<br>
203
- 8G VRAM only enables layout and formula recognition acceleration</td>
204
- <td rowspan="2">None</td>
205
- </tr>
206
- <tr>
207
- <td colspan="2">Recommended Configuration 16G+ VRAM</td>
208
- <td colspan="2">3090/3090ti/4070ti super/4080/4090<br>
209
- 16G or more can enable layout, formula recognition, and OCR acceleration simultaneously<br>
210
- 24G or more can enable layout, formula recognition, OCR acceleration and table recognition simultaneously
211
- </td>
212
- </tr>
213
- </table>
214
-
215
- ### Online Demo
216
-
217
- [Click here for the online demo](https://opendatalab.com/OpenSourceTools/Extractor/PDF)
218
-
219
- ### Quick CPU Demo
220
-
221
- #### 1. Install magic-pdf
222
-
223
- ```bash
224
- conda create -n MinerU python=3.10
225
- conda activate MinerU
226
- pip install -U magic-pdf[full] --extra-index-url https://wheels.myhloli.com
227
- ```
228
-
229
- #### 2. Download model weight files
230
-
231
- Refer to [How to Download Model Files](docs/how_to_download_models_en.md) for detailed instructions.
232
-
233
- > ❗️After downloading the models, please make sure to verify the completeness of the model files.
234
- >
235
- > Check if the model file sizes match the description on the webpage. If possible, use sha256 to verify the integrity of the files.
236
-
237
- #### 3. Copy and configure the template file
238
-
239
- You can find the `magic-pdf.template.json` template configuration file in the root directory of the repository.
240
-
241
- > ❗️Make sure to execute the following command to copy the configuration file to your **user directory**; otherwise, the program will not run.
242
- >
243
- > The user directory for Windows is `C:\Users\YourUsername`, for Linux it is `/home/YourUsername`, and for macOS it is `/Users/YourUsername`.
244
-
245
- ```bash
246
- cp magic-pdf.template.json ~/magic-pdf.json
247
- ```
248
-
249
- Find the `magic-pdf.json` file in your user directory and configure the "models-dir" path to point to the directory where the model weight files were downloaded in [Step 2](#2-download-model-weight-files).
250
-
251
- > ❗️Make sure to correctly configure the **absolute path** to the model weight files directory, otherwise the program will not run because it can't find the model files.
252
- >
253
- > On Windows, this path should include the drive letter and all backslashes (`\`) in the path should be replaced with forward slashes (`/`) to avoid syntax errors in the JSON file due to escape sequences.
254
- >
255
- > For example: If the models are stored in the "models" directory at the root of the D drive, the "model-dir" value should be `D:/models`.
256
-
257
- ```json
258
- {
259
- // other config
260
- "models-dir": "D:/models",
261
- "table-config": {
262
- "model": "TableMaster", // Another option of this value is 'struct_eqtable'
263
- "is_table_recog_enable": false, // Table recognition is disabled by default, modify this value to enable it
264
- "max_time": 400
265
- }
266
- }
267
- ```
268
-
269
- ### Using GPU
270
-
271
- If your device supports CUDA and meets the GPU requirements of the mainline environment, you can use GPU acceleration. Please select the appropriate guide based on your system:
272
-
273
- - [Ubuntu 22.04 LTS + GPU](docs/README_Ubuntu_CUDA_Acceleration_en_US.md)
274
- - [Windows 10/11 + GPU](docs/README_Windows_CUDA_Acceleration_en_US.md)
275
- - Quick Deployment with Docker
276
- > Docker requires a GPU with at least 16GB of VRAM, and all acceleration features are enabled by default.
277
- >
278
- > Before running this Docker, you can use the following command to check if your device supports CUDA acceleration on Docker.
279
- >
280
- > ```bash
281
- > docker run --rm --gpus=all nvidia/cuda:12.1.0-base-ubuntu22.04 nvidia-smi
282
- > ```
283
- ```bash
284
- wget https://github.com/opendatalab/MinerU/raw/master/Dockerfile
285
- docker build -t mineru:latest .
286
- docker run --rm -it --gpus=all mineru:latest /bin/bash
287
- magic-pdf --help
288
- ```
289
-
290
- ## Usage
291
-
292
- ### Command Line
293
-
294
- ```bash
295
- magic-pdf --help
296
- Usage: magic-pdf [OPTIONS]
297
-
298
- Options:
299
- -v, --version display the version and exit
300
- -p, --path PATH local pdf filepath or directory [required]
301
- -o, --output-dir TEXT output local directory
302
- -m, --method [ocr|txt|auto] the method for parsing pdf.
303
- ocr: using ocr technique to extract information from pdf,
304
- txt: suitable for the text-based pdf only and outperform ocr,
305
- auto: automatically choose the best method for parsing pdf
306
- from ocr and txt.
307
- without method specified, auto will be used by default.
308
- --help Show this message and exit.
309
-
310
-
311
- ## show version
312
- magic-pdf -v
313
-
314
- ## command line example
315
- magic-pdf -p {some_pdf} -o {some_output_dir} -m auto
316
- ```
317
-
318
- `{some_pdf}` can be a single PDF file or a directory containing multiple PDFs.
319
- The results will be saved in the `{some_output_dir}` directory. The output file list is as follows:
320
-
321
- ```text
322
- ├── some_pdf.md # markdown file
323
- ├── images # directory for storing images
324
- ├── some_pdf_layout.pdf # layout diagram
325
- ├── some_pdf_middle.json # MinerU intermediate processing result
326
- ├── some_pdf_model.json # model inference result
327
- ├── some_pdf_origin.pdf # original PDF file
328
- └── some_pdf_spans.pdf # smallest granularity bbox position information diagram
329
- ```
330
-
331
- For more information about the output files, please refer to the [Output File Description](docs/output_file_en_us.md).
332
-
333
- ### API
334
-
335
- Processing files from local disk
336
-
337
- ```python
338
- image_writer = DiskReaderWriter(local_image_dir)
339
- image_dir = str(os.path.basename(local_image_dir))
340
- jso_useful_key = {"_pdf_type": "", "model_list": []}
341
- pipe = UNIPipe(pdf_bytes, jso_useful_key, image_writer)
342
- pipe.pipe_classify()
343
- pipe.pipe_analyze()
344
- pipe.pipe_parse()
345
- md_content = pipe.pipe_mk_markdown(image_dir, drop_mode="none")
346
- ```
347
-
348
- Processing files from object storage
349
-
350
- ```python
351
- s3pdf_cli = S3ReaderWriter(pdf_ak, pdf_sk, pdf_endpoint)
352
- image_dir = "s3://img_bucket/"
353
- s3image_cli = S3ReaderWriter(img_ak, img_sk, img_endpoint, parent_path=image_dir)
354
- pdf_bytes = s3pdf_cli.read(s3_pdf_path, mode=s3pdf_cli.MODE_BIN)
355
- jso_useful_key = {"_pdf_type": "", "model_list": []}
356
- pipe = UNIPipe(pdf_bytes, jso_useful_key, s3image_cli)
357
- pipe.pipe_classify()
358
- pipe.pipe_analyze()
359
- pipe.pipe_parse()
360
- md_content = pipe.pipe_mk_markdown(image_dir, drop_mode="none")
361
- ```
362
-
363
- For detailed implementation, refer to:
364
-
365
- - [demo.py Simplest Processing Method](demo/demo.py)
366
- - [magic_pdf_parse_main.py More Detailed Processing Workflow](demo/magic_pdf_parse_main.py)
367
-
368
- ### Development Guide
369
-
370
- TODO
371
-
372
- # TODO
373
-
374
- - [ ] Semantic-based reading order
375
- - [ ] List recognition within the text
376
- - [ ] Code block recognition within the text
377
- - [ ] Table of contents recognition
378
- - [x] Table recognition
379
- - [ ] [Chemical formula recognition](docs/chemical_knowledge_introduction/introduction.pdf)
380
- - [ ] Geometric shape recognition
381
-
382
- # Known Issues
383
-
384
- - Reading order is segmented based on rules, which can cause disordered sequences in some cases
385
- - Vertical text is not supported
386
- - Lists, code blocks, and table of contents are not yet supported in the layout model
387
- - Comic books, art books, elementary school textbooks, and exercise books are not well-parsed yet
388
- - Enabling OCR may produce better results in PDFs with a high density of formulas
389
- - If you are processing PDFs with a large number of formulas, it is strongly recommended to enable the OCR function. When using PyMuPDF to extract text, overlapping text lines can occur, leading to inaccurate formula insertion positions.
390
-
391
-
392
- # FAQ
393
-
394
- [FAQ in Chinese](docs/FAQ_zh_cn.md)
395
-
396
- [FAQ in English](docs/FAQ_en_us.md)
397
-
398
- # All Thanks To Our Contributors
399
-
400
- <a href="https://github.com/opendatalab/MinerU/graphs/contributors">
401
- <img src="https://contrib.rocks/image?repo=opendatalab/MinerU" />
402
- </a>
403
-
404
- # License Information
405
-
406
- [LICENSE.md](LICENSE.md)
407
-
408
- This project currently uses PyMuPDF to achieve advanced functionality. However, since it adheres to the AGPL license, it may impose restrictions on certain usage scenarios. In future iterations, we plan to explore and replace it with a more permissive PDF processing library to enhance user-friendliness and flexibility.
409
-
410
- # Acknowledgments
411
-
412
- - [PDF-Extract-Kit](https://github.com/opendatalab/PDF-Extract-Kit)
413
- - [StructEqTable](https://github.com/UniModal4Reasoning/StructEqTable-Deploy)
414
- - [PaddleOCR](https://github.com/PaddlePaddle/PaddleOCR)
415
- - [PyMuPDF](https://github.com/pymupdf/PyMuPDF)
416
- - [fast-langdetect](https://github.com/LlmKira/fast-langdetect)
417
- - [pdfminer.six](https://github.com/pdfminer/pdfminer.six)
418
-
419
- # Citation
420
-
421
- ```bibtex
422
- @article{he2024opendatalab,
423
- title={Opendatalab: Empowering general artificial intelligence with open datasets},
424
- author={He, Conghui and Li, Wei and Jin, Zhenjiang and Xu, Chao and Wang, Bin and Lin, Dahua},
425
- journal={arXiv preprint arXiv:2407.13773},
426
- year={2024}
427
- }
428
-
429
- @misc{2024mineru,
430
- title={MinerU: A One-stop, Open-source, High-quality Data Extraction Tool},
431
- author={MinerU Contributors},
432
- howpublished = {\url{https://github.com/opendatalab/MinerU}},
433
- year={2024}
434
- }
435
- ```
436
-
437
- # Star History
438
-
439
- <a>
440
- <picture>
441
- <source media="(prefers-color-scheme: dark)" srcset="https://api.star-history.com/svg?repos=opendatalab/MinerU&type=Date&theme=dark" />
442
- <source media="(prefers-color-scheme: light)" srcset="https://api.star-history.com/svg?repos=opendatalab/MinerU&type=Date" />
443
- <img alt="Star History Chart" src="https://api.star-history.com/svg?repos=opendatalab/MinerU&type=Date" />
444
- </picture>
445
- </a>
446
-
447
- # Magic-doc
448
-
449
- [Magic-Doc](https://github.com/InternLM/magic-doc) Fast speed ppt/pptx/doc/docx/pdf extraction tool
450
-
451
- # Magic-html
452
-
453
- [Magic-HTML](https://github.com/opendatalab/magic-html) Mixed web page extraction tool
454
-
455
- # Links
456
-
457
- - [LabelU (A Lightweight Multi-modal Data Annotation Tool)](https://github.com/opendatalab/labelU)
458
- - [LabelLLM (An Open-source LLM Dialogue Annotation Platform)](https://github.com/opendatalab/LabelLLM)
459
- - [PDF-Extract-Kit (A Comprehensive Toolkit for High-Quality PDF Content Extraction)](https://github.com/opendatalab/PDF-Extract-Kit)