docvortex 0.2.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- docvortex/__init__.py +21 -0
- docvortex/analyzers/__init__.py +3 -0
- docvortex/analyzers/native/__init__.py +37 -0
- docvortex/analyzers/native/_shared/__init__.py +3 -0
- docvortex/analyzers/native/_shared/hyperlink.py +16 -0
- docvortex/analyzers/native/_shared/image.py +8 -0
- docvortex/analyzers/native/_shared/markup/__init__.py +47 -0
- docvortex/analyzers/native/_shared/markup/anchors.py +30 -0
- docvortex/analyzers/native/_shared/markup/formula.py +37 -0
- docvortex/analyzers/native/_shared/markup/projector.py +57 -0
- docvortex/analyzers/native/_shared/markup/styles.py +19 -0
- docvortex/analyzers/native/_shared/mathml.py +17 -0
- docvortex/analyzers/native/_shared/names.py +7 -0
- docvortex/analyzers/native/_shared/xycut.py +414 -0
- docvortex/analyzers/native/contracts.py +44 -0
- docvortex/analyzers/native/csv.py +351 -0
- docvortex/analyzers/native/epub/__init__.py +18 -0
- docvortex/analyzers/native/epub/constants.py +45 -0
- docvortex/analyzers/native/epub/converter.py +80 -0
- docvortex/analyzers/native/epub/errors.py +20 -0
- docvortex/analyzers/native/epub/metadata.py +31 -0
- docvortex/analyzers/native/epub/package.py +649 -0
- docvortex/analyzers/native/epub/xhtml.py +306 -0
- docvortex/analyzers/native/html/__init__.py +6 -0
- docvortex/analyzers/native/html/anchors.py +271 -0
- docvortex/analyzers/native/html/constants.py +25 -0
- docvortex/analyzers/native/html/contracts.py +7 -0
- docvortex/analyzers/native/html/converter.py +117 -0
- docvortex/analyzers/native/html/document.py +389 -0
- docvortex/analyzers/native/html/errors.py +12 -0
- docvortex/analyzers/native/html/resources.py +365 -0
- docvortex/analyzers/native/html/selector.py +421 -0
- docvortex/analyzers/native/models.py +209 -0
- docvortex/analyzers/native/ofd/__init__.py +14 -0
- docvortex/analyzers/native/ofd/constants.py +59 -0
- docvortex/analyzers/native/ofd/converter.py +31 -0
- docvortex/analyzers/native/ofd/errors.py +16 -0
- docvortex/analyzers/native/ofd/geometry.py +215 -0
- docvortex/analyzers/native/ofd/images.py +124 -0
- docvortex/analyzers/native/ofd/metadata.py +53 -0
- docvortex/analyzers/native/ofd/models.py +170 -0
- docvortex/analyzers/native/ofd/package.py +346 -0
- docvortex/analyzers/native/ofd/path.py +201 -0
- docvortex/analyzers/native/ofd/reading_order.py +293 -0
- docvortex/analyzers/native/ofd/resources.py +122 -0
- docvortex/analyzers/native/ofd/scene.py +347 -0
- docvortex/analyzers/native/ofd/table.py +240 -0
- docvortex/analyzers/native/ofd/text.py +521 -0
- docvortex/analyzers/native/office/__init__.py +3 -0
- docvortex/analyzers/native/office/doc/__init__.py +3 -0
- docvortex/analyzers/native/office/doc/bookmarks.py +81 -0
- docvortex/analyzers/native/office/doc/doc_converter.py +573 -0
- docvortex/analyzers/native/office/doc/fib.py +261 -0
- docvortex/analyzers/native/office/doc/fields.py +101 -0
- docvortex/analyzers/native/office/doc/formatting.py +171 -0
- docvortex/analyzers/native/office/doc/images.py +162 -0
- docvortex/analyzers/native/office/doc/lists.py +356 -0
- docvortex/analyzers/native/office/doc/models.py +164 -0
- docvortex/analyzers/native/office/doc/parser.py +851 -0
- docvortex/analyzers/native/office/doc/pieces.py +256 -0
- docvortex/analyzers/native/office/doc/records.py +40 -0
- docvortex/analyzers/native/office/doc/sprm.py +269 -0
- docvortex/analyzers/native/office/doc/styles.py +215 -0
- docvortex/analyzers/native/office/docx/__init__.py +3 -0
- docvortex/analyzers/native/office/docx/context.py +71 -0
- docvortex/analyzers/native/office/docx/docx_converter.py +614 -0
- docvortex/analyzers/native/office/docx/equationxml.py +119 -0
- docvortex/analyzers/native/office/docx/fields.py +816 -0
- docvortex/analyzers/native/office/docx/formatting_types.py +23 -0
- docvortex/analyzers/native/office/docx/main.py +50 -0
- docvortex/analyzers/native/office/docx/numbering.py +491 -0
- docvortex/analyzers/native/office/docx/office_xml.py +57 -0
- docvortex/analyzers/native/office/docx/package_normalizer.py +248 -0
- docvortex/analyzers/native/office/docx/resources.py +526 -0
- docvortex/analyzers/native/office/docx/styles.py +693 -0
- docvortex/analyzers/native/office/docx/tables.py +515 -0
- docvortex/analyzers/native/office/equation/__init__.py +3 -0
- docvortex/analyzers/native/office/equation/image.py +470 -0
- docvortex/analyzers/native/office/equation/latex_dict.py +324 -0
- docvortex/analyzers/native/office/equation/mtef.py +885 -0
- docvortex/analyzers/native/office/equation/mtef_v5.py +941 -0
- docvortex/analyzers/native/office/equation/omml.py +561 -0
- docvortex/analyzers/native/office/equation/ooxml.py +62 -0
- docvortex/analyzers/native/office/errors.py +33 -0
- docvortex/analyzers/native/office/image.py +307 -0
- docvortex/analyzers/native/office/legacy/__init__.py +3 -0
- docvortex/analyzers/native/office/legacy/binary.py +43 -0
- docvortex/analyzers/native/office/legacy/officeart.py +362 -0
- docvortex/analyzers/native/office/legacy/ole.py +110 -0
- docvortex/analyzers/native/office/limits.py +12 -0
- docvortex/analyzers/native/office/odf/__init__.py +3 -0
- docvortex/analyzers/native/office/odf/chart.py +76 -0
- docvortex/analyzers/native/office/odf/constants.py +70 -0
- docvortex/analyzers/native/office/odf/converters.py +403 -0
- docvortex/analyzers/native/office/odf/errors.py +18 -0
- docvortex/analyzers/native/office/odf/metadata.py +91 -0
- docvortex/analyzers/native/office/odf/models.py +176 -0
- docvortex/analyzers/native/office/odf/package.py +270 -0
- docvortex/analyzers/native/office/odf/styles.py +329 -0
- docvortex/analyzers/native/office/odf/table.py +469 -0
- docvortex/analyzers/native/office/odf/text.py +1002 -0
- docvortex/analyzers/native/office/ooxml_chart.py +1016 -0
- docvortex/analyzers/native/office/opc.py +38 -0
- docvortex/analyzers/native/office/ppt/__init__.py +3 -0
- docvortex/analyzers/native/office/ppt/models.py +118 -0
- docvortex/analyzers/native/office/ppt/parser.py +1895 -0
- docvortex/analyzers/native/office/ppt/ppt_converter.py +292 -0
- docvortex/analyzers/native/office/ppt/records.py +131 -0
- docvortex/analyzers/native/office/ppt/style_text.py +247 -0
- docvortex/analyzers/native/office/pptx/__init__.py +3 -0
- docvortex/analyzers/native/office/pptx/context.py +113 -0
- docvortex/analyzers/native/office/pptx/lists.py +558 -0
- docvortex/analyzers/native/office/pptx/main.py +20 -0
- docvortex/analyzers/native/office/pptx/package_normalizer.py +321 -0
- docvortex/analyzers/native/office/pptx/pptx_converter.py +323 -0
- docvortex/analyzers/native/office/pptx/resources.py +329 -0
- docvortex/analyzers/native/office/pptx/shapes.py +393 -0
- docvortex/analyzers/native/office/pptx/text_styles.py +625 -0
- docvortex/analyzers/native/office/pptx/titles.py +178 -0
- docvortex/analyzers/native/office/rich_text.py +420 -0
- docvortex/analyzers/native/office/rtf/__init__.py +3 -0
- docvortex/analyzers/native/office/rtf/converter.py +708 -0
- docvortex/analyzers/native/office/rtf/lexer.py +213 -0
- docvortex/analyzers/native/office/rtf/math.py +339 -0
- docvortex/analyzers/native/office/rtf/models.py +185 -0
- docvortex/analyzers/native/office/rtf/parser.py +1553 -0
- docvortex/analyzers/native/office/spreadsheet/__init__.py +3 -0
- docvortex/analyzers/native/office/spreadsheet/html.py +81 -0
- docvortex/analyzers/native/office/spreadsheet/models.py +79 -0
- docvortex/analyzers/native/office/spreadsheet/projector.py +928 -0
- docvortex/analyzers/native/office/streams.py +18 -0
- docvortex/analyzers/native/office/xls/__init__.py +3 -0
- docvortex/analyzers/native/office/xls/chart.py +132 -0
- docvortex/analyzers/native/office/xls/embedded_chart.py +299 -0
- docvortex/analyzers/native/office/xls/models.py +109 -0
- docvortex/analyzers/native/office/xls/number_format.py +521 -0
- docvortex/analyzers/native/office/xls/parser.py +1145 -0
- docvortex/analyzers/native/office/xls/records.py +201 -0
- docvortex/analyzers/native/office/xls/strings.py +205 -0
- docvortex/analyzers/native/office/xls/xls_converter.py +354 -0
- docvortex/analyzers/native/office/xlsx/__init__.py +3 -0
- docvortex/analyzers/native/office/xlsx/main.py +20 -0
- docvortex/analyzers/native/office/xlsx/ooxml_ole.py +522 -0
- docvortex/analyzers/native/office/xlsx/package_normalizer.py +310 -0
- docvortex/analyzers/native/office/xlsx/xlsx_converter.py +716 -0
- docvortex/analyzers/native/pdf/__init__.py +3 -0
- docvortex/analyzers/native/pdf/auxiliary_text.py +1671 -0
- docvortex/analyzers/native/pdf/char_geometry.py +1662 -0
- docvortex/analyzers/native/pdf/code_blocks.py +535 -0
- docvortex/analyzers/native/pdf/formulas.py +1985 -0
- docvortex/analyzers/native/pdf/geometry.py +281 -0
- docvortex/analyzers/native/pdf/graphics.py +1501 -0
- docvortex/analyzers/native/pdf/index_blocks.py +268 -0
- docvortex/analyzers/native/pdf/inline/__init__.py +3 -0
- docvortex/analyzers/native/pdf/inline/common.py +112 -0
- docvortex/analyzers/native/pdf/inline/detection.py +590 -0
- docvortex/analyzers/native/pdf/inline/matching.py +1185 -0
- docvortex/analyzers/native/pdf/inline/materialize.py +457 -0
- docvortex/analyzers/native/pdf/inline/scripts.py +975 -0
- docvortex/analyzers/native/pdf/inline/types.py +385 -0
- docvortex/analyzers/native/pdf/line_layout.py +1106 -0
- docvortex/analyzers/native/pdf/line_merging.py +1223 -0
- docvortex/analyzers/native/pdf/models.py +246 -0
- docvortex/analyzers/native/pdf/native_text.py +1004 -0
- docvortex/analyzers/native/pdf/pipeline.py +1390 -0
- docvortex/analyzers/native/pdf/script_geometry.py +636 -0
- docvortex/analyzers/native/pdf/shared.py +30 -0
- docvortex/analyzers/native/pdf/spatial_text.py +383 -0
- docvortex/analyzers/native/pdf/table_annotations.py +446 -0
- docvortex/analyzers/native/pdf/table_constants.py +48 -0
- docvortex/analyzers/native/pdf/table_detection.py +227 -0
- docvortex/analyzers/native/pdf/table_filled_grid.py +218 -0
- docvortex/analyzers/native/pdf/table_geometry.py +40 -0
- docvortex/analyzers/native/pdf/table_materialization.py +444 -0
- docvortex/analyzers/native/pdf/table_recovery/__init__.py +15 -0
- docvortex/analyzers/native/pdf/table_recovery/candidate.py +356 -0
- docvortex/analyzers/native/pdf/table_recovery/contracts.py +154 -0
- docvortex/analyzers/native/pdf/table_recovery/engine.py +609 -0
- docvortex/analyzers/native/pdf/table_recovery/geometry.py +164 -0
- docvortex/analyzers/native/pdf/table_recovery/sparse_common.py +85 -0
- docvortex/analyzers/native/pdf/table_recovery/sparse_hybrid.py +804 -0
- docvortex/analyzers/native/pdf/table_recovery/sparse_multiline.py +1122 -0
- docvortex/analyzers/native/pdf/table_recovery/text.py +414 -0
- docvortex/analyzers/native/pdf/table_recovery/text_grid.py +618 -0
- docvortex/analyzers/native/pdf/table_recovery/vector.py +1931 -0
- docvortex/analyzers/native/pdf/table_rows.py +34 -0
- docvortex/analyzers/native/pdf/table_rules.py +1129 -0
- docvortex/analyzers/native/pdf/table_text_styles.py +283 -0
- docvortex/analyzers/native/pdf/tables.py +147 -0
- docvortex/analyzers/native/pdf/text_assembly/__init__.py +3 -0
- docvortex/analyzers/native/pdf/text_assembly/annotations.py +581 -0
- docvortex/analyzers/native/pdf/text_assembly/assembly.py +292 -0
- docvortex/analyzers/native/pdf/text_assembly/common.py +477 -0
- docvortex/analyzers/native/pdf/text_assembly/footnotes.py +394 -0
- docvortex/analyzers/native/pdf/text_assembly/merging.py +1274 -0
- docvortex/analyzers/native/pdf/text_assembly/rows.py +692 -0
- docvortex/analyzers/native/pdf/text_blocks.py +82 -0
- docvortex/analyzers/native/pdf/text_styles.py +55 -0
- docvortex/analyzers/native/pdf/title_analysis/__init__.py +3 -0
- docvortex/analyzers/native/pdf/title_analysis/body_profile.py +215 -0
- docvortex/analyzers/native/pdf/title_analysis/common.py +117 -0
- docvortex/analyzers/native/pdf/title_analysis/document_profile.py +164 -0
- docvortex/analyzers/native/pdf/title_analysis/lane_titles.py +758 -0
- docvortex/analyzers/native/pdf/title_analysis/page_titles.py +1024 -0
- docvortex/analyzers/native/pdf/title_analysis/prototype.py +194 -0
- docvortex/analyzers/native/pdf/title_analysis/structural.py +1081 -0
- docvortex/analyzers/native/pdf/titles.py +75 -0
- docvortex/analyzers/native/pdf/typography.py +19 -0
- docvortex/analyzers/native/pdf/visual_annotations.py +1262 -0
- docvortex/api.py +180 -0
- docvortex/assets/__init__.py +5 -0
- docvortex/assets/store.py +51 -0
- docvortex/cli.py +54 -0
- docvortex/codecs/__init__.py +3 -0
- docvortex/codecs/html/__init__.py +22 -0
- docvortex/codecs/html/contracts.py +236 -0
- docvortex/codecs/html/materializer.py +331 -0
- docvortex/codecs/html/parser.py +763 -0
- docvortex/codecs/html/resources.py +34 -0
- docvortex/codecs/json.py +17 -0
- docvortex/content/__init__.py +5 -0
- docvortex/content/inline.py +248 -0
- docvortex/content/markup/__init__.py +44 -0
- docvortex/content/markup/anchors.py +188 -0
- docvortex/content/markup/formula.py +280 -0
- docvortex/content/markup/projector.py +1237 -0
- docvortex/content/markup/styles.py +327 -0
- docvortex/content/mathml.py +167 -0
- docvortex/content/normalization.py +188 -0
- docvortex/content/spans.py +183 -0
- docvortex/content/table/__init__.py +18 -0
- docvortex/content/table/blocks.py +152 -0
- docvortex/content/table/content.py +425 -0
- docvortex/content/table/document.py +104 -0
- docvortex/content/table/html.py +399 -0
- docvortex/content/table/models.py +76 -0
- docvortex/content/table/rules.py +42 -0
- docvortex/content/table/structure.py +221 -0
- docvortex/content/tree.py +5 -0
- docvortex/document/__init__.py +3 -0
- docvortex/document/contracts.py +22 -0
- docvortex/document/detection.py +389 -0
- docvortex/document/filetypes.py +175 -0
- docvortex/document/page_range.py +167 -0
- docvortex/document/pdf/__init__.py +19 -0
- docvortex/document/pdf/classify.py +1138 -0
- docvortex/document/pdf/constants.py +132 -0
- docvortex/document/pdf/diagnostics.py +350 -0
- docvortex/document/pdf/document.py +582 -0
- docvortex/document/pdf/font_runtime.py +335 -0
- docvortex/document/pdf/geometry.py +31 -0
- docvortex/document/pdf/images.py +654 -0
- docvortex/document/pdf/native_annotations.py +367 -0
- docvortex/document/pdf/native_contracts.py +169 -0
- docvortex/document/pdf/native_coordinates.py +216 -0
- docvortex/document/pdf/native_lifecycle.py +16 -0
- docvortex/document/pdf/native_objects.py +902 -0
- docvortex/document/pdf/native_text_geometry.py +315 -0
- docvortex/document/pdf/pdfium.py +325 -0
- docvortex/document/pdf/raster.py +46 -0
- docvortex/document/pdf/text/__init__.py +62 -0
- docvortex/document/pdf/text/contracts.py +211 -0
- docvortex/document/pdf/text/extract.py +165 -0
- docvortex/document/pdf/text/geometry.py +16 -0
- docvortex/document/pdf/text/groups.py +162 -0
- docvortex/document/pdf/visual_geometry.py +201 -0
- docvortex/document/pdf/visuals.py +343 -0
- docvortex/document/source.py +92 -0
- docvortex/errors.py +21 -0
- docvortex/export/__init__.py +3 -0
- docvortex/export/bundle.py +98 -0
- docvortex/export/files.py +66 -0
- docvortex/export/middle.py +208 -0
- docvortex/foundation/__init__.py +3 -0
- docvortex/foundation/geometry.py +125 -0
- docvortex/foundation/hyperlink.py +65 -0
- docvortex/foundation/image.py +48 -0
- docvortex/foundation/image_encoding.py +30 -0
- docvortex/foundation/image_payload.py +280 -0
- docvortex/foundation/language.py +92 -0
- docvortex/foundation/platform.py +38 -0
- docvortex/foundation/text.py +153 -0
- docvortex/foundation/type_identity.py +20 -0
- docvortex/foundation/xml_names.py +20 -0
- docvortex/options.py +30 -0
- docvortex/postprocess/__init__.py +3 -0
- docvortex/postprocess/content.py +53 -0
- docvortex/postprocess/document.py +19 -0
- docvortex/postprocess/lists.py +236 -0
- docvortex/postprocess/page_blocks.py +214 -0
- docvortex/postprocess/pages.py +95 -0
- docvortex/postprocess/paragraphs.py +580 -0
- docvortex/postprocess/visual.py +715 -0
- docvortex/render/__init__.py +48 -0
- docvortex/render/_internal/__init__.py +3 -0
- docvortex/render/_internal/common/__init__.py +3 -0
- docvortex/render/_internal/common/context.py +43 -0
- docvortex/render/_internal/common/html_table.py +178 -0
- docvortex/render/_internal/common/index.py +33 -0
- docvortex/render/_internal/common/list_items.py +158 -0
- docvortex/render/_internal/common/planner.py +140 -0
- docvortex/render/_internal/docx/__init__.py +3 -0
- docvortex/render/_internal/docx/assets.py +202 -0
- docvortex/render/_internal/docx/inline.py +434 -0
- docvortex/render/_internal/docx/math.py +220 -0
- docvortex/render/_internal/docx/renderer.py +905 -0
- docvortex/render/_internal/docx/styles.py +195 -0
- docvortex/render/_internal/docx/table.py +442 -0
- docvortex/render/_internal/epub/__init__.py +5 -0
- docvortex/render/_internal/epub/assets.py +173 -0
- docvortex/render/_internal/epub/package.py +249 -0
- docvortex/render/_internal/epub/renderer.py +1156 -0
- docvortex/render/_internal/html/__init__.py +3 -0
- docvortex/render/_internal/html/inline.py +346 -0
- docvortex/render/_internal/html/renderer.py +1041 -0
- docvortex/render/_internal/html/sanitizer.py +478 -0
- docvortex/render/_internal/html/table.py +121 -0
- docvortex/render/_internal/latex/__init__.py +1 -0
- docvortex/render/_internal/latex/assets.py +85 -0
- docvortex/render/_internal/latex/inline.py +145 -0
- docvortex/render/_internal/latex/renderer.py +506 -0
- docvortex/render/_internal/latex/table.py +347 -0
- docvortex/render/_internal/markdown/__init__.py +3 -0
- docvortex/render/_internal/markdown/assets.py +78 -0
- docvortex/render/_internal/markdown/blocks.py +635 -0
- docvortex/render/_internal/markdown/escaping.py +51 -0
- docvortex/render/_internal/markdown/inline.py +260 -0
- docvortex/render/_internal/markdown/renderer.py +93 -0
- docvortex/render/_internal/markdown/table.py +281 -0
- docvortex/render/_internal/pdf/__init__.py +3 -0
- docvortex/render/_internal/pdf/assets.py +197 -0
- docvortex/render/_internal/pdf/formula.py +417 -0
- docvortex/render/_internal/pdf/inline.py +343 -0
- docvortex/render/_internal/pdf/renderer.py +734 -0
- docvortex/render/_internal/pdf/styles.py +206 -0
- docvortex/render/_internal/pdf/table.py +272 -0
- docvortex/render/_internal/structured_content/__init__.py +3 -0
- docvortex/render/_internal/structured_content/renderer.py +193 -0
- docvortex/render/api.py +199 -0
- docvortex/render/contracts.py +205 -0
- docvortex/render/docx.py +38 -0
- docvortex/render/epub.py +35 -0
- docvortex/render/fragments.py +70 -0
- docvortex/render/html.py +29 -0
- docvortex/render/latex.py +24 -0
- docvortex/render/markdown.py +50 -0
- docvortex/render/pdf.py +25 -0
- docvortex/render/structured_content.py +23 -0
- docvortex/resources/epub/docvortex.css +91 -0
- docvortex/resources/fasttext-langdetect/lid.176.ftz +0 -0
- docvortex/resources/fonts/DroidSansFallbackFull.ttf +0 -0
- docvortex/resources/fonts/NOTICE +190 -0
- docvortex/resources/fonts/manifest.json +9 -0
- docvortex/resources/html/docvortex.css +601 -0
- docvortex/resources/html/docvortex.min.css +1 -0
- docvortex/result.py +91 -0
- docvortex/schema.py +1141 -0
- docvortex/version.py +3 -0
- docvortex-0.2.1.dist-info/METADATA +193 -0
- docvortex-0.2.1.dist-info/RECORD +364 -0
- docvortex-0.2.1.dist-info/WHEEL +5 -0
- docvortex-0.2.1.dist-info/entry_points.txt +2 -0
- docvortex-0.2.1.dist-info/licenses/LICENSE.md +21 -0
- docvortex-0.2.1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,1390 @@
|
|
|
1
|
+
"""编排 Flash 原生 PDF 的页面准备、语义处理和输出归一化。"""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections import deque
|
|
6
|
+
from dataclasses import dataclass, replace
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
from ..contracts import NativePdfSource, RawBlock
|
|
10
|
+
|
|
11
|
+
from ....schema import BBox
|
|
12
|
+
from .._shared.xycut import sort_entries
|
|
13
|
+
from ....document.pdf.document import PDFDocument as PDFDocument, PDFImageInfo, PDFPageTextGeometry, get_lines_from_chars
|
|
14
|
+
from .inline.detection import detect_pdf_text_link_lines, detect_pdf_text_style_lines
|
|
15
|
+
from .inline.matching import _realign_repaired_text_evidence
|
|
16
|
+
from .inline.materialize import (
|
|
17
|
+
apply_pdf_text_links,
|
|
18
|
+
apply_pdf_text_scripts,
|
|
19
|
+
apply_pdf_text_styles,
|
|
20
|
+
materialize_pdf_inline_spans,
|
|
21
|
+
)
|
|
22
|
+
from .inline.scripts import detect_pdf_text_script_lines
|
|
23
|
+
from .inline.types import PDFTextLinkLine, PDFTextStyleLine
|
|
24
|
+
|
|
25
|
+
from .models import (
|
|
26
|
+
_AxisLine,
|
|
27
|
+
_DocumentBodyProfile,
|
|
28
|
+
_DocumentTitleProfile,
|
|
29
|
+
_LineItem,
|
|
30
|
+
_MarginalCandidate,
|
|
31
|
+
_PageSource,
|
|
32
|
+
_PreparedPage,
|
|
33
|
+
)
|
|
34
|
+
from .geometry import (
|
|
35
|
+
_bbox_area,
|
|
36
|
+
_bbox_axis_overlap_ratio,
|
|
37
|
+
_bbox_center_y,
|
|
38
|
+
_bbox_overlap_in_smaller,
|
|
39
|
+
_bbox_union_many,
|
|
40
|
+
_clip_bbox,
|
|
41
|
+
_coerce_bbox,
|
|
42
|
+
_normalize_bbox_to_unit,
|
|
43
|
+
_rotate_bbox_from_upright,
|
|
44
|
+
_rotate_bbox_to_upright,
|
|
45
|
+
)
|
|
46
|
+
from .native_text import (
|
|
47
|
+
_build_native_line_items,
|
|
48
|
+
_extract_decorative_text_rules,
|
|
49
|
+
_coerce_pdf_drawing_lines,
|
|
50
|
+
_median_native_glyph_width,
|
|
51
|
+
_sanitize_pdf_control_text,
|
|
52
|
+
_resplit_native_visual_runs,
|
|
53
|
+
)
|
|
54
|
+
from .char_geometry import DocumentGeometryPlan, apply_line_geometry_repairs, build_document_geometry_plan
|
|
55
|
+
from .line_merging import (
|
|
56
|
+
_merge_overlapping_inline_text_clusters,
|
|
57
|
+
_merge_post_semantic_text_runs,
|
|
58
|
+
_merge_same_baseline_text_lines,
|
|
59
|
+
_merge_title_resolved_visual_rows,
|
|
60
|
+
_restore_dense_split_visual_rows,
|
|
61
|
+
)
|
|
62
|
+
from .index_blocks import _extract_index_blocks
|
|
63
|
+
from .tables import _connected_horizontal_rule_bboxes, _detect_table_candidates, _materialize_table_blocks
|
|
64
|
+
from .graphics import (
|
|
65
|
+
_IMAGE_CONTAINER_OVERLAP_THRESHOLD,
|
|
66
|
+
_build_form_image_blocks,
|
|
67
|
+
_build_graphic_like_blocks,
|
|
68
|
+
_build_raster_image_blocks,
|
|
69
|
+
_detect_strong_graphic_bboxes,
|
|
70
|
+
_form_supersedes_nested_bbox,
|
|
71
|
+
_select_form_image_bboxes,
|
|
72
|
+
_split_parallel_graphic_rule_rows,
|
|
73
|
+
)
|
|
74
|
+
from .formulas import _build_formula_like_blocks, _build_vector_formula_blocks
|
|
75
|
+
from .code_blocks import _build_code_blocks, _build_rule_delimited_code_blocks
|
|
76
|
+
from .auxiliary_text import (
|
|
77
|
+
_build_marginal_candidate,
|
|
78
|
+
_classify_deferred_image_footnotes,
|
|
79
|
+
_classify_isolated_first_page_footer,
|
|
80
|
+
_classify_page_footnote_trailing_footers,
|
|
81
|
+
_classify_page_number_outer_companions,
|
|
82
|
+
_classify_page_auxiliary_text,
|
|
83
|
+
_classify_raw_page_marginals,
|
|
84
|
+
_classify_rule_delimited_footers,
|
|
85
|
+
_classify_rule_delimited_headers,
|
|
86
|
+
_classify_split_marginal_row_companions,
|
|
87
|
+
_classify_repeated_page_marginals,
|
|
88
|
+
_classify_repeated_visual_headers,
|
|
89
|
+
_classify_single_page_compound_headers,
|
|
90
|
+
_marginal_geometry_matches,
|
|
91
|
+
_marginal_text_matches,
|
|
92
|
+
)
|
|
93
|
+
from .title_analysis.body_profile import _infer_document_body_profile
|
|
94
|
+
from .title_analysis.document_profile import _infer_document_title_profile
|
|
95
|
+
from .title_analysis.page_titles import _classify_page_titles
|
|
96
|
+
from .title_analysis.structural import (
|
|
97
|
+
_classify_body_height_section_titles,
|
|
98
|
+
_classify_explicit_section_titles,
|
|
99
|
+
_classify_inline_typography_reset_titles,
|
|
100
|
+
_classify_document_structural_titles,
|
|
101
|
+
_promote_noninitial_document_title_band,
|
|
102
|
+
)
|
|
103
|
+
from .text_assembly.annotations import (
|
|
104
|
+
_merge_fragmented_header_blocks,
|
|
105
|
+
_merge_front_matter_column_blocks,
|
|
106
|
+
_merge_image_caption_text_blocks,
|
|
107
|
+
_merge_multiline_title_blocks,
|
|
108
|
+
_merge_repeated_compact_title_continuations,
|
|
109
|
+
)
|
|
110
|
+
from .text_assembly.assembly import _build_text_blocks
|
|
111
|
+
from .text_assembly.common import _merge_internal_text_block_group
|
|
112
|
+
from .visual_annotations import _classify_and_bind_visual_annotations
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
_TEXT_SEMANTIC_TYPES = {
|
|
116
|
+
"doc_title",
|
|
117
|
+
"paragraph_title",
|
|
118
|
+
"ref_text",
|
|
119
|
+
"header",
|
|
120
|
+
"footer",
|
|
121
|
+
"page_number",
|
|
122
|
+
"caption",
|
|
123
|
+
"footnote",
|
|
124
|
+
"page_footnote",
|
|
125
|
+
"aside_text",
|
|
126
|
+
"index",
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
_OUTPUT_BLOCK_TYPES = {"text", "table", "image", "equation", "code"} | _TEXT_SEMANTIC_TYPES
|
|
131
|
+
_LINE_METADATA_OUTPUT_TYPES = {
|
|
132
|
+
"text",
|
|
133
|
+
"ref_text",
|
|
134
|
+
"doc_title",
|
|
135
|
+
"paragraph_title",
|
|
136
|
+
"caption",
|
|
137
|
+
"footnote",
|
|
138
|
+
}
|
|
139
|
+
_REPEATED_RASTER_IMAGE_MIN_PAGE_AREA_RATIO = 0.08
|
|
140
|
+
_REPEATED_RASTER_IMAGE_MIN_DISTINCT_PAGES = 3
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def _is_large_raster_image(
|
|
144
|
+
image_info: PDFImageInfo,
|
|
145
|
+
page_size: tuple[float, float],
|
|
146
|
+
) -> bool:
|
|
147
|
+
"""判断点阵图裁剪后面积是否达到页面面积的 8%。"""
|
|
148
|
+
|
|
149
|
+
bbox = _coerce_bbox(image_info.bbox)
|
|
150
|
+
page_area = max(0.0, page_size[0]) * max(0.0, page_size[1])
|
|
151
|
+
return bbox is not None and page_area > 0 and _bbox_area(bbox) / page_area >= _REPEATED_RASTER_IMAGE_MIN_PAGE_AREA_RATIO
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def _detect_repeated_raster_watermark_fingerprints(
|
|
155
|
+
page_image_infos: list[list[PDFImageInfo]],
|
|
156
|
+
page_sizes: list[tuple[float, float]],
|
|
157
|
+
) -> set[str]:
|
|
158
|
+
"""按大图指纹统计不同页号,出现至少三页时判为跨页图片水印。"""
|
|
159
|
+
|
|
160
|
+
page_indices_by_fingerprint: dict[str, set[int]] = {}
|
|
161
|
+
for page_idx, (image_infos, page_size) in enumerate(zip(page_image_infos, page_sizes, strict=True)):
|
|
162
|
+
for image_info in image_infos:
|
|
163
|
+
if image_info.fingerprint is None or not _is_large_raster_image(image_info, page_size):
|
|
164
|
+
continue
|
|
165
|
+
page_indices_by_fingerprint.setdefault(image_info.fingerprint, set()).add(page_idx)
|
|
166
|
+
return {
|
|
167
|
+
fingerprint
|
|
168
|
+
for fingerprint, page_indices in page_indices_by_fingerprint.items()
|
|
169
|
+
if len(page_indices) >= _REPEATED_RASTER_IMAGE_MIN_DISTINCT_PAGES
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def _filter_repeated_raster_watermark_bboxes(
|
|
174
|
+
image_infos: list[PDFImageInfo],
|
|
175
|
+
page_size: tuple[float, float],
|
|
176
|
+
watermark_fingerprints: set[str],
|
|
177
|
+
) -> list[tuple[float, float, float, float]]:
|
|
178
|
+
"""仅删除命中跨页水印指纹且面积达标的 bbox,小尺寸同图继续保留。"""
|
|
179
|
+
|
|
180
|
+
return [
|
|
181
|
+
image_info.bbox
|
|
182
|
+
for image_info in image_infos
|
|
183
|
+
if not (image_info.fingerprint in watermark_fingerprints and _is_large_raster_image(image_info, page_size))
|
|
184
|
+
]
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def _table_detection_drawing_lines(
|
|
188
|
+
source: _PageSource,
|
|
189
|
+
confirmed_header_separators: set[BBox] | None = None,
|
|
190
|
+
) -> list[_AxisLine]:
|
|
191
|
+
"""从表格候选路径中移除已确认页眉下方的通栏分隔线。"""
|
|
192
|
+
|
|
193
|
+
confirmed = confirmed_header_separators or set()
|
|
194
|
+
return [drawing_line for drawing_line in source.drawing_lines if drawing_line.bbox not in confirmed]
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def _detect_repeated_header_separator_bboxes(
|
|
198
|
+
sources: list[_PageSource],
|
|
199
|
+
) -> list[set[BBox]]:
|
|
200
|
+
"""用重复刊头文本和非表格横线共同确认跨页页眉分隔线。"""
|
|
201
|
+
|
|
202
|
+
candidates_by_signature: dict[
|
|
203
|
+
tuple[float, float, float],
|
|
204
|
+
list[tuple[int, BBox]],
|
|
205
|
+
] = {}
|
|
206
|
+
for page_index, source in enumerate(sources):
|
|
207
|
+
page_width, page_height = source.page_size
|
|
208
|
+
if page_width <= 0 or page_height <= 0:
|
|
209
|
+
continue
|
|
210
|
+
connected_table_rules = _connected_horizontal_rule_bboxes(source)
|
|
211
|
+
for drawing_line in source.drawing_lines:
|
|
212
|
+
bbox = drawing_line.bbox
|
|
213
|
+
center_y = _bbox_center_y(bbox)
|
|
214
|
+
if (
|
|
215
|
+
drawing_line.orientation != "horizontal"
|
|
216
|
+
or center_y > 0.15 * page_height
|
|
217
|
+
or bbox[2] - bbox[0] < 0.6 * page_width
|
|
218
|
+
or sum(line.angle == 0 and line.bbox[3] <= center_y for line in source.lines) < 2
|
|
219
|
+
or bbox in connected_table_rules
|
|
220
|
+
):
|
|
221
|
+
continue
|
|
222
|
+
signature = (
|
|
223
|
+
round(bbox[0] / page_width, 2),
|
|
224
|
+
round(bbox[2] / page_width, 2),
|
|
225
|
+
round(center_y / page_height, 3),
|
|
226
|
+
)
|
|
227
|
+
candidates_by_signature.setdefault(
|
|
228
|
+
signature,
|
|
229
|
+
[],
|
|
230
|
+
).append((page_index, bbox))
|
|
231
|
+
|
|
232
|
+
output = [set() for _source in sources]
|
|
233
|
+
for members in candidates_by_signature.values():
|
|
234
|
+
if len({page_index for page_index, _bbox in members}) < 3:
|
|
235
|
+
continue
|
|
236
|
+
if len(_repeated_header_evidence_pages(sources, members)) < 2:
|
|
237
|
+
continue
|
|
238
|
+
for page_index, bbox in members:
|
|
239
|
+
output[page_index].add(bbox)
|
|
240
|
+
return output
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def _repeated_header_evidence_pages(
|
|
244
|
+
sources: list[_PageSource],
|
|
245
|
+
separator_members: list[tuple[int, BBox]],
|
|
246
|
+
) -> set[int]:
|
|
247
|
+
"""返回具有已分类或跨页重复刊头文本证据的页号集合。"""
|
|
248
|
+
|
|
249
|
+
supported_pages: set[int] = set()
|
|
250
|
+
candidates: list[tuple[int, _MarginalCandidate]] = []
|
|
251
|
+
for page_index, separator_bbox in separator_members:
|
|
252
|
+
center_y = _bbox_center_y(separator_bbox)
|
|
253
|
+
for line in sources[page_index].lines:
|
|
254
|
+
if line.angle != 0 or line.bbox[3] > center_y:
|
|
255
|
+
continue
|
|
256
|
+
if line.semantic_type == "header":
|
|
257
|
+
supported_pages.add(page_index)
|
|
258
|
+
continue
|
|
259
|
+
candidate = _build_marginal_candidate(
|
|
260
|
+
page_index,
|
|
261
|
+
line,
|
|
262
|
+
sources[page_index].page_size,
|
|
263
|
+
)
|
|
264
|
+
if candidate is not None and candidate.region == "header":
|
|
265
|
+
candidates.append((page_index, candidate))
|
|
266
|
+
|
|
267
|
+
for left_index, (left_page, left) in enumerate(candidates):
|
|
268
|
+
for right_page, right in candidates[left_index + 1 :]:
|
|
269
|
+
page_delta = right_page - left_page
|
|
270
|
+
if page_delta > 2:
|
|
271
|
+
break
|
|
272
|
+
if (
|
|
273
|
+
page_delta > 0
|
|
274
|
+
and _marginal_geometry_matches(left, right)
|
|
275
|
+
and _marginal_text_matches(left.line.text, right.line.text)
|
|
276
|
+
):
|
|
277
|
+
supported_pages.update((left_page, right_page))
|
|
278
|
+
return supported_pages
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
@dataclass(slots=True)
|
|
282
|
+
class _DocumentSources:
|
|
283
|
+
"""持有跨页校准前的原始页面,以及最终物化仍需的紧凑样式证据。"""
|
|
284
|
+
|
|
285
|
+
page_sources: list[_PageSource]
|
|
286
|
+
page_text_geometries: list[PDFPageTextGeometry]
|
|
287
|
+
page_sizes: list[tuple[float, float]]
|
|
288
|
+
page_style_lines: list[list[PDFTextStyleLine]]
|
|
289
|
+
page_link_lines: list[list[PDFTextLinkLine]]
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
def _collect_document_sources(pdf_doc: NativePdfSource) -> _DocumentSources:
|
|
293
|
+
"""逐页收集原生证据,局部快照与字符引用在收集阶段退出时释放。"""
|
|
294
|
+
|
|
295
|
+
page_sizes: list[tuple[float, float]] = []
|
|
296
|
+
page_image_infos: list[list[PDFImageInfo]] = []
|
|
297
|
+
page_sources: list[_PageSource] = []
|
|
298
|
+
page_text_geometries = []
|
|
299
|
+
page_style_lines: list[list[PDFTextStyleLine]] = []
|
|
300
|
+
page_link_lines: list[list[PDFTextLinkLine]] = []
|
|
301
|
+
for page_idx in range(pdf_doc.page_count):
|
|
302
|
+
snapshot = pdf_doc._extract_native_page(page_idx)
|
|
303
|
+
page_size = snapshot.page_size
|
|
304
|
+
page_sizes.append(page_size)
|
|
305
|
+
page_image_infos.append(snapshot.image_infos)
|
|
306
|
+
text_geometry = snapshot.text_geometry
|
|
307
|
+
chars = text_geometry.chars
|
|
308
|
+
lines = _build_native_line_items(
|
|
309
|
+
get_lines_from_chars(chars),
|
|
310
|
+
page_size,
|
|
311
|
+
page_rotation=snapshot.rotation,
|
|
312
|
+
)
|
|
313
|
+
drawing_lines = _coerce_pdf_drawing_lines(snapshot.drawing_lines)
|
|
314
|
+
lines, decorative_rules = _extract_decorative_text_rules(
|
|
315
|
+
lines,
|
|
316
|
+
page_size,
|
|
317
|
+
)
|
|
318
|
+
drawing_lines.extend(decorative_rules)
|
|
319
|
+
page_style_lines.append(detect_pdf_text_style_lines(lines, drawing_lines))
|
|
320
|
+
page_link_lines.append(
|
|
321
|
+
detect_pdf_text_link_lines(
|
|
322
|
+
lines,
|
|
323
|
+
snapshot.link_annotations,
|
|
324
|
+
)
|
|
325
|
+
)
|
|
326
|
+
source = _PageSource(
|
|
327
|
+
page_size=page_size,
|
|
328
|
+
lines=lines,
|
|
329
|
+
chars=chars,
|
|
330
|
+
drawing_lines=drawing_lines,
|
|
331
|
+
signature_bboxes=snapshot.signature_bboxes,
|
|
332
|
+
form_bboxes=snapshot.form_bboxes,
|
|
333
|
+
path_infos=snapshot.path_infos,
|
|
334
|
+
)
|
|
335
|
+
page_sources.append(source)
|
|
336
|
+
page_text_geometries.append(text_geometry)
|
|
337
|
+
|
|
338
|
+
watermark_fingerprints = _detect_repeated_raster_watermark_fingerprints(page_image_infos, page_sizes)
|
|
339
|
+
for source, image_infos in zip(page_sources, page_image_infos, strict=True):
|
|
340
|
+
source.image_bboxes = _filter_repeated_raster_watermark_bboxes(
|
|
341
|
+
image_infos,
|
|
342
|
+
source.page_size,
|
|
343
|
+
watermark_fingerprints,
|
|
344
|
+
)
|
|
345
|
+
|
|
346
|
+
return _DocumentSources(page_sources, page_text_geometries, page_sizes, page_style_lines, page_link_lines)
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
def _prepare_document_sources(
|
|
350
|
+
sources: _DocumentSources,
|
|
351
|
+
*,
|
|
352
|
+
geometry_diagnostics: list[dict[str, Any]] | None = None,
|
|
353
|
+
) -> list[_PreparedPage]:
|
|
354
|
+
"""先完成全文几何和页眉判定,再按顺序消费原始页面并释放已用证据。"""
|
|
355
|
+
|
|
356
|
+
geometry_plan = build_document_geometry_plan(
|
|
357
|
+
[source.lines for source in sources.page_sources],
|
|
358
|
+
sources.page_text_geometries,
|
|
359
|
+
sources.page_sizes,
|
|
360
|
+
)
|
|
361
|
+
for page_index, source in enumerate(sources.page_sources):
|
|
362
|
+
# 容器认领前只允许 X 修复,表格和图形认领后再启用 Y trim。
|
|
363
|
+
apply_line_geometry_repairs(source.lines, page_index=page_index, plan=geometry_plan, allow_y_trim=False)
|
|
364
|
+
if geometry_diagnostics is not None:
|
|
365
|
+
geometry_diagnostics.append(geometry_plan.to_dict())
|
|
366
|
+
_classify_raw_page_marginals(sources.page_sources)
|
|
367
|
+
separators = _detect_repeated_header_separator_bboxes(sources.page_sources)
|
|
368
|
+
repaired_chars_by_page: dict[int, dict[int, BBox]] = {}
|
|
369
|
+
for (page_index, char_idx), repair in geometry_plan.char_repairs.items():
|
|
370
|
+
repaired_chars_by_page.setdefault(page_index, {})[char_idx] = repair.layout_bbox
|
|
371
|
+
|
|
372
|
+
pending = deque(zip(sources.page_sources, sources.page_text_geometries, strict=True))
|
|
373
|
+
sources.page_sources.clear()
|
|
374
|
+
sources.page_text_geometries.clear()
|
|
375
|
+
prepared_pages: list[_PreparedPage] = []
|
|
376
|
+
while pending:
|
|
377
|
+
page_index = len(prepared_pages)
|
|
378
|
+
source, geometry = pending.popleft()
|
|
379
|
+
prepared_pages.append(
|
|
380
|
+
_prepare_page_source(
|
|
381
|
+
source,
|
|
382
|
+
tight_bboxes=geometry.tight_bboxes,
|
|
383
|
+
origins=geometry.origins,
|
|
384
|
+
geometry_plan=geometry_plan,
|
|
385
|
+
page_index=page_index,
|
|
386
|
+
style_lines=sources.page_style_lines[page_index],
|
|
387
|
+
link_lines=sources.page_link_lines[page_index],
|
|
388
|
+
table_header_separator_bboxes=separators[page_index],
|
|
389
|
+
repaired_char_bboxes=repaired_chars_by_page.pop(page_index, {}),
|
|
390
|
+
)
|
|
391
|
+
)
|
|
392
|
+
# 删除对象所有者引用,不清空共享字符容器,公式重建副本仍可安全使用。
|
|
393
|
+
del source, geometry
|
|
394
|
+
return prepared_pages
|
|
395
|
+
|
|
396
|
+
|
|
397
|
+
@dataclass(frozen=True, slots=True)
|
|
398
|
+
class _DocumentTextProfiles:
|
|
399
|
+
"""分别保存原始正文尺度、规范正文尺度及全文标题原型。"""
|
|
400
|
+
|
|
401
|
+
body: _DocumentBodyProfile | None
|
|
402
|
+
canonical_body: _DocumentBodyProfile | None
|
|
403
|
+
title: _DocumentTitleProfile | None
|
|
404
|
+
|
|
405
|
+
|
|
406
|
+
def _classify_document_text(prepared_pages: list[_PreparedPage]) -> _DocumentTextProfiles:
|
|
407
|
+
"""按既有顺序分类跨页辅助文本,再统计正文并确认结构标题。"""
|
|
408
|
+
|
|
409
|
+
_classify_repeated_visual_headers(prepared_pages)
|
|
410
|
+
_classify_repeated_page_marginals(prepared_pages)
|
|
411
|
+
_classify_split_marginal_row_companions(prepared_pages)
|
|
412
|
+
_classify_single_page_compound_headers(prepared_pages)
|
|
413
|
+
_classify_rule_delimited_headers(prepared_pages)
|
|
414
|
+
_classify_rule_delimited_footers(prepared_pages)
|
|
415
|
+
_classify_page_number_outer_companions(prepared_pages)
|
|
416
|
+
_classify_page_footnote_trailing_footers(prepared_pages)
|
|
417
|
+
_classify_isolated_first_page_footer(prepared_pages)
|
|
418
|
+
document_body_profile = _infer_document_body_profile(prepared_pages)
|
|
419
|
+
canonical_body_profile = _infer_document_body_profile(
|
|
420
|
+
prepared_pages,
|
|
421
|
+
use_canonical_scale=True,
|
|
422
|
+
)
|
|
423
|
+
if document_body_profile is not None:
|
|
424
|
+
_classify_deferred_image_footnotes(
|
|
425
|
+
prepared_pages,
|
|
426
|
+
document_body_profile.body_height,
|
|
427
|
+
)
|
|
428
|
+
document_title_profile = _infer_document_title_profile(
|
|
429
|
+
prepared_pages,
|
|
430
|
+
document_body_profile,
|
|
431
|
+
)
|
|
432
|
+
_classify_document_structural_titles(
|
|
433
|
+
prepared_pages,
|
|
434
|
+
canonical_body_profile,
|
|
435
|
+
legacy_body_profile=document_body_profile,
|
|
436
|
+
document_title_profile=document_title_profile,
|
|
437
|
+
)
|
|
438
|
+
return _DocumentTextProfiles(document_body_profile, canonical_body_profile, document_title_profile)
|
|
439
|
+
|
|
440
|
+
|
|
441
|
+
def _materialize_document_inline(
|
|
442
|
+
finalized_pages: list[list[dict[str, Any]]],
|
|
443
|
+
prepared_pages: list[_PreparedPage],
|
|
444
|
+
sources: _DocumentSources,
|
|
445
|
+
script_diagnostics: list[dict[str, Any]] | None,
|
|
446
|
+
) -> None:
|
|
447
|
+
"""在页面归一化后按链接、样式、上下标顺序物化最终行内语义。"""
|
|
448
|
+
|
|
449
|
+
for page_index, (page_blocks, prepared, style_lines, link_lines, page_size) in enumerate(
|
|
450
|
+
zip(
|
|
451
|
+
finalized_pages,
|
|
452
|
+
prepared_pages,
|
|
453
|
+
sources.page_style_lines,
|
|
454
|
+
sources.page_link_lines,
|
|
455
|
+
sources.page_sizes,
|
|
456
|
+
strict=True,
|
|
457
|
+
)
|
|
458
|
+
):
|
|
459
|
+
apply_pdf_text_links(page_blocks, link_lines, page_size)
|
|
460
|
+
apply_pdf_text_styles(page_blocks, style_lines, page_size)
|
|
461
|
+
materialized_diagnostics = None
|
|
462
|
+
if script_diagnostics is not None:
|
|
463
|
+
materialized_diagnostics = []
|
|
464
|
+
script_diagnostics[page_index]["materialized_ranges"] = materialized_diagnostics
|
|
465
|
+
apply_pdf_text_scripts(
|
|
466
|
+
page_blocks,
|
|
467
|
+
prepared.script_lines,
|
|
468
|
+
page_size,
|
|
469
|
+
materialized_diagnostics=materialized_diagnostics,
|
|
470
|
+
)
|
|
471
|
+
materialize_pdf_inline_spans(page_blocks)
|
|
472
|
+
|
|
473
|
+
|
|
474
|
+
def _analyze_native_document(
|
|
475
|
+
pdf_doc: NativePdfSource,
|
|
476
|
+
*,
|
|
477
|
+
script_diagnostics: list[dict[str, Any]] | None = None,
|
|
478
|
+
geometry_diagnostics: list[dict[str, Any]] | None = None,
|
|
479
|
+
) -> list[list[dict[str, Any]]]:
|
|
480
|
+
"""逐页读取数字 PDF,并在轻量页面上完成跨页文本类型判定。"""
|
|
481
|
+
|
|
482
|
+
sources = _collect_document_sources(pdf_doc)
|
|
483
|
+
prepared_pages = _prepare_document_sources(sources, geometry_diagnostics=geometry_diagnostics)
|
|
484
|
+
if script_diagnostics is not None:
|
|
485
|
+
script_diagnostics.extend(
|
|
486
|
+
{
|
|
487
|
+
"page_index": page_index,
|
|
488
|
+
"page_size": prepared.page_size,
|
|
489
|
+
"script_lines": list(prepared.script_lines),
|
|
490
|
+
"lines": [
|
|
491
|
+
{
|
|
492
|
+
"source_index": line.source_index,
|
|
493
|
+
"text": line.text,
|
|
494
|
+
"bbox": line.bbox,
|
|
495
|
+
"angle": line.angle,
|
|
496
|
+
}
|
|
497
|
+
for line in prepared.remaining_lines
|
|
498
|
+
],
|
|
499
|
+
}
|
|
500
|
+
for page_index, prepared in enumerate(prepared_pages)
|
|
501
|
+
)
|
|
502
|
+
|
|
503
|
+
profiles = _classify_document_text(prepared_pages)
|
|
504
|
+
finalized_pages = [
|
|
505
|
+
_finalize_prepared_page(
|
|
506
|
+
prepared,
|
|
507
|
+
page_index,
|
|
508
|
+
canonical_body_profile=profiles.canonical_body,
|
|
509
|
+
document_body_profile=profiles.body,
|
|
510
|
+
document_title_profile=profiles.title,
|
|
511
|
+
)
|
|
512
|
+
for page_index, prepared in enumerate(prepared_pages)
|
|
513
|
+
]
|
|
514
|
+
_materialize_document_inline(finalized_pages, prepared_pages, sources, script_diagnostics)
|
|
515
|
+
return finalized_pages
|
|
516
|
+
|
|
517
|
+
|
|
518
|
+
def _prepare_page_source(
|
|
519
|
+
source: _PageSource,
|
|
520
|
+
*,
|
|
521
|
+
tight_bboxes: dict[int, BBox] | None = None,
|
|
522
|
+
origins: dict[int, tuple[float, float]] | None = None,
|
|
523
|
+
geometry_plan: DocumentGeometryPlan | None = None,
|
|
524
|
+
page_index: int = 0,
|
|
525
|
+
style_lines: list[PDFTextStyleLine] | None = None,
|
|
526
|
+
link_lines: list[PDFTextLinkLine] | None = None,
|
|
527
|
+
table_header_separator_bboxes: set[BBox] | None = None,
|
|
528
|
+
repaired_char_bboxes: dict[int, BBox] | None = None,
|
|
529
|
+
) -> _PreparedPage:
|
|
530
|
+
"""先认领视觉容器,再标注辅助文本并留下可跨页比较的轻量文本行。"""
|
|
531
|
+
|
|
532
|
+
protected_line_indices = {line.source_index for line in source.lines if line.semantic_type is not None}
|
|
533
|
+
analysis_source = replace(
|
|
534
|
+
source,
|
|
535
|
+
lines=[line for line in source.lines if line.source_index not in protected_line_indices],
|
|
536
|
+
)
|
|
537
|
+
form_bboxes = _select_form_image_bboxes(source)
|
|
538
|
+
strong_graphic_bboxes = _detect_strong_graphic_bboxes(analysis_source)
|
|
539
|
+
rule_code_blocks, claimed_rule_code_line_indices = _build_rule_delimited_code_blocks(
|
|
540
|
+
analysis_source,
|
|
541
|
+
form_bboxes + strong_graphic_bboxes + list(source.image_bboxes) + list(source.signature_bboxes),
|
|
542
|
+
)
|
|
543
|
+
rule_code_bboxes = [block["bbox"] for block in rule_code_blocks]
|
|
544
|
+
table_analysis_source = replace(
|
|
545
|
+
analysis_source,
|
|
546
|
+
drawing_lines=_table_detection_drawing_lines(
|
|
547
|
+
source,
|
|
548
|
+
table_header_separator_bboxes,
|
|
549
|
+
),
|
|
550
|
+
)
|
|
551
|
+
candidates = [
|
|
552
|
+
candidate
|
|
553
|
+
for candidate in _detect_table_candidates(
|
|
554
|
+
table_analysis_source,
|
|
555
|
+
excluded_bboxes=strong_graphic_bboxes + rule_code_bboxes,
|
|
556
|
+
)
|
|
557
|
+
if not any(_form_supersedes_nested_bbox(form_bbox, candidate.bbox) for form_bbox in form_bboxes)
|
|
558
|
+
]
|
|
559
|
+
# 候选检测仍避开预分类边缘文本;已确认表格物化时回到原始行,
|
|
560
|
+
# 让 core_bbox 内的误标页脚可被重新认领,表格外边缘文本不会被矩形扩张带入。
|
|
561
|
+
table_blocks, table_annotation_blocks, claimed_line_indices = _materialize_table_blocks(
|
|
562
|
+
source,
|
|
563
|
+
candidates,
|
|
564
|
+
tight_bboxes=tight_bboxes,
|
|
565
|
+
origins=origins,
|
|
566
|
+
)
|
|
567
|
+
claimed_line_indices.update(claimed_rule_code_line_indices)
|
|
568
|
+
table_bboxes = [block["bbox"] for block in table_blocks]
|
|
569
|
+
active_form_bboxes = [
|
|
570
|
+
form_bbox
|
|
571
|
+
for form_bbox in form_bboxes
|
|
572
|
+
if not any(
|
|
573
|
+
_bbox_overlap_in_smaller(form_bbox, table_bbox) >= _IMAGE_CONTAINER_OVERLAP_THRESHOLD for table_bbox in table_bboxes
|
|
574
|
+
)
|
|
575
|
+
]
|
|
576
|
+
code_blocks, claimed_code_line_indices = _build_code_blocks(
|
|
577
|
+
analysis_source,
|
|
578
|
+
table_bboxes + active_form_bboxes + strong_graphic_bboxes + list(source.image_bboxes) + list(source.signature_bboxes),
|
|
579
|
+
claimed_line_indices,
|
|
580
|
+
)
|
|
581
|
+
code_bboxes = [block["bbox"] for block in code_blocks]
|
|
582
|
+
form_image_blocks, claimed_form_line_indices = _build_form_image_blocks(
|
|
583
|
+
analysis_source,
|
|
584
|
+
active_form_bboxes,
|
|
585
|
+
claimed_line_indices | claimed_code_line_indices,
|
|
586
|
+
)
|
|
587
|
+
graphic_blocks, claimed_graphic_line_indices = _build_graphic_like_blocks(
|
|
588
|
+
analysis_source,
|
|
589
|
+
table_bboxes + active_form_bboxes + rule_code_bboxes + code_bboxes,
|
|
590
|
+
claimed_line_indices | claimed_code_line_indices | claimed_form_line_indices,
|
|
591
|
+
strong_graphic_bboxes,
|
|
592
|
+
)
|
|
593
|
+
raster_image_blocks, claimed_raster_line_indices = _build_raster_image_blocks(
|
|
594
|
+
analysis_source,
|
|
595
|
+
table_blocks + rule_code_blocks + code_blocks + form_image_blocks + graphic_blocks,
|
|
596
|
+
claimed_line_indices | claimed_code_line_indices | claimed_form_line_indices | claimed_graphic_line_indices,
|
|
597
|
+
)
|
|
598
|
+
vector_formula_blocks, claimed_vector_number_indices = _build_vector_formula_blocks(
|
|
599
|
+
analysis_source,
|
|
600
|
+
table_blocks + rule_code_blocks + code_blocks + form_image_blocks + graphic_blocks + raster_image_blocks,
|
|
601
|
+
claimed_line_indices
|
|
602
|
+
| claimed_code_line_indices
|
|
603
|
+
| claimed_form_line_indices
|
|
604
|
+
| claimed_graphic_line_indices
|
|
605
|
+
| claimed_raster_line_indices,
|
|
606
|
+
)
|
|
607
|
+
claimed_line_indices = (
|
|
608
|
+
claimed_line_indices
|
|
609
|
+
| claimed_code_line_indices
|
|
610
|
+
| claimed_form_line_indices
|
|
611
|
+
| claimed_graphic_line_indices
|
|
612
|
+
| claimed_raster_line_indices
|
|
613
|
+
| claimed_vector_number_indices
|
|
614
|
+
)
|
|
615
|
+
unclaimed_lines = [line for line in source.lines if line.source_index not in claimed_line_indices]
|
|
616
|
+
if geometry_plan is not None:
|
|
617
|
+
apply_line_geometry_repairs(
|
|
618
|
+
unclaimed_lines,
|
|
619
|
+
page_index=page_index,
|
|
620
|
+
plan=geometry_plan,
|
|
621
|
+
allow_y_trim=True,
|
|
622
|
+
)
|
|
623
|
+
next_source_index = (
|
|
624
|
+
max(
|
|
625
|
+
(line.source_index for line in source.lines),
|
|
626
|
+
default=-1,
|
|
627
|
+
)
|
|
628
|
+
+ 1
|
|
629
|
+
)
|
|
630
|
+
if geometry_plan is not None:
|
|
631
|
+
repaired_line_bboxes = {line.source_index: line.bbox for line in source.lines}
|
|
632
|
+
if repaired_char_bboxes is None:
|
|
633
|
+
# 单页内部入口保留独立调用能力;文档主链路已提前按页建立索引。
|
|
634
|
+
repaired_char_bboxes = {
|
|
635
|
+
char_idx: repair.layout_bbox
|
|
636
|
+
for (repair_page_index, char_idx), repair in geometry_plan.char_repairs.items()
|
|
637
|
+
if repair_page_index == page_index
|
|
638
|
+
}
|
|
639
|
+
unclaimed_lines, resplits = _resplit_native_visual_runs(
|
|
640
|
+
unclaimed_lines,
|
|
641
|
+
source.page_size,
|
|
642
|
+
repaired_char_bboxes,
|
|
643
|
+
source_index_start=next_source_index,
|
|
644
|
+
)
|
|
645
|
+
aligned_styles, aligned_links = _realign_repaired_text_evidence(
|
|
646
|
+
style_lines or [],
|
|
647
|
+
link_lines or [],
|
|
648
|
+
repaired_line_bboxes,
|
|
649
|
+
resplits,
|
|
650
|
+
)
|
|
651
|
+
if style_lines is not None and aligned_styles is not style_lines:
|
|
652
|
+
style_lines[:] = aligned_styles
|
|
653
|
+
if link_lines is not None and aligned_links is not link_lines:
|
|
654
|
+
link_lines[:] = aligned_links
|
|
655
|
+
next_source_index = max(
|
|
656
|
+
next_source_index,
|
|
657
|
+
max(
|
|
658
|
+
(line.source_index for line in unclaimed_lines),
|
|
659
|
+
default=-1,
|
|
660
|
+
)
|
|
661
|
+
+ 1,
|
|
662
|
+
)
|
|
663
|
+
remaining_lines = _split_parallel_graphic_rule_rows(
|
|
664
|
+
unclaimed_lines,
|
|
665
|
+
source.drawing_lines,
|
|
666
|
+
[block["bbox"] for block in (form_image_blocks + graphic_blocks + raster_image_blocks)],
|
|
667
|
+
table_bboxes,
|
|
668
|
+
source.page_size,
|
|
669
|
+
source_index_start=next_source_index,
|
|
670
|
+
)
|
|
671
|
+
canonical_formula_source_lines = (
|
|
672
|
+
[
|
|
673
|
+
replace(
|
|
674
|
+
line,
|
|
675
|
+
chars=list(line.chars),
|
|
676
|
+
inline_math_regions=list(line.inline_math_regions),
|
|
677
|
+
)
|
|
678
|
+
for line in remaining_lines
|
|
679
|
+
]
|
|
680
|
+
if geometry_plan is not None and geometry_plan.document_style_anomaly
|
|
681
|
+
else []
|
|
682
|
+
)
|
|
683
|
+
remaining_lines = _merge_same_baseline_text_lines(
|
|
684
|
+
remaining_lines,
|
|
685
|
+
source.page_size,
|
|
686
|
+
table_bboxes,
|
|
687
|
+
)
|
|
688
|
+
remaining_lines = _merge_overlapping_inline_text_clusters(
|
|
689
|
+
remaining_lines,
|
|
690
|
+
source.page_size,
|
|
691
|
+
table_bboxes,
|
|
692
|
+
)
|
|
693
|
+
# 首轮同行合并可能补齐宿主 bbox,使相邻 hard-split 尾段具备二次闭包条件。
|
|
694
|
+
remaining_lines = _merge_same_baseline_text_lines(
|
|
695
|
+
remaining_lines,
|
|
696
|
+
source.page_size,
|
|
697
|
+
table_bboxes,
|
|
698
|
+
)
|
|
699
|
+
formula_candidate_lines = [line for line in remaining_lines if line.formula_candidate_only]
|
|
700
|
+
remaining_lines = [line for line in remaining_lines if not line.formula_candidate_only]
|
|
701
|
+
script_lines = detect_pdf_text_script_lines(
|
|
702
|
+
remaining_lines,
|
|
703
|
+
source.page_size,
|
|
704
|
+
tight_bboxes or {},
|
|
705
|
+
origins or {},
|
|
706
|
+
all_chars=source.chars,
|
|
707
|
+
drawing_lines=source.drawing_lines,
|
|
708
|
+
)
|
|
709
|
+
_compact_prepared_lines(remaining_lines, source.page_size)
|
|
710
|
+
_compact_prepared_lines(formula_candidate_lines, source.page_size)
|
|
711
|
+
prepared = _PreparedPage(
|
|
712
|
+
page_size=source.page_size,
|
|
713
|
+
remaining_lines=remaining_lines,
|
|
714
|
+
table_bboxes=table_bboxes,
|
|
715
|
+
drawing_lines=source.drawing_lines,
|
|
716
|
+
fixed_blocks=(
|
|
717
|
+
rule_code_blocks
|
|
718
|
+
+ table_annotation_blocks
|
|
719
|
+
+ table_blocks
|
|
720
|
+
+ code_blocks
|
|
721
|
+
+ form_image_blocks
|
|
722
|
+
+ graphic_blocks
|
|
723
|
+
+ raster_image_blocks
|
|
724
|
+
+ vector_formula_blocks
|
|
725
|
+
),
|
|
726
|
+
canonical_formula_source_lines=canonical_formula_source_lines,
|
|
727
|
+
script_lines=script_lines,
|
|
728
|
+
formula_candidate_lines=formula_candidate_lines,
|
|
729
|
+
)
|
|
730
|
+
_classify_page_auxiliary_text(prepared)
|
|
731
|
+
return prepared
|
|
732
|
+
|
|
733
|
+
|
|
734
|
+
def _compact_prepared_lines(
|
|
735
|
+
lines: list[_LineItem],
|
|
736
|
+
page_size: tuple[float, float],
|
|
737
|
+
) -> None:
|
|
738
|
+
"""缓存后续仍需的字符尺度并释放字符字典,限制跨页阶段内存占用。"""
|
|
739
|
+
|
|
740
|
+
for line in lines:
|
|
741
|
+
if line.median_glyph_width is None:
|
|
742
|
+
line.median_glyph_width = _median_native_glyph_width(line, page_size)
|
|
743
|
+
line.chars.clear()
|
|
744
|
+
|
|
745
|
+
|
|
746
|
+
def _rebuild_canonical_formula_blocks(
|
|
747
|
+
prepared: _PreparedPage,
|
|
748
|
+
excluded_source_indices: set[int],
|
|
749
|
+
) -> list[dict[str, Any]]:
|
|
750
|
+
"""从容器认领后的未合并行重放 canonical 公式路径,避免 loose 行高改变公式成员顺序。"""
|
|
751
|
+
|
|
752
|
+
replay_lines = [
|
|
753
|
+
replace(
|
|
754
|
+
line,
|
|
755
|
+
chars=list(line.chars),
|
|
756
|
+
style_scale_repaired=True,
|
|
757
|
+
inline_math_regions=list(line.inline_math_regions),
|
|
758
|
+
)
|
|
759
|
+
for line in prepared.canonical_formula_source_lines
|
|
760
|
+
if line.source_index not in excluded_source_indices
|
|
761
|
+
]
|
|
762
|
+
replay_lines = _merge_same_baseline_text_lines(
|
|
763
|
+
replay_lines,
|
|
764
|
+
prepared.page_size,
|
|
765
|
+
prepared.table_bboxes,
|
|
766
|
+
)
|
|
767
|
+
replay_lines = _merge_overlapping_inline_text_clusters(
|
|
768
|
+
replay_lines,
|
|
769
|
+
prepared.page_size,
|
|
770
|
+
prepared.table_bboxes,
|
|
771
|
+
)
|
|
772
|
+
replay_lines = _merge_same_baseline_text_lines(
|
|
773
|
+
replay_lines,
|
|
774
|
+
prepared.page_size,
|
|
775
|
+
prepared.table_bboxes,
|
|
776
|
+
)
|
|
777
|
+
formula_candidate_lines = [line for line in replay_lines if line.formula_candidate_only]
|
|
778
|
+
replay_lines = [line for line in replay_lines if not line.formula_candidate_only]
|
|
779
|
+
formula_input = _restore_dense_split_visual_rows(
|
|
780
|
+
replay_lines + formula_candidate_lines,
|
|
781
|
+
prepared.page_size,
|
|
782
|
+
prepared.table_bboxes,
|
|
783
|
+
)
|
|
784
|
+
formula_input = _merge_same_baseline_text_lines(
|
|
785
|
+
formula_input,
|
|
786
|
+
prepared.page_size,
|
|
787
|
+
prepared.table_bboxes,
|
|
788
|
+
)
|
|
789
|
+
blocks, _remaining_lines = _build_formula_like_blocks(
|
|
790
|
+
formula_input,
|
|
791
|
+
prepared.table_bboxes,
|
|
792
|
+
prepared.page_size,
|
|
793
|
+
drawing_lines=prepared.drawing_lines,
|
|
794
|
+
)
|
|
795
|
+
return blocks
|
|
796
|
+
|
|
797
|
+
|
|
798
|
+
def _formula_block_inventory(
|
|
799
|
+
blocks: list[dict[str, Any]],
|
|
800
|
+
) -> list[tuple[float, float, float, float]]:
|
|
801
|
+
"""返回公式重放与正常路径可比较的稳定 bbox 库存。"""
|
|
802
|
+
|
|
803
|
+
return sorted(
|
|
804
|
+
tuple(float(value) for value in block["bbox"])
|
|
805
|
+
for block in blocks
|
|
806
|
+
if block.get("type") == "equation" and isinstance(block.get("bbox"), (list, tuple)) and len(block["bbox"]) == 4
|
|
807
|
+
)
|
|
808
|
+
|
|
809
|
+
|
|
810
|
+
def _apply_post_aggregation_tight_bboxes(
|
|
811
|
+
blocks: list[dict[str, Any]],
|
|
812
|
+
page_size: tuple[float, float],
|
|
813
|
+
) -> None:
|
|
814
|
+
"""在 block 聚合完成后应用 tight+1pt 框,并同步最终公开行框。"""
|
|
815
|
+
|
|
816
|
+
for block in blocks:
|
|
817
|
+
candidate_bbox = _coerce_bbox(
|
|
818
|
+
block.pop("_tight_output_bbox", None),
|
|
819
|
+
)
|
|
820
|
+
output_line_bboxes = block.pop(
|
|
821
|
+
"_local_output_line_bboxes",
|
|
822
|
+
None,
|
|
823
|
+
)
|
|
824
|
+
output_bbox_repaired = block.pop(
|
|
825
|
+
"_output_bbox_repaired",
|
|
826
|
+
False,
|
|
827
|
+
)
|
|
828
|
+
if output_bbox_repaired is True and isinstance(output_line_bboxes, list) and output_line_bboxes:
|
|
829
|
+
local_bboxes = [_coerce_bbox(value) for value in output_line_bboxes]
|
|
830
|
+
if all(value is not None for value in local_bboxes):
|
|
831
|
+
resolved_local_bboxes = [value for value in local_bboxes if value is not None]
|
|
832
|
+
block["_local_line_bboxes"] = resolved_local_bboxes
|
|
833
|
+
angle = int(block.get("angle", 0) or 0) % 360
|
|
834
|
+
candidate_bbox = _bbox_union_many(
|
|
835
|
+
[
|
|
836
|
+
_rotate_bbox_from_upright(
|
|
837
|
+
value,
|
|
838
|
+
page_size,
|
|
839
|
+
angle,
|
|
840
|
+
)
|
|
841
|
+
for value in resolved_local_bboxes
|
|
842
|
+
]
|
|
843
|
+
)
|
|
844
|
+
clipped_bbox = _clip_bbox(candidate_bbox, page_size)
|
|
845
|
+
if clipped_bbox is not None:
|
|
846
|
+
block["bbox"] = clipped_bbox
|
|
847
|
+
|
|
848
|
+
|
|
849
|
+
def _finalize_prepared_page(
|
|
850
|
+
prepared: _PreparedPage,
|
|
851
|
+
page_index: int,
|
|
852
|
+
*,
|
|
853
|
+
canonical_body_profile: _DocumentBodyProfile | None = None,
|
|
854
|
+
document_body_profile: _DocumentBodyProfile | None = None,
|
|
855
|
+
document_title_profile: _DocumentTitleProfile | None = None,
|
|
856
|
+
) -> list[dict[str, Any]]:
|
|
857
|
+
"""按预分类语义、公式、标题、正文的优先级完成单页文本并排序。"""
|
|
858
|
+
|
|
859
|
+
semantic_lines = [line for line in prepared.remaining_lines if line.semantic_type is not None]
|
|
860
|
+
unresolved_lines = [line for line in prepared.remaining_lines if line.semantic_type is None]
|
|
861
|
+
container_bboxes = [block["bbox"] for block in prepared.fixed_blocks]
|
|
862
|
+
index_blocks, unresolved_lines = _extract_index_blocks(
|
|
863
|
+
unresolved_lines,
|
|
864
|
+
prepared.page_size,
|
|
865
|
+
container_bboxes,
|
|
866
|
+
require_heading=True,
|
|
867
|
+
)
|
|
868
|
+
semantic_lines.extend(line for line in unresolved_lines if line.semantic_type is not None)
|
|
869
|
+
formula_input = [
|
|
870
|
+
*(line for line in unresolved_lines if line.semantic_type is None),
|
|
871
|
+
*prepared.formula_candidate_lines,
|
|
872
|
+
]
|
|
873
|
+
original_style_scale_state = {line.source_index: line.style_scale_repaired for line in formula_input}
|
|
874
|
+
if prepared.canonical_formula_geometry:
|
|
875
|
+
for line in formula_input:
|
|
876
|
+
line.style_scale_repaired = True
|
|
877
|
+
formula_input = _restore_dense_split_visual_rows(
|
|
878
|
+
formula_input,
|
|
879
|
+
prepared.page_size,
|
|
880
|
+
prepared.table_bboxes,
|
|
881
|
+
)
|
|
882
|
+
formula_input = _merge_same_baseline_text_lines(
|
|
883
|
+
formula_input,
|
|
884
|
+
prepared.page_size,
|
|
885
|
+
prepared.table_bboxes,
|
|
886
|
+
)
|
|
887
|
+
formula_blocks, remaining_lines = _build_formula_like_blocks(
|
|
888
|
+
formula_input,
|
|
889
|
+
prepared.table_bboxes,
|
|
890
|
+
prepared.page_size,
|
|
891
|
+
drawing_lines=prepared.drawing_lines,
|
|
892
|
+
)
|
|
893
|
+
if prepared.canonical_formula_geometry and prepared.canonical_formula_source_lines:
|
|
894
|
+
canonical_formula_blocks = _rebuild_canonical_formula_blocks(
|
|
895
|
+
prepared,
|
|
896
|
+
{line.source_index for line in semantic_lines},
|
|
897
|
+
)
|
|
898
|
+
if _formula_block_inventory(
|
|
899
|
+
canonical_formula_blocks,
|
|
900
|
+
) == _formula_block_inventory(formula_blocks):
|
|
901
|
+
formula_blocks = canonical_formula_blocks
|
|
902
|
+
prepared.canonical_formula_source_lines.clear()
|
|
903
|
+
if prepared.canonical_formula_geometry:
|
|
904
|
+
for line in remaining_lines:
|
|
905
|
+
line.style_scale_repaired = original_style_scale_state.get(
|
|
906
|
+
line.source_index,
|
|
907
|
+
line.style_scale_repaired,
|
|
908
|
+
)
|
|
909
|
+
fallback_index_blocks, remaining_lines = _extract_index_blocks(
|
|
910
|
+
remaining_lines,
|
|
911
|
+
prepared.page_size,
|
|
912
|
+
container_bboxes,
|
|
913
|
+
)
|
|
914
|
+
index_blocks.extend(fallback_index_blocks)
|
|
915
|
+
semantic_lines.extend(line for line in remaining_lines if line.semantic_type is not None)
|
|
916
|
+
remaining_lines = [line for line in remaining_lines if line.semantic_type is None]
|
|
917
|
+
title_container_bboxes = [
|
|
918
|
+
block["bbox"] for block in prepared.fixed_blocks if not isinstance(block.get("_inline_visual_row_id"), int)
|
|
919
|
+
]
|
|
920
|
+
caption_container_bboxes = [block["bbox"] for block in prepared.fixed_blocks if block.get("type") in {"image", "code"}]
|
|
921
|
+
_classify_explicit_section_titles(
|
|
922
|
+
remaining_lines,
|
|
923
|
+
prepared.page_size,
|
|
924
|
+
container_bboxes=title_container_bboxes,
|
|
925
|
+
document_body_profile=document_body_profile,
|
|
926
|
+
)
|
|
927
|
+
_classify_inline_typography_reset_titles(
|
|
928
|
+
remaining_lines,
|
|
929
|
+
prepared.page_size,
|
|
930
|
+
container_bboxes=title_container_bboxes,
|
|
931
|
+
document_body_profile=document_body_profile,
|
|
932
|
+
)
|
|
933
|
+
_classify_body_height_section_titles(
|
|
934
|
+
remaining_lines,
|
|
935
|
+
prepared.page_size,
|
|
936
|
+
container_bboxes=title_container_bboxes,
|
|
937
|
+
document_body_profile=document_body_profile,
|
|
938
|
+
page_index=page_index,
|
|
939
|
+
)
|
|
940
|
+
_classify_page_titles(
|
|
941
|
+
remaining_lines,
|
|
942
|
+
prepared.page_size,
|
|
943
|
+
page_index=page_index,
|
|
944
|
+
container_bboxes=title_container_bboxes,
|
|
945
|
+
caption_container_bboxes=caption_container_bboxes,
|
|
946
|
+
document_body_profile=document_body_profile,
|
|
947
|
+
document_title_profile=document_title_profile,
|
|
948
|
+
)
|
|
949
|
+
_promote_noninitial_document_title_band(
|
|
950
|
+
remaining_lines,
|
|
951
|
+
prepared.page_size,
|
|
952
|
+
page_index=page_index,
|
|
953
|
+
container_bboxes=title_container_bboxes,
|
|
954
|
+
document_body_profile=(canonical_body_profile or document_body_profile),
|
|
955
|
+
title_candidate_source_indices={
|
|
956
|
+
line.source_index for line in remaining_lines if line.semantic_type == "paragraph_title"
|
|
957
|
+
},
|
|
958
|
+
)
|
|
959
|
+
remaining_lines = _merge_title_resolved_visual_rows(
|
|
960
|
+
remaining_lines,
|
|
961
|
+
prepared.page_size,
|
|
962
|
+
)
|
|
963
|
+
remaining_lines = _merge_post_semantic_text_runs(
|
|
964
|
+
remaining_lines,
|
|
965
|
+
prepared.page_size,
|
|
966
|
+
prepared.table_bboxes,
|
|
967
|
+
)
|
|
968
|
+
text_blocks = _build_text_blocks(
|
|
969
|
+
semantic_lines + remaining_lines,
|
|
970
|
+
prepared.table_bboxes,
|
|
971
|
+
prepared.page_size,
|
|
972
|
+
prepared.drawing_lines,
|
|
973
|
+
page_footnote_groups=prepared.page_footnote_groups,
|
|
974
|
+
page_index=page_index,
|
|
975
|
+
visual_bboxes=[block["bbox"] for block in prepared.fixed_blocks if block.get("type") == "image"],
|
|
976
|
+
)
|
|
977
|
+
text_blocks = _merge_multiline_title_blocks(text_blocks)
|
|
978
|
+
text_blocks = _merge_front_matter_column_blocks(
|
|
979
|
+
text_blocks,
|
|
980
|
+
prepared.page_size,
|
|
981
|
+
page_index=page_index,
|
|
982
|
+
)
|
|
983
|
+
text_blocks = _merge_image_caption_text_blocks(
|
|
984
|
+
text_blocks,
|
|
985
|
+
[block["bbox"] for block in prepared.fixed_blocks if block.get("type") == "image"],
|
|
986
|
+
)
|
|
987
|
+
text_blocks = _merge_fragmented_header_blocks(text_blocks)
|
|
988
|
+
text_blocks = _merge_repeated_compact_title_continuations(
|
|
989
|
+
text_blocks,
|
|
990
|
+
prepared.page_size,
|
|
991
|
+
)
|
|
992
|
+
absolute_blocks = prepared.fixed_blocks + formula_blocks + index_blocks + text_blocks
|
|
993
|
+
_apply_post_aggregation_tight_bboxes(
|
|
994
|
+
absolute_blocks,
|
|
995
|
+
prepared.page_size,
|
|
996
|
+
)
|
|
997
|
+
visual_annotation_regions = _classify_and_bind_visual_annotations(
|
|
998
|
+
absolute_blocks,
|
|
999
|
+
prepared.page_size,
|
|
1000
|
+
merge_text_block_group=_merge_internal_text_block_group,
|
|
1001
|
+
)
|
|
1002
|
+
sorted_blocks = _sort_blocks_with_visual_row_groups(
|
|
1003
|
+
absolute_blocks,
|
|
1004
|
+
prepared.page_size,
|
|
1005
|
+
visual_annotation_regions=visual_annotation_regions,
|
|
1006
|
+
)
|
|
1007
|
+
return [
|
|
1008
|
+
normalized for block in sorted_blocks if (normalized := _normalize_output_block(block, prepared.page_size)) is not None
|
|
1009
|
+
]
|
|
1010
|
+
|
|
1011
|
+
|
|
1012
|
+
def _analyze_page_source(source: _PageSource) -> list[dict[str, Any]]:
|
|
1013
|
+
"""兼容单页测试入口;单页不凭边缘位置猜测页眉、页脚或页码。"""
|
|
1014
|
+
|
|
1015
|
+
if not source.lines and not source.image_bboxes and not source.signature_bboxes:
|
|
1016
|
+
return []
|
|
1017
|
+
return _finalize_prepared_page(_prepare_page_source(source), page_index=0)
|
|
1018
|
+
|
|
1019
|
+
|
|
1020
|
+
def _sort_blocks_with_visual_row_groups(
|
|
1021
|
+
blocks: list[dict[str, Any]],
|
|
1022
|
+
page_size: tuple[float, float],
|
|
1023
|
+
*,
|
|
1024
|
+
visual_annotation_regions: list[list[dict[str, Any]]] | None = None,
|
|
1025
|
+
) -> list[dict[str, Any]]:
|
|
1026
|
+
"""把视觉注释区域和拆分粗行包装成虚拟项排序,再按各自局部顺序展开。"""
|
|
1027
|
+
|
|
1028
|
+
top_marginals: list[dict[str, Any]] = []
|
|
1029
|
+
bottom_marginals: list[dict[str, Any]] = []
|
|
1030
|
+
body_blocks: list[dict[str, Any]] = []
|
|
1031
|
+
local_page_height = page_size[1]
|
|
1032
|
+
for block in blocks:
|
|
1033
|
+
block_type = block.get("type")
|
|
1034
|
+
bbox = block.get("bbox")
|
|
1035
|
+
if block_type == "header" or (
|
|
1036
|
+
block_type == "page_number" and isinstance(bbox, (list, tuple)) and _bbox_center_y(bbox) <= 0.5 * local_page_height
|
|
1037
|
+
):
|
|
1038
|
+
top_marginals.append(block)
|
|
1039
|
+
elif block_type in {"footer", "page_footnote"} or block_type == "page_number":
|
|
1040
|
+
bottom_marginals.append(block)
|
|
1041
|
+
else:
|
|
1042
|
+
body_blocks.append(block)
|
|
1043
|
+
|
|
1044
|
+
body_index_by_identity = {id(block): index for index, block in enumerate(body_blocks)}
|
|
1045
|
+
region_groups: list[list[dict[str, Any]]] = []
|
|
1046
|
+
region_consumed_indices: set[int] = set()
|
|
1047
|
+
for region in visual_annotation_regions or []:
|
|
1048
|
+
indices = [body_index_by_identity[id(member)] for member in region if id(member) in body_index_by_identity]
|
|
1049
|
+
if len(indices) < 2 or any(index in region_consumed_indices for index in indices):
|
|
1050
|
+
continue
|
|
1051
|
+
members = [body_blocks[index] for index in indices]
|
|
1052
|
+
for member in members:
|
|
1053
|
+
member["_visual_annotation_region_member"] = True
|
|
1054
|
+
region_groups.append(members)
|
|
1055
|
+
region_consumed_indices.update(indices)
|
|
1056
|
+
|
|
1057
|
+
inline_grouped_indices = _collect_inline_image_text_groups(
|
|
1058
|
+
body_blocks,
|
|
1059
|
+
excluded_indices=region_consumed_indices,
|
|
1060
|
+
)
|
|
1061
|
+
inline_consumed_indices = {index for indices in inline_grouped_indices.values() for index in indices}
|
|
1062
|
+
grouped_indices: dict[int, list[int]] = {}
|
|
1063
|
+
for index, block in enumerate(body_blocks):
|
|
1064
|
+
if index in region_consumed_indices or index in inline_consumed_indices:
|
|
1065
|
+
continue
|
|
1066
|
+
row_id = block.get("_single_run_row_id")
|
|
1067
|
+
if isinstance(row_id, int):
|
|
1068
|
+
grouped_indices.setdefault(row_id, []).append(index)
|
|
1069
|
+
|
|
1070
|
+
virtual_groups: list[dict[str, Any]] = []
|
|
1071
|
+
consumed_indices: set[int] = set(region_consumed_indices | inline_consumed_indices)
|
|
1072
|
+
for members in region_groups:
|
|
1073
|
+
virtual_groups.append(
|
|
1074
|
+
{
|
|
1075
|
+
"type": "_xycut_visual_annotation_region",
|
|
1076
|
+
"bbox": _bbox_union_many([member["bbox"] for member in members]),
|
|
1077
|
+
"angle": members[0].get("angle", 0),
|
|
1078
|
+
"content": "",
|
|
1079
|
+
"_members": members,
|
|
1080
|
+
"_visual_annotation_region": True,
|
|
1081
|
+
}
|
|
1082
|
+
)
|
|
1083
|
+
for row_id, indices in inline_grouped_indices.items():
|
|
1084
|
+
members = [body_blocks[index] for index in indices]
|
|
1085
|
+
virtual_groups.append(
|
|
1086
|
+
{
|
|
1087
|
+
"type": "_xycut_visual_row_group",
|
|
1088
|
+
"bbox": _bbox_union_many([member["bbox"] for member in members]),
|
|
1089
|
+
"angle": members[0].get("angle", 0),
|
|
1090
|
+
"content": "",
|
|
1091
|
+
"_members": members,
|
|
1092
|
+
"_inline_visual_row_id": row_id,
|
|
1093
|
+
}
|
|
1094
|
+
)
|
|
1095
|
+
for row_id, indices in grouped_indices.items():
|
|
1096
|
+
if len(indices) < 2:
|
|
1097
|
+
continue
|
|
1098
|
+
members = [body_blocks[index] for index in indices]
|
|
1099
|
+
virtual_group = {
|
|
1100
|
+
"type": "_xycut_visual_row_group",
|
|
1101
|
+
"bbox": _bbox_union_many([member["bbox"] for member in members]),
|
|
1102
|
+
"angle": members[0].get("angle", 0),
|
|
1103
|
+
"content": "",
|
|
1104
|
+
"_members": members,
|
|
1105
|
+
}
|
|
1106
|
+
virtual_groups.append(virtual_group)
|
|
1107
|
+
consumed_indices.update(indices)
|
|
1108
|
+
|
|
1109
|
+
sortable_blocks = [block for index, block in enumerate(body_blocks) if index not in consumed_indices]
|
|
1110
|
+
sortable_blocks.extend(virtual_groups)
|
|
1111
|
+
sorted_payloads = sort_entries(sortable_blocks)
|
|
1112
|
+
output: list[dict[str, Any]] = []
|
|
1113
|
+
for payload in sorted_payloads:
|
|
1114
|
+
members = payload.get("_members")
|
|
1115
|
+
if not isinstance(members, list):
|
|
1116
|
+
output.append(payload)
|
|
1117
|
+
continue
|
|
1118
|
+
if payload.get("_visual_annotation_region") is True:
|
|
1119
|
+
output.extend(members)
|
|
1120
|
+
continue
|
|
1121
|
+
if isinstance(payload.get("_inline_visual_row_id"), int):
|
|
1122
|
+
members.sort(
|
|
1123
|
+
key=lambda member: _inline_visual_group_member_sort_key(
|
|
1124
|
+
member,
|
|
1125
|
+
page_size,
|
|
1126
|
+
)
|
|
1127
|
+
)
|
|
1128
|
+
output.extend(members)
|
|
1129
|
+
continue
|
|
1130
|
+
angle = int(payload.get("angle", 0) or 0) % 360
|
|
1131
|
+
members.sort(
|
|
1132
|
+
key=lambda member: (
|
|
1133
|
+
_rotate_bbox_to_upright(member["bbox"], page_size, angle)[0],
|
|
1134
|
+
_rotate_bbox_to_upright(member["bbox"], page_size, angle)[1],
|
|
1135
|
+
)
|
|
1136
|
+
)
|
|
1137
|
+
output.extend(members)
|
|
1138
|
+
output = _stabilize_overlapping_lane_order(output, page_size)
|
|
1139
|
+
return [
|
|
1140
|
+
*_sort_marginal_blocks(top_marginals, page_size),
|
|
1141
|
+
*output,
|
|
1142
|
+
*_sort_marginal_blocks(bottom_marginals, page_size),
|
|
1143
|
+
]
|
|
1144
|
+
|
|
1145
|
+
|
|
1146
|
+
def _collect_inline_image_text_groups(
|
|
1147
|
+
body_blocks: list[dict[str, Any]],
|
|
1148
|
+
*,
|
|
1149
|
+
excluded_indices: set[int] | None = None,
|
|
1150
|
+
) -> dict[int, list[int]]:
|
|
1151
|
+
"""把复合图片与包含同一首行的正文块组成专用排序组。"""
|
|
1152
|
+
|
|
1153
|
+
excluded_indices = excluded_indices or set()
|
|
1154
|
+
image_indices_by_row: dict[int, list[int]] = {}
|
|
1155
|
+
for index, block in enumerate(body_blocks):
|
|
1156
|
+
if index in excluded_indices:
|
|
1157
|
+
continue
|
|
1158
|
+
row_id = block.get("_inline_visual_row_id")
|
|
1159
|
+
if block.get("type") == "image" and isinstance(row_id, int):
|
|
1160
|
+
image_indices_by_row.setdefault(row_id, []).append(index)
|
|
1161
|
+
|
|
1162
|
+
output: dict[int, list[int]] = {}
|
|
1163
|
+
consumed_text_indices: set[int] = set()
|
|
1164
|
+
for row_id, image_indices in sorted(image_indices_by_row.items()):
|
|
1165
|
+
text_indices = [
|
|
1166
|
+
index
|
|
1167
|
+
for index, block in enumerate(body_blocks)
|
|
1168
|
+
if index not in excluded_indices
|
|
1169
|
+
and index not in consumed_text_indices
|
|
1170
|
+
and block.get("type") == "text"
|
|
1171
|
+
and isinstance(block.get("_visual_row_ids"), set)
|
|
1172
|
+
and row_id in block["_visual_row_ids"]
|
|
1173
|
+
]
|
|
1174
|
+
if not text_indices:
|
|
1175
|
+
continue
|
|
1176
|
+
output[row_id] = [*image_indices, *text_indices]
|
|
1177
|
+
consumed_text_indices.update(text_indices)
|
|
1178
|
+
return output
|
|
1179
|
+
|
|
1180
|
+
|
|
1181
|
+
def _inline_visual_group_member_sort_key(
|
|
1182
|
+
block: dict[str, Any],
|
|
1183
|
+
page_size: tuple[float, float],
|
|
1184
|
+
) -> tuple[float, float, int]:
|
|
1185
|
+
"""按图片位置或正文首个局部行位置确定复合视觉行的组内顺序。"""
|
|
1186
|
+
|
|
1187
|
+
local_line_bboxes = block.get("_local_line_bboxes")
|
|
1188
|
+
if block.get("type") == "text" and isinstance(local_line_bboxes, list) and local_line_bboxes:
|
|
1189
|
+
local_bbox = local_line_bboxes[0]
|
|
1190
|
+
else:
|
|
1191
|
+
angle = int(block.get("angle", 0) or 0) % 360
|
|
1192
|
+
local_bbox = _rotate_bbox_to_upright(
|
|
1193
|
+
block["bbox"],
|
|
1194
|
+
page_size,
|
|
1195
|
+
angle,
|
|
1196
|
+
)
|
|
1197
|
+
return (
|
|
1198
|
+
float(local_bbox[0]),
|
|
1199
|
+
float(local_bbox[1]),
|
|
1200
|
+
0 if block.get("type") == "image" else 1,
|
|
1201
|
+
)
|
|
1202
|
+
|
|
1203
|
+
|
|
1204
|
+
def _sort_marginal_blocks(
|
|
1205
|
+
blocks: list[dict[str, Any]],
|
|
1206
|
+
page_size: tuple[float, float],
|
|
1207
|
+
) -> list[dict[str, Any]]:
|
|
1208
|
+
"""先按视觉中心聚合边缘同排块,再按行内 x 排序以消除字体框顶边抖动。"""
|
|
1209
|
+
|
|
1210
|
+
if len(blocks) < 2:
|
|
1211
|
+
return list(blocks)
|
|
1212
|
+
geometry = [
|
|
1213
|
+
(
|
|
1214
|
+
block,
|
|
1215
|
+
_rotate_bbox_to_upright(
|
|
1216
|
+
block["bbox"],
|
|
1217
|
+
page_size,
|
|
1218
|
+
int(block.get("angle", 0) or 0) % 360,
|
|
1219
|
+
),
|
|
1220
|
+
)
|
|
1221
|
+
for block in blocks
|
|
1222
|
+
]
|
|
1223
|
+
heights = [
|
|
1224
|
+
float(height)
|
|
1225
|
+
for block in blocks
|
|
1226
|
+
for height in block.get("_line_heights", [])
|
|
1227
|
+
if isinstance(height, (int, float)) and height > 0
|
|
1228
|
+
]
|
|
1229
|
+
median_height = sorted(heights)[len(heights) // 2] if heights else 1.0
|
|
1230
|
+
rows: list[list[tuple[dict[str, Any], tuple[float, float, float, float]]]] = []
|
|
1231
|
+
for item in sorted(geometry, key=lambda value: _bbox_center_y(value[1])):
|
|
1232
|
+
target = next(
|
|
1233
|
+
(
|
|
1234
|
+
row
|
|
1235
|
+
for row in rows
|
|
1236
|
+
if abs(_bbox_center_y(item[1]) - sum(_bbox_center_y(member[1]) for member in row) / len(row))
|
|
1237
|
+
<= 0.75 * median_height
|
|
1238
|
+
),
|
|
1239
|
+
None,
|
|
1240
|
+
)
|
|
1241
|
+
if target is None:
|
|
1242
|
+
rows.append([item])
|
|
1243
|
+
else:
|
|
1244
|
+
target.append(item)
|
|
1245
|
+
rows.sort(key=lambda row: sum(_bbox_center_y(member[1]) for member in row) / len(row))
|
|
1246
|
+
return [block for row in rows for block, _bbox in sorted(row, key=lambda member: member[1][0])]
|
|
1247
|
+
|
|
1248
|
+
|
|
1249
|
+
def _stabilize_overlapping_lane_order(
|
|
1250
|
+
blocks: list[dict[str, Any]],
|
|
1251
|
+
page_size: tuple[float, float],
|
|
1252
|
+
) -> list[dict[str, Any]]:
|
|
1253
|
+
"""对同栏轻微重叠块按视觉中心纠正局部逆序,不改变跨栏主阅读顺序。"""
|
|
1254
|
+
|
|
1255
|
+
output = list(blocks)
|
|
1256
|
+
for _pass_index in range(len(output)):
|
|
1257
|
+
changed = False
|
|
1258
|
+
for index in range(len(output) - 1):
|
|
1259
|
+
first = output[index]
|
|
1260
|
+
second = output[index + 1]
|
|
1261
|
+
if not _overlapping_lane_pair_is_inverted(first, second, page_size):
|
|
1262
|
+
continue
|
|
1263
|
+
output[index], output[index + 1] = second, first
|
|
1264
|
+
changed = True
|
|
1265
|
+
if not changed:
|
|
1266
|
+
break
|
|
1267
|
+
return output
|
|
1268
|
+
|
|
1269
|
+
|
|
1270
|
+
def _overlapping_lane_pair_is_inverted(
|
|
1271
|
+
first: dict[str, Any],
|
|
1272
|
+
second: dict[str, Any],
|
|
1273
|
+
page_size: tuple[float, float],
|
|
1274
|
+
) -> bool:
|
|
1275
|
+
"""判断相邻块是否属于同一内部栏带且视觉中心顺序与当前结果相反。"""
|
|
1276
|
+
|
|
1277
|
+
if first.get("_visual_annotation_region_member") or second.get("_visual_annotation_region_member"):
|
|
1278
|
+
return False
|
|
1279
|
+
first_interval = first.get("_lane_interval")
|
|
1280
|
+
second_interval = second.get("_lane_interval")
|
|
1281
|
+
if (
|
|
1282
|
+
not isinstance(first_interval, (list, tuple))
|
|
1283
|
+
or not isinstance(second_interval, (list, tuple))
|
|
1284
|
+
or len(first_interval) != 2
|
|
1285
|
+
or len(second_interval) != 2
|
|
1286
|
+
or first.get("_lane_is_span") != second.get("_lane_is_span")
|
|
1287
|
+
or int(first.get("angle", 0) or 0) % 360 != int(second.get("angle", 0) or 0) % 360
|
|
1288
|
+
):
|
|
1289
|
+
return False
|
|
1290
|
+
first_bbox = _rotate_bbox_to_upright(
|
|
1291
|
+
first["bbox"],
|
|
1292
|
+
page_size,
|
|
1293
|
+
int(first.get("angle", 0) or 0) % 360,
|
|
1294
|
+
)
|
|
1295
|
+
second_bbox = _rotate_bbox_to_upright(
|
|
1296
|
+
second["bbox"],
|
|
1297
|
+
page_size,
|
|
1298
|
+
int(second.get("angle", 0) or 0) % 360,
|
|
1299
|
+
)
|
|
1300
|
+
line_heights = [
|
|
1301
|
+
float(height)
|
|
1302
|
+
for block in (first, second)
|
|
1303
|
+
for height in block.get("_line_heights", [])
|
|
1304
|
+
if isinstance(height, (int, float)) and height > 0
|
|
1305
|
+
]
|
|
1306
|
+
tolerance = 0.75 * (min(line_heights) if line_heights else 1.0)
|
|
1307
|
+
same_lane = (
|
|
1308
|
+
abs(float(first_interval[0]) - float(second_interval[0])) <= tolerance
|
|
1309
|
+
and abs(float(first_interval[1]) - float(second_interval[1])) <= tolerance
|
|
1310
|
+
)
|
|
1311
|
+
vertical_overlap = min(first_bbox[3], second_bbox[3]) - max(
|
|
1312
|
+
first_bbox[1],
|
|
1313
|
+
second_bbox[1],
|
|
1314
|
+
)
|
|
1315
|
+
return (
|
|
1316
|
+
same_lane
|
|
1317
|
+
and _bbox_center_y(first_bbox) > _bbox_center_y(second_bbox) + 0.1 * tolerance
|
|
1318
|
+
and _bbox_axis_overlap_ratio(first_bbox, second_bbox, axis="x") >= 0.35
|
|
1319
|
+
and vertical_overlap >= 0.0
|
|
1320
|
+
)
|
|
1321
|
+
|
|
1322
|
+
|
|
1323
|
+
def _normalize_output_block(
|
|
1324
|
+
block: dict[str, Any],
|
|
1325
|
+
page_size: tuple[float, float],
|
|
1326
|
+
) -> dict[str, Any] | None:
|
|
1327
|
+
"""在排序完成后将绝对 bbox 裁剪并归一化为 model_list 坐标。"""
|
|
1328
|
+
|
|
1329
|
+
page_width, page_height = page_size
|
|
1330
|
+
bbox = _clip_bbox(_coerce_bbox(block.get("bbox")), page_size)
|
|
1331
|
+
if bbox is None or page_width <= 0 or page_height <= 0:
|
|
1332
|
+
return None
|
|
1333
|
+
content = block.get("content")
|
|
1334
|
+
if not isinstance(content, str):
|
|
1335
|
+
return None
|
|
1336
|
+
content = _sanitize_pdf_control_text(content, preserve_newlines=True)
|
|
1337
|
+
block_type = block.get("type")
|
|
1338
|
+
normalized_type = block_type if block_type in _OUTPUT_BLOCK_TYPES else "text"
|
|
1339
|
+
if normalized_type not in {"image", "equation", "header", "footer"} and not content.strip():
|
|
1340
|
+
return None
|
|
1341
|
+
normalized_bbox = _normalize_bbox_to_unit(bbox, page_size)
|
|
1342
|
+
output_block: RawBlock = {
|
|
1343
|
+
"type": normalized_type,
|
|
1344
|
+
"bbox": normalized_bbox,
|
|
1345
|
+
"angle": 0 if normalized_type == "image" else int(block.get("angle", 0) or 0) % 360,
|
|
1346
|
+
"content": content,
|
|
1347
|
+
}
|
|
1348
|
+
inline_math_regions = []
|
|
1349
|
+
for value in block.get("_inline_math_regions", []):
|
|
1350
|
+
raw_region = _coerce_bbox(value)
|
|
1351
|
+
if raw_region is None:
|
|
1352
|
+
continue
|
|
1353
|
+
region = _clip_bbox(raw_region, page_size)
|
|
1354
|
+
if region is not None:
|
|
1355
|
+
inline_math_regions.append(_normalize_bbox_to_unit(region, page_size))
|
|
1356
|
+
if inline_math_regions:
|
|
1357
|
+
output_block["_inline_math_regions"] = inline_math_regions
|
|
1358
|
+
if normalized_type in _LINE_METADATA_OUTPUT_TYPES:
|
|
1359
|
+
output_block["lines"] = _normalize_output_line_items(block, page_size)
|
|
1360
|
+
return output_block
|
|
1361
|
+
|
|
1362
|
+
|
|
1363
|
+
def _normalize_output_line_items(
|
|
1364
|
+
block: dict[str, Any],
|
|
1365
|
+
page_size: tuple[float, float],
|
|
1366
|
+
) -> list[dict[str, list[float]]]:
|
|
1367
|
+
"""将 Flash 正向局部行框逆变换为页面坐标并归一化输出。"""
|
|
1368
|
+
|
|
1369
|
+
local_line_bboxes = block.get("_local_line_bboxes")
|
|
1370
|
+
if not isinstance(local_line_bboxes, list):
|
|
1371
|
+
return []
|
|
1372
|
+
|
|
1373
|
+
angle = int(block.get("angle", 0) or 0) % 360
|
|
1374
|
+
line_items: list[dict[str, list[float]]] = []
|
|
1375
|
+
for local_line_bbox in local_line_bboxes:
|
|
1376
|
+
try:
|
|
1377
|
+
raw_bbox = tuple(float(value) for value in local_line_bbox)
|
|
1378
|
+
except (TypeError, ValueError):
|
|
1379
|
+
return []
|
|
1380
|
+
coerced_bbox = _coerce_bbox(raw_bbox)
|
|
1381
|
+
if len(raw_bbox) != 4 or coerced_bbox is None or raw_bbox[2] <= raw_bbox[0] or raw_bbox[3] <= raw_bbox[1]:
|
|
1382
|
+
return []
|
|
1383
|
+
page_bbox = _clip_bbox(
|
|
1384
|
+
_rotate_bbox_from_upright(coerced_bbox, page_size, angle),
|
|
1385
|
+
page_size,
|
|
1386
|
+
)
|
|
1387
|
+
if page_bbox is None:
|
|
1388
|
+
return []
|
|
1389
|
+
line_items.append({"bbox": _normalize_bbox_to_unit(page_bbox, page_size)})
|
|
1390
|
+
return line_items
|