docvortex 0.2.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- docvortex/__init__.py +21 -0
- docvortex/analyzers/__init__.py +3 -0
- docvortex/analyzers/native/__init__.py +37 -0
- docvortex/analyzers/native/_shared/__init__.py +3 -0
- docvortex/analyzers/native/_shared/hyperlink.py +16 -0
- docvortex/analyzers/native/_shared/image.py +8 -0
- docvortex/analyzers/native/_shared/markup/__init__.py +47 -0
- docvortex/analyzers/native/_shared/markup/anchors.py +30 -0
- docvortex/analyzers/native/_shared/markup/formula.py +37 -0
- docvortex/analyzers/native/_shared/markup/projector.py +57 -0
- docvortex/analyzers/native/_shared/markup/styles.py +19 -0
- docvortex/analyzers/native/_shared/mathml.py +17 -0
- docvortex/analyzers/native/_shared/names.py +7 -0
- docvortex/analyzers/native/_shared/xycut.py +414 -0
- docvortex/analyzers/native/contracts.py +44 -0
- docvortex/analyzers/native/csv.py +351 -0
- docvortex/analyzers/native/epub/__init__.py +18 -0
- docvortex/analyzers/native/epub/constants.py +45 -0
- docvortex/analyzers/native/epub/converter.py +80 -0
- docvortex/analyzers/native/epub/errors.py +20 -0
- docvortex/analyzers/native/epub/metadata.py +31 -0
- docvortex/analyzers/native/epub/package.py +649 -0
- docvortex/analyzers/native/epub/xhtml.py +306 -0
- docvortex/analyzers/native/html/__init__.py +6 -0
- docvortex/analyzers/native/html/anchors.py +271 -0
- docvortex/analyzers/native/html/constants.py +25 -0
- docvortex/analyzers/native/html/contracts.py +7 -0
- docvortex/analyzers/native/html/converter.py +117 -0
- docvortex/analyzers/native/html/document.py +389 -0
- docvortex/analyzers/native/html/errors.py +12 -0
- docvortex/analyzers/native/html/resources.py +365 -0
- docvortex/analyzers/native/html/selector.py +421 -0
- docvortex/analyzers/native/models.py +209 -0
- docvortex/analyzers/native/ofd/__init__.py +14 -0
- docvortex/analyzers/native/ofd/constants.py +59 -0
- docvortex/analyzers/native/ofd/converter.py +31 -0
- docvortex/analyzers/native/ofd/errors.py +16 -0
- docvortex/analyzers/native/ofd/geometry.py +215 -0
- docvortex/analyzers/native/ofd/images.py +124 -0
- docvortex/analyzers/native/ofd/metadata.py +53 -0
- docvortex/analyzers/native/ofd/models.py +170 -0
- docvortex/analyzers/native/ofd/package.py +346 -0
- docvortex/analyzers/native/ofd/path.py +201 -0
- docvortex/analyzers/native/ofd/reading_order.py +293 -0
- docvortex/analyzers/native/ofd/resources.py +122 -0
- docvortex/analyzers/native/ofd/scene.py +347 -0
- docvortex/analyzers/native/ofd/table.py +240 -0
- docvortex/analyzers/native/ofd/text.py +521 -0
- docvortex/analyzers/native/office/__init__.py +3 -0
- docvortex/analyzers/native/office/doc/__init__.py +3 -0
- docvortex/analyzers/native/office/doc/bookmarks.py +81 -0
- docvortex/analyzers/native/office/doc/doc_converter.py +573 -0
- docvortex/analyzers/native/office/doc/fib.py +261 -0
- docvortex/analyzers/native/office/doc/fields.py +101 -0
- docvortex/analyzers/native/office/doc/formatting.py +171 -0
- docvortex/analyzers/native/office/doc/images.py +162 -0
- docvortex/analyzers/native/office/doc/lists.py +356 -0
- docvortex/analyzers/native/office/doc/models.py +164 -0
- docvortex/analyzers/native/office/doc/parser.py +851 -0
- docvortex/analyzers/native/office/doc/pieces.py +256 -0
- docvortex/analyzers/native/office/doc/records.py +40 -0
- docvortex/analyzers/native/office/doc/sprm.py +269 -0
- docvortex/analyzers/native/office/doc/styles.py +215 -0
- docvortex/analyzers/native/office/docx/__init__.py +3 -0
- docvortex/analyzers/native/office/docx/context.py +71 -0
- docvortex/analyzers/native/office/docx/docx_converter.py +614 -0
- docvortex/analyzers/native/office/docx/equationxml.py +119 -0
- docvortex/analyzers/native/office/docx/fields.py +816 -0
- docvortex/analyzers/native/office/docx/formatting_types.py +23 -0
- docvortex/analyzers/native/office/docx/main.py +50 -0
- docvortex/analyzers/native/office/docx/numbering.py +491 -0
- docvortex/analyzers/native/office/docx/office_xml.py +57 -0
- docvortex/analyzers/native/office/docx/package_normalizer.py +248 -0
- docvortex/analyzers/native/office/docx/resources.py +526 -0
- docvortex/analyzers/native/office/docx/styles.py +693 -0
- docvortex/analyzers/native/office/docx/tables.py +515 -0
- docvortex/analyzers/native/office/equation/__init__.py +3 -0
- docvortex/analyzers/native/office/equation/image.py +470 -0
- docvortex/analyzers/native/office/equation/latex_dict.py +324 -0
- docvortex/analyzers/native/office/equation/mtef.py +885 -0
- docvortex/analyzers/native/office/equation/mtef_v5.py +941 -0
- docvortex/analyzers/native/office/equation/omml.py +561 -0
- docvortex/analyzers/native/office/equation/ooxml.py +62 -0
- docvortex/analyzers/native/office/errors.py +33 -0
- docvortex/analyzers/native/office/image.py +307 -0
- docvortex/analyzers/native/office/legacy/__init__.py +3 -0
- docvortex/analyzers/native/office/legacy/binary.py +43 -0
- docvortex/analyzers/native/office/legacy/officeart.py +362 -0
- docvortex/analyzers/native/office/legacy/ole.py +110 -0
- docvortex/analyzers/native/office/limits.py +12 -0
- docvortex/analyzers/native/office/odf/__init__.py +3 -0
- docvortex/analyzers/native/office/odf/chart.py +76 -0
- docvortex/analyzers/native/office/odf/constants.py +70 -0
- docvortex/analyzers/native/office/odf/converters.py +403 -0
- docvortex/analyzers/native/office/odf/errors.py +18 -0
- docvortex/analyzers/native/office/odf/metadata.py +91 -0
- docvortex/analyzers/native/office/odf/models.py +176 -0
- docvortex/analyzers/native/office/odf/package.py +270 -0
- docvortex/analyzers/native/office/odf/styles.py +329 -0
- docvortex/analyzers/native/office/odf/table.py +469 -0
- docvortex/analyzers/native/office/odf/text.py +1002 -0
- docvortex/analyzers/native/office/ooxml_chart.py +1016 -0
- docvortex/analyzers/native/office/opc.py +38 -0
- docvortex/analyzers/native/office/ppt/__init__.py +3 -0
- docvortex/analyzers/native/office/ppt/models.py +118 -0
- docvortex/analyzers/native/office/ppt/parser.py +1895 -0
- docvortex/analyzers/native/office/ppt/ppt_converter.py +292 -0
- docvortex/analyzers/native/office/ppt/records.py +131 -0
- docvortex/analyzers/native/office/ppt/style_text.py +247 -0
- docvortex/analyzers/native/office/pptx/__init__.py +3 -0
- docvortex/analyzers/native/office/pptx/context.py +113 -0
- docvortex/analyzers/native/office/pptx/lists.py +558 -0
- docvortex/analyzers/native/office/pptx/main.py +20 -0
- docvortex/analyzers/native/office/pptx/package_normalizer.py +321 -0
- docvortex/analyzers/native/office/pptx/pptx_converter.py +323 -0
- docvortex/analyzers/native/office/pptx/resources.py +329 -0
- docvortex/analyzers/native/office/pptx/shapes.py +393 -0
- docvortex/analyzers/native/office/pptx/text_styles.py +625 -0
- docvortex/analyzers/native/office/pptx/titles.py +178 -0
- docvortex/analyzers/native/office/rich_text.py +420 -0
- docvortex/analyzers/native/office/rtf/__init__.py +3 -0
- docvortex/analyzers/native/office/rtf/converter.py +708 -0
- docvortex/analyzers/native/office/rtf/lexer.py +213 -0
- docvortex/analyzers/native/office/rtf/math.py +339 -0
- docvortex/analyzers/native/office/rtf/models.py +185 -0
- docvortex/analyzers/native/office/rtf/parser.py +1553 -0
- docvortex/analyzers/native/office/spreadsheet/__init__.py +3 -0
- docvortex/analyzers/native/office/spreadsheet/html.py +81 -0
- docvortex/analyzers/native/office/spreadsheet/models.py +79 -0
- docvortex/analyzers/native/office/spreadsheet/projector.py +928 -0
- docvortex/analyzers/native/office/streams.py +18 -0
- docvortex/analyzers/native/office/xls/__init__.py +3 -0
- docvortex/analyzers/native/office/xls/chart.py +132 -0
- docvortex/analyzers/native/office/xls/embedded_chart.py +299 -0
- docvortex/analyzers/native/office/xls/models.py +109 -0
- docvortex/analyzers/native/office/xls/number_format.py +521 -0
- docvortex/analyzers/native/office/xls/parser.py +1145 -0
- docvortex/analyzers/native/office/xls/records.py +201 -0
- docvortex/analyzers/native/office/xls/strings.py +205 -0
- docvortex/analyzers/native/office/xls/xls_converter.py +354 -0
- docvortex/analyzers/native/office/xlsx/__init__.py +3 -0
- docvortex/analyzers/native/office/xlsx/main.py +20 -0
- docvortex/analyzers/native/office/xlsx/ooxml_ole.py +522 -0
- docvortex/analyzers/native/office/xlsx/package_normalizer.py +310 -0
- docvortex/analyzers/native/office/xlsx/xlsx_converter.py +716 -0
- docvortex/analyzers/native/pdf/__init__.py +3 -0
- docvortex/analyzers/native/pdf/auxiliary_text.py +1671 -0
- docvortex/analyzers/native/pdf/char_geometry.py +1662 -0
- docvortex/analyzers/native/pdf/code_blocks.py +535 -0
- docvortex/analyzers/native/pdf/formulas.py +1985 -0
- docvortex/analyzers/native/pdf/geometry.py +281 -0
- docvortex/analyzers/native/pdf/graphics.py +1501 -0
- docvortex/analyzers/native/pdf/index_blocks.py +268 -0
- docvortex/analyzers/native/pdf/inline/__init__.py +3 -0
- docvortex/analyzers/native/pdf/inline/common.py +112 -0
- docvortex/analyzers/native/pdf/inline/detection.py +590 -0
- docvortex/analyzers/native/pdf/inline/matching.py +1185 -0
- docvortex/analyzers/native/pdf/inline/materialize.py +457 -0
- docvortex/analyzers/native/pdf/inline/scripts.py +975 -0
- docvortex/analyzers/native/pdf/inline/types.py +385 -0
- docvortex/analyzers/native/pdf/line_layout.py +1106 -0
- docvortex/analyzers/native/pdf/line_merging.py +1223 -0
- docvortex/analyzers/native/pdf/models.py +246 -0
- docvortex/analyzers/native/pdf/native_text.py +1004 -0
- docvortex/analyzers/native/pdf/pipeline.py +1390 -0
- docvortex/analyzers/native/pdf/script_geometry.py +636 -0
- docvortex/analyzers/native/pdf/shared.py +30 -0
- docvortex/analyzers/native/pdf/spatial_text.py +383 -0
- docvortex/analyzers/native/pdf/table_annotations.py +446 -0
- docvortex/analyzers/native/pdf/table_constants.py +48 -0
- docvortex/analyzers/native/pdf/table_detection.py +227 -0
- docvortex/analyzers/native/pdf/table_filled_grid.py +218 -0
- docvortex/analyzers/native/pdf/table_geometry.py +40 -0
- docvortex/analyzers/native/pdf/table_materialization.py +444 -0
- docvortex/analyzers/native/pdf/table_recovery/__init__.py +15 -0
- docvortex/analyzers/native/pdf/table_recovery/candidate.py +356 -0
- docvortex/analyzers/native/pdf/table_recovery/contracts.py +154 -0
- docvortex/analyzers/native/pdf/table_recovery/engine.py +609 -0
- docvortex/analyzers/native/pdf/table_recovery/geometry.py +164 -0
- docvortex/analyzers/native/pdf/table_recovery/sparse_common.py +85 -0
- docvortex/analyzers/native/pdf/table_recovery/sparse_hybrid.py +804 -0
- docvortex/analyzers/native/pdf/table_recovery/sparse_multiline.py +1122 -0
- docvortex/analyzers/native/pdf/table_recovery/text.py +414 -0
- docvortex/analyzers/native/pdf/table_recovery/text_grid.py +618 -0
- docvortex/analyzers/native/pdf/table_recovery/vector.py +1931 -0
- docvortex/analyzers/native/pdf/table_rows.py +34 -0
- docvortex/analyzers/native/pdf/table_rules.py +1129 -0
- docvortex/analyzers/native/pdf/table_text_styles.py +283 -0
- docvortex/analyzers/native/pdf/tables.py +147 -0
- docvortex/analyzers/native/pdf/text_assembly/__init__.py +3 -0
- docvortex/analyzers/native/pdf/text_assembly/annotations.py +581 -0
- docvortex/analyzers/native/pdf/text_assembly/assembly.py +292 -0
- docvortex/analyzers/native/pdf/text_assembly/common.py +477 -0
- docvortex/analyzers/native/pdf/text_assembly/footnotes.py +394 -0
- docvortex/analyzers/native/pdf/text_assembly/merging.py +1274 -0
- docvortex/analyzers/native/pdf/text_assembly/rows.py +692 -0
- docvortex/analyzers/native/pdf/text_blocks.py +82 -0
- docvortex/analyzers/native/pdf/text_styles.py +55 -0
- docvortex/analyzers/native/pdf/title_analysis/__init__.py +3 -0
- docvortex/analyzers/native/pdf/title_analysis/body_profile.py +215 -0
- docvortex/analyzers/native/pdf/title_analysis/common.py +117 -0
- docvortex/analyzers/native/pdf/title_analysis/document_profile.py +164 -0
- docvortex/analyzers/native/pdf/title_analysis/lane_titles.py +758 -0
- docvortex/analyzers/native/pdf/title_analysis/page_titles.py +1024 -0
- docvortex/analyzers/native/pdf/title_analysis/prototype.py +194 -0
- docvortex/analyzers/native/pdf/title_analysis/structural.py +1081 -0
- docvortex/analyzers/native/pdf/titles.py +75 -0
- docvortex/analyzers/native/pdf/typography.py +19 -0
- docvortex/analyzers/native/pdf/visual_annotations.py +1262 -0
- docvortex/api.py +180 -0
- docvortex/assets/__init__.py +5 -0
- docvortex/assets/store.py +51 -0
- docvortex/cli.py +54 -0
- docvortex/codecs/__init__.py +3 -0
- docvortex/codecs/html/__init__.py +22 -0
- docvortex/codecs/html/contracts.py +236 -0
- docvortex/codecs/html/materializer.py +331 -0
- docvortex/codecs/html/parser.py +763 -0
- docvortex/codecs/html/resources.py +34 -0
- docvortex/codecs/json.py +17 -0
- docvortex/content/__init__.py +5 -0
- docvortex/content/inline.py +248 -0
- docvortex/content/markup/__init__.py +44 -0
- docvortex/content/markup/anchors.py +188 -0
- docvortex/content/markup/formula.py +280 -0
- docvortex/content/markup/projector.py +1237 -0
- docvortex/content/markup/styles.py +327 -0
- docvortex/content/mathml.py +167 -0
- docvortex/content/normalization.py +188 -0
- docvortex/content/spans.py +183 -0
- docvortex/content/table/__init__.py +18 -0
- docvortex/content/table/blocks.py +152 -0
- docvortex/content/table/content.py +425 -0
- docvortex/content/table/document.py +104 -0
- docvortex/content/table/html.py +399 -0
- docvortex/content/table/models.py +76 -0
- docvortex/content/table/rules.py +42 -0
- docvortex/content/table/structure.py +221 -0
- docvortex/content/tree.py +5 -0
- docvortex/document/__init__.py +3 -0
- docvortex/document/contracts.py +22 -0
- docvortex/document/detection.py +389 -0
- docvortex/document/filetypes.py +175 -0
- docvortex/document/page_range.py +167 -0
- docvortex/document/pdf/__init__.py +19 -0
- docvortex/document/pdf/classify.py +1138 -0
- docvortex/document/pdf/constants.py +132 -0
- docvortex/document/pdf/diagnostics.py +350 -0
- docvortex/document/pdf/document.py +582 -0
- docvortex/document/pdf/font_runtime.py +335 -0
- docvortex/document/pdf/geometry.py +31 -0
- docvortex/document/pdf/images.py +654 -0
- docvortex/document/pdf/native_annotations.py +367 -0
- docvortex/document/pdf/native_contracts.py +169 -0
- docvortex/document/pdf/native_coordinates.py +216 -0
- docvortex/document/pdf/native_lifecycle.py +16 -0
- docvortex/document/pdf/native_objects.py +902 -0
- docvortex/document/pdf/native_text_geometry.py +315 -0
- docvortex/document/pdf/pdfium.py +325 -0
- docvortex/document/pdf/raster.py +46 -0
- docvortex/document/pdf/text/__init__.py +62 -0
- docvortex/document/pdf/text/contracts.py +211 -0
- docvortex/document/pdf/text/extract.py +165 -0
- docvortex/document/pdf/text/geometry.py +16 -0
- docvortex/document/pdf/text/groups.py +162 -0
- docvortex/document/pdf/visual_geometry.py +201 -0
- docvortex/document/pdf/visuals.py +343 -0
- docvortex/document/source.py +92 -0
- docvortex/errors.py +21 -0
- docvortex/export/__init__.py +3 -0
- docvortex/export/bundle.py +98 -0
- docvortex/export/files.py +66 -0
- docvortex/export/middle.py +208 -0
- docvortex/foundation/__init__.py +3 -0
- docvortex/foundation/geometry.py +125 -0
- docvortex/foundation/hyperlink.py +65 -0
- docvortex/foundation/image.py +48 -0
- docvortex/foundation/image_encoding.py +30 -0
- docvortex/foundation/image_payload.py +280 -0
- docvortex/foundation/language.py +92 -0
- docvortex/foundation/platform.py +38 -0
- docvortex/foundation/text.py +153 -0
- docvortex/foundation/type_identity.py +20 -0
- docvortex/foundation/xml_names.py +20 -0
- docvortex/options.py +30 -0
- docvortex/postprocess/__init__.py +3 -0
- docvortex/postprocess/content.py +53 -0
- docvortex/postprocess/document.py +19 -0
- docvortex/postprocess/lists.py +236 -0
- docvortex/postprocess/page_blocks.py +214 -0
- docvortex/postprocess/pages.py +95 -0
- docvortex/postprocess/paragraphs.py +580 -0
- docvortex/postprocess/visual.py +715 -0
- docvortex/render/__init__.py +48 -0
- docvortex/render/_internal/__init__.py +3 -0
- docvortex/render/_internal/common/__init__.py +3 -0
- docvortex/render/_internal/common/context.py +43 -0
- docvortex/render/_internal/common/html_table.py +178 -0
- docvortex/render/_internal/common/index.py +33 -0
- docvortex/render/_internal/common/list_items.py +158 -0
- docvortex/render/_internal/common/planner.py +140 -0
- docvortex/render/_internal/docx/__init__.py +3 -0
- docvortex/render/_internal/docx/assets.py +202 -0
- docvortex/render/_internal/docx/inline.py +434 -0
- docvortex/render/_internal/docx/math.py +220 -0
- docvortex/render/_internal/docx/renderer.py +905 -0
- docvortex/render/_internal/docx/styles.py +195 -0
- docvortex/render/_internal/docx/table.py +442 -0
- docvortex/render/_internal/epub/__init__.py +5 -0
- docvortex/render/_internal/epub/assets.py +173 -0
- docvortex/render/_internal/epub/package.py +249 -0
- docvortex/render/_internal/epub/renderer.py +1156 -0
- docvortex/render/_internal/html/__init__.py +3 -0
- docvortex/render/_internal/html/inline.py +346 -0
- docvortex/render/_internal/html/renderer.py +1041 -0
- docvortex/render/_internal/html/sanitizer.py +478 -0
- docvortex/render/_internal/html/table.py +121 -0
- docvortex/render/_internal/latex/__init__.py +1 -0
- docvortex/render/_internal/latex/assets.py +85 -0
- docvortex/render/_internal/latex/inline.py +145 -0
- docvortex/render/_internal/latex/renderer.py +506 -0
- docvortex/render/_internal/latex/table.py +347 -0
- docvortex/render/_internal/markdown/__init__.py +3 -0
- docvortex/render/_internal/markdown/assets.py +78 -0
- docvortex/render/_internal/markdown/blocks.py +635 -0
- docvortex/render/_internal/markdown/escaping.py +51 -0
- docvortex/render/_internal/markdown/inline.py +260 -0
- docvortex/render/_internal/markdown/renderer.py +93 -0
- docvortex/render/_internal/markdown/table.py +281 -0
- docvortex/render/_internal/pdf/__init__.py +3 -0
- docvortex/render/_internal/pdf/assets.py +197 -0
- docvortex/render/_internal/pdf/formula.py +417 -0
- docvortex/render/_internal/pdf/inline.py +343 -0
- docvortex/render/_internal/pdf/renderer.py +734 -0
- docvortex/render/_internal/pdf/styles.py +206 -0
- docvortex/render/_internal/pdf/table.py +272 -0
- docvortex/render/_internal/structured_content/__init__.py +3 -0
- docvortex/render/_internal/structured_content/renderer.py +193 -0
- docvortex/render/api.py +199 -0
- docvortex/render/contracts.py +205 -0
- docvortex/render/docx.py +38 -0
- docvortex/render/epub.py +35 -0
- docvortex/render/fragments.py +70 -0
- docvortex/render/html.py +29 -0
- docvortex/render/latex.py +24 -0
- docvortex/render/markdown.py +50 -0
- docvortex/render/pdf.py +25 -0
- docvortex/render/structured_content.py +23 -0
- docvortex/resources/epub/docvortex.css +91 -0
- docvortex/resources/fasttext-langdetect/lid.176.ftz +0 -0
- docvortex/resources/fonts/DroidSansFallbackFull.ttf +0 -0
- docvortex/resources/fonts/NOTICE +190 -0
- docvortex/resources/fonts/manifest.json +9 -0
- docvortex/resources/html/docvortex.css +601 -0
- docvortex/resources/html/docvortex.min.css +1 -0
- docvortex/result.py +91 -0
- docvortex/schema.py +1141 -0
- docvortex/version.py +3 -0
- docvortex-0.2.1.dist-info/METADATA +193 -0
- docvortex-0.2.1.dist-info/RECORD +364 -0
- docvortex-0.2.1.dist-info/WHEEL +5 -0
- docvortex-0.2.1.dist-info/entry_points.txt +2 -0
- docvortex-0.2.1.dist-info/licenses/LICENSE.md +21 -0
- docvortex-0.2.1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,1185 @@
|
|
|
1
|
+
"""对齐原生行证据与输出块文本,保留来源和偏移。"""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import html
|
|
6
|
+
from typing import TYPE_CHECKING, Any, Sequence
|
|
7
|
+
|
|
8
|
+
from loguru import logger
|
|
9
|
+
|
|
10
|
+
from .....schema import BBox
|
|
11
|
+
from .....foundation.text import is_hyphen_at_line_end
|
|
12
|
+
|
|
13
|
+
if TYPE_CHECKING:
|
|
14
|
+
from ..models import _LineItem
|
|
15
|
+
from ..native_text import _NativeVisualResplit
|
|
16
|
+
from .common import _bbox_overlap_ratio, _canonical_styles, _coerce_bbox, _normalize_match_fragment, _ordered_line_chars
|
|
17
|
+
from .types import (
|
|
18
|
+
_PDF_GEOMETRIC_TEXT_STYLES,
|
|
19
|
+
_PDF_TEXT_STYLE_TARGET_BLOCK_TYPES,
|
|
20
|
+
PDF_NATURAL_TEXT_STYLE_BLOCK_TYPES,
|
|
21
|
+
PDFTextEvidenceLine,
|
|
22
|
+
PDFTextLinkLine,
|
|
23
|
+
PDFTextLinkRange,
|
|
24
|
+
PDFTextScriptLine,
|
|
25
|
+
PDFTextScriptRange,
|
|
26
|
+
PDFTextStyle,
|
|
27
|
+
PDFTextStyleLine,
|
|
28
|
+
PDFTextStyleRange,
|
|
29
|
+
_LineProjectionMatch,
|
|
30
|
+
_MatchedLinkRange,
|
|
31
|
+
_ProjectedChar,
|
|
32
|
+
_RawLinkInterval,
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _resplit_evidence_segments(
|
|
37
|
+
evidence_text: str,
|
|
38
|
+
resplit: _NativeVisualResplit,
|
|
39
|
+
) -> list[tuple[_LineItem, int, int, str]] | None:
|
|
40
|
+
"""按原字符身份把紧凑 evidence 文本映射到每个重切成员区间。"""
|
|
41
|
+
|
|
42
|
+
source_spans: list[tuple[int, int]] = []
|
|
43
|
+
spans_by_object_id: dict[int, tuple[int, int]] = {}
|
|
44
|
+
spans_by_char_idx: dict[int, list[tuple[int, int]]] = {}
|
|
45
|
+
source_parts: list[str] = []
|
|
46
|
+
cursor = 0
|
|
47
|
+
for char in _ordered_line_chars(resplit.source):
|
|
48
|
+
fragment = _normalize_match_fragment(char.get("char"))
|
|
49
|
+
if not fragment:
|
|
50
|
+
continue
|
|
51
|
+
span = (cursor, cursor + len(fragment))
|
|
52
|
+
source_spans.append(span)
|
|
53
|
+
spans_by_object_id[id(char)] = span
|
|
54
|
+
char_idx = char.get("char_idx")
|
|
55
|
+
if isinstance(char_idx, int) and not isinstance(char_idx, bool):
|
|
56
|
+
spans_by_char_idx.setdefault(char_idx, []).append(span)
|
|
57
|
+
source_parts.append(fragment)
|
|
58
|
+
cursor = span[1]
|
|
59
|
+
if "".join(source_parts) != evidence_text:
|
|
60
|
+
return None
|
|
61
|
+
|
|
62
|
+
used_spans: list[tuple[int, int]] = []
|
|
63
|
+
segments: list[tuple[_LineItem, int, int, str]] = []
|
|
64
|
+
for member in sorted(
|
|
65
|
+
resplit.members,
|
|
66
|
+
key=lambda item: (item.run_index, item.source_index),
|
|
67
|
+
):
|
|
68
|
+
member_spans: list[tuple[int, int]] = []
|
|
69
|
+
member_parts: list[str] = []
|
|
70
|
+
for char in _ordered_line_chars(member):
|
|
71
|
+
fragment = _normalize_match_fragment(char.get("char"))
|
|
72
|
+
if not fragment:
|
|
73
|
+
continue
|
|
74
|
+
span = spans_by_object_id.get(id(char))
|
|
75
|
+
if span is None:
|
|
76
|
+
char_idx = char.get("char_idx")
|
|
77
|
+
candidates = (
|
|
78
|
+
spans_by_char_idx.get(char_idx, []) if isinstance(char_idx, int) and not isinstance(char_idx, bool) else []
|
|
79
|
+
)
|
|
80
|
+
span = candidates[0] if len(candidates) == 1 else None
|
|
81
|
+
if span is None or evidence_text[span[0] : span[1]] != fragment:
|
|
82
|
+
return None
|
|
83
|
+
member_spans.append(span)
|
|
84
|
+
member_parts.append(fragment)
|
|
85
|
+
if not member_spans:
|
|
86
|
+
return None
|
|
87
|
+
member_start = min(start for start, _end in member_spans)
|
|
88
|
+
member_end = max(end for _start, end in member_spans)
|
|
89
|
+
member_text = "".join(member_parts)
|
|
90
|
+
if (
|
|
91
|
+
sum(end - start for start, end in member_spans) != member_end - member_start
|
|
92
|
+
or evidence_text[member_start:member_end] != member_text
|
|
93
|
+
):
|
|
94
|
+
return None
|
|
95
|
+
used_spans.extend(member_spans)
|
|
96
|
+
segments.append(
|
|
97
|
+
(member, member_start, member_end, member_text),
|
|
98
|
+
)
|
|
99
|
+
if sorted(used_spans) != source_spans:
|
|
100
|
+
return None
|
|
101
|
+
return segments
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _partition_resplit_text_evidence(
|
|
105
|
+
style_lines: list[PDFTextStyleLine],
|
|
106
|
+
link_lines: list[PDFTextLinkLine],
|
|
107
|
+
resplits: dict[int, _NativeVisualResplit],
|
|
108
|
+
) -> tuple[list[PDFTextStyleLine], list[PDFTextLinkLine]]:
|
|
109
|
+
"""只替换被重切粗行的样式与链接 evidence,其它行保持原对象和顺序。"""
|
|
110
|
+
|
|
111
|
+
if not resplits:
|
|
112
|
+
return style_lines, link_lines
|
|
113
|
+
|
|
114
|
+
partitioned_styles: list[PDFTextStyleLine] = []
|
|
115
|
+
for line in style_lines:
|
|
116
|
+
resplit = resplits.get(line.source_index)
|
|
117
|
+
if resplit is None:
|
|
118
|
+
partitioned_styles.append(line)
|
|
119
|
+
continue
|
|
120
|
+
segments = _resplit_evidence_segments(line.text, resplit)
|
|
121
|
+
if segments is None:
|
|
122
|
+
logger.warning(
|
|
123
|
+
"Keep coarse PDF style evidence after an unsafe resplit mapping: "
|
|
124
|
+
f"source_index={line.source_index}, text={line.text!r}"
|
|
125
|
+
)
|
|
126
|
+
partitioned_styles.append(line)
|
|
127
|
+
continue
|
|
128
|
+
for member, member_start, member_end, member_text in segments:
|
|
129
|
+
style_ranges = tuple(
|
|
130
|
+
PDFTextStyleRange(
|
|
131
|
+
start=max(member_start, style_range.start) - member_start,
|
|
132
|
+
end=min(member_end, style_range.end) - member_start,
|
|
133
|
+
styles=style_range.styles,
|
|
134
|
+
)
|
|
135
|
+
for style_range in line.style_ranges
|
|
136
|
+
if max(member_start, style_range.start) < min(member_end, style_range.end)
|
|
137
|
+
)
|
|
138
|
+
partitioned_styles.append(
|
|
139
|
+
PDFTextStyleLine(
|
|
140
|
+
bbox=member.bbox,
|
|
141
|
+
text=member_text,
|
|
142
|
+
style_ranges=style_ranges,
|
|
143
|
+
source_index=member.source_index,
|
|
144
|
+
)
|
|
145
|
+
)
|
|
146
|
+
|
|
147
|
+
partitioned_links: list[PDFTextLinkLine] = []
|
|
148
|
+
for line in link_lines:
|
|
149
|
+
resplit = resplits.get(line.source_index)
|
|
150
|
+
if resplit is None:
|
|
151
|
+
partitioned_links.append(line)
|
|
152
|
+
continue
|
|
153
|
+
segments = _resplit_evidence_segments(line.text, resplit)
|
|
154
|
+
if segments is None:
|
|
155
|
+
logger.warning(
|
|
156
|
+
"Keep coarse PDF link evidence after an unsafe resplit mapping: "
|
|
157
|
+
f"source_index={line.source_index}, text={line.text!r}"
|
|
158
|
+
)
|
|
159
|
+
partitioned_links.append(line)
|
|
160
|
+
continue
|
|
161
|
+
for member, member_start, member_end, member_text in segments:
|
|
162
|
+
link_ranges = tuple(
|
|
163
|
+
PDFTextLinkRange(
|
|
164
|
+
start=max(member_start, link_range.start) - member_start,
|
|
165
|
+
end=min(member_end, link_range.end) - member_start,
|
|
166
|
+
target=link_range.target,
|
|
167
|
+
)
|
|
168
|
+
for link_range in line.link_ranges
|
|
169
|
+
if max(member_start, link_range.start) < min(member_end, link_range.end)
|
|
170
|
+
)
|
|
171
|
+
if not link_ranges:
|
|
172
|
+
continue
|
|
173
|
+
partitioned_links.append(
|
|
174
|
+
PDFTextLinkLine(
|
|
175
|
+
bbox=member.bbox,
|
|
176
|
+
text=member_text,
|
|
177
|
+
link_ranges=link_ranges,
|
|
178
|
+
source_index=member.source_index,
|
|
179
|
+
)
|
|
180
|
+
)
|
|
181
|
+
return partitioned_styles, partitioned_links
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def _realign_repaired_text_evidence(
|
|
185
|
+
style_lines: list[PDFTextStyleLine],
|
|
186
|
+
link_lines: list[PDFTextLinkLine],
|
|
187
|
+
line_bboxes: dict[int, BBox],
|
|
188
|
+
resplits: dict[int, _NativeVisualResplit],
|
|
189
|
+
) -> tuple[list[PDFTextStyleLine], list[PDFTextLinkLine]]:
|
|
190
|
+
"""同步未重切修复行的 evidence 框,再按字符身份切分发生重切的样式与链接。"""
|
|
191
|
+
|
|
192
|
+
aligned_styles = style_lines
|
|
193
|
+
for index, line in enumerate(style_lines):
|
|
194
|
+
bbox = line_bboxes.get(line.source_index)
|
|
195
|
+
if line.source_index in resplits or bbox is None or bbox == line.bbox:
|
|
196
|
+
continue
|
|
197
|
+
if aligned_styles is style_lines:
|
|
198
|
+
aligned_styles = list(style_lines)
|
|
199
|
+
aligned_styles[index] = PDFTextStyleLine(
|
|
200
|
+
bbox=bbox,
|
|
201
|
+
text=line.text,
|
|
202
|
+
style_ranges=line.style_ranges,
|
|
203
|
+
source_index=line.source_index,
|
|
204
|
+
)
|
|
205
|
+
|
|
206
|
+
aligned_links = link_lines
|
|
207
|
+
for index, line in enumerate(link_lines):
|
|
208
|
+
bbox = line_bboxes.get(line.source_index)
|
|
209
|
+
if line.source_index in resplits or bbox is None or bbox == line.bbox:
|
|
210
|
+
continue
|
|
211
|
+
if aligned_links is link_lines:
|
|
212
|
+
aligned_links = list(link_lines)
|
|
213
|
+
aligned_links[index] = PDFTextLinkLine(
|
|
214
|
+
bbox=bbox,
|
|
215
|
+
text=line.text,
|
|
216
|
+
link_ranges=line.link_ranges,
|
|
217
|
+
source_index=line.source_index,
|
|
218
|
+
)
|
|
219
|
+
|
|
220
|
+
return _partition_resplit_text_evidence(
|
|
221
|
+
aligned_styles,
|
|
222
|
+
aligned_links,
|
|
223
|
+
resplits,
|
|
224
|
+
)
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def _block_bbox_to_page_bbox(value: Any, page_size: tuple[float, float]) -> BBox | None:
|
|
228
|
+
"""把 model-list 的归一化 bbox 转回页面 point,同时兼容已是绝对坐标的内部输入。"""
|
|
229
|
+
|
|
230
|
+
bbox = _coerce_bbox(value)
|
|
231
|
+
if bbox is None:
|
|
232
|
+
return None
|
|
233
|
+
if all(0.0 <= coordinate <= 1.0 for coordinate in bbox):
|
|
234
|
+
return (
|
|
235
|
+
bbox[0] * page_size[0],
|
|
236
|
+
bbox[1] * page_size[1],
|
|
237
|
+
bbox[2] * page_size[0],
|
|
238
|
+
bbox[3] * page_size[1],
|
|
239
|
+
)
|
|
240
|
+
return bbox
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def _line_block_score(line_bbox: BBox, block_bbox: BBox) -> tuple[float, float, float]:
|
|
244
|
+
"""计算文本行归属 block 的中心包含、重叠率与紧致度评分。"""
|
|
245
|
+
|
|
246
|
+
center_x = (line_bbox[0] + line_bbox[2]) / 2
|
|
247
|
+
center_y = (line_bbox[1] + line_bbox[3]) / 2
|
|
248
|
+
center_inside = float(block_bbox[0] <= center_x <= block_bbox[2] and block_bbox[1] <= center_y <= block_bbox[3])
|
|
249
|
+
overlap_ratio = _bbox_overlap_ratio(line_bbox, block_bbox)
|
|
250
|
+
block_area = (block_bbox[2] - block_bbox[0]) * (block_bbox[3] - block_bbox[1])
|
|
251
|
+
return center_inside, overlap_ratio, -block_area
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def _assign_lines_to_blocks(
|
|
255
|
+
blocks: list[dict[str, Any]],
|
|
256
|
+
lines: Sequence[PDFTextEvidenceLine],
|
|
257
|
+
page_size: tuple[float, float],
|
|
258
|
+
) -> dict[int, list[PDFTextEvidenceLine]]:
|
|
259
|
+
"""把每个视觉文本行唯一分配给最匹配的自然语言 block。"""
|
|
260
|
+
|
|
261
|
+
target_bboxes = {
|
|
262
|
+
block_index: block_bbox
|
|
263
|
+
for block_index, block in enumerate(blocks)
|
|
264
|
+
if block.get("type") in PDF_NATURAL_TEXT_STYLE_BLOCK_TYPES
|
|
265
|
+
and isinstance(block.get("content"), str)
|
|
266
|
+
and (block_bbox := _block_bbox_to_page_bbox(block.get("bbox"), page_size)) is not None
|
|
267
|
+
}
|
|
268
|
+
assignments: dict[int, list[PDFTextEvidenceLine]] = {}
|
|
269
|
+
for line in lines:
|
|
270
|
+
matches = [
|
|
271
|
+
(block_index, _line_block_score(line.bbox, block_bbox))
|
|
272
|
+
for block_index, block_bbox in target_bboxes.items()
|
|
273
|
+
if (
|
|
274
|
+
block_bbox[0] <= (line.bbox[0] + line.bbox[2]) / 2 <= block_bbox[2]
|
|
275
|
+
and block_bbox[1] <= (line.bbox[1] + line.bbox[3]) / 2 <= block_bbox[3]
|
|
276
|
+
)
|
|
277
|
+
or _bbox_overlap_ratio(line.bbox, block_bbox) >= 0.5
|
|
278
|
+
]
|
|
279
|
+
if not matches:
|
|
280
|
+
continue
|
|
281
|
+
block_index, _score = max(matches, key=lambda item: (*item[1], -item[0]))
|
|
282
|
+
assignments.setdefault(block_index, []).append(line)
|
|
283
|
+
for block_lines in assignments.values():
|
|
284
|
+
block_lines.sort(
|
|
285
|
+
key=lambda line: (
|
|
286
|
+
line.source_index,
|
|
287
|
+
line.bbox[1],
|
|
288
|
+
line.bbox[0],
|
|
289
|
+
)
|
|
290
|
+
)
|
|
291
|
+
return assignments
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
def _assign_script_lines_to_blocks(
|
|
295
|
+
blocks: list[dict[str, Any]],
|
|
296
|
+
lines: Sequence[PDFTextScriptLine],
|
|
297
|
+
page_size: tuple[float, float],
|
|
298
|
+
) -> dict[int, list[PDFTextScriptLine]]:
|
|
299
|
+
"""保留整行主归属,并为无法投影的脚本区间补充 tight bbox 备用归属。"""
|
|
300
|
+
|
|
301
|
+
target_bboxes = {
|
|
302
|
+
block_index: block_bbox
|
|
303
|
+
for block_index, block in enumerate(blocks)
|
|
304
|
+
if block.get("type") in PDF_NATURAL_TEXT_STYLE_BLOCK_TYPES
|
|
305
|
+
and isinstance(block.get("content"), str)
|
|
306
|
+
and (block_bbox := _block_bbox_to_page_bbox(block.get("bbox"), page_size)) is not None
|
|
307
|
+
}
|
|
308
|
+
target_projected = {
|
|
309
|
+
block_index: _project_content_chars(str(blocks[block_index]["content"])) for block_index in target_bboxes
|
|
310
|
+
}
|
|
311
|
+
primary_assignments = _assign_lines_to_blocks(blocks, lines, page_size)
|
|
312
|
+
assignments: dict[int, list[PDFTextScriptLine]] = {
|
|
313
|
+
block_index: [line for line in block_lines if isinstance(line, PDFTextScriptLine)]
|
|
314
|
+
for block_index, block_lines in primary_assignments.items()
|
|
315
|
+
}
|
|
316
|
+
primary_block_by_line = {id(line): block_index for block_index, block_lines in assignments.items() for line in block_lines}
|
|
317
|
+
fallback_ranges: dict[tuple[int, int], list[PDFTextScriptRange]] = {}
|
|
318
|
+
for line_index, line in enumerate(lines):
|
|
319
|
+
for script_range in line.script_ranges:
|
|
320
|
+
evidence_line = PDFTextStyleLine(
|
|
321
|
+
bbox=line.bbox,
|
|
322
|
+
text=line.text,
|
|
323
|
+
style_ranges=(
|
|
324
|
+
PDFTextStyleRange(
|
|
325
|
+
script_range.start,
|
|
326
|
+
script_range.end,
|
|
327
|
+
(script_range.style,),
|
|
328
|
+
),
|
|
329
|
+
),
|
|
330
|
+
source_index=line.source_index,
|
|
331
|
+
)
|
|
332
|
+
primary_block_index = primary_block_by_line.get(id(line))
|
|
333
|
+
if primary_block_index is not None and _match_script_line_ranges(
|
|
334
|
+
target_projected[primary_block_index],
|
|
335
|
+
evidence_line,
|
|
336
|
+
):
|
|
337
|
+
continue
|
|
338
|
+
matches = [
|
|
339
|
+
(
|
|
340
|
+
block_index,
|
|
341
|
+
_line_block_score(script_range.bbox, block_bbox),
|
|
342
|
+
)
|
|
343
|
+
for block_index, block_bbox in target_bboxes.items()
|
|
344
|
+
if block_index != primary_block_index
|
|
345
|
+
and _match_script_line_ranges(
|
|
346
|
+
target_projected[block_index],
|
|
347
|
+
evidence_line,
|
|
348
|
+
)
|
|
349
|
+
if (
|
|
350
|
+
block_bbox[0] <= (script_range.bbox[0] + script_range.bbox[2]) / 2 <= block_bbox[2]
|
|
351
|
+
and block_bbox[1] <= (script_range.bbox[1] + script_range.bbox[3]) / 2 <= block_bbox[3]
|
|
352
|
+
)
|
|
353
|
+
or _bbox_overlap_ratio(script_range.bbox, block_bbox) >= 0.5
|
|
354
|
+
]
|
|
355
|
+
if not matches:
|
|
356
|
+
continue
|
|
357
|
+
block_index, _score = max(matches, key=lambda item: (*item[1], -item[0]))
|
|
358
|
+
fallback_ranges.setdefault((block_index, line_index), []).append(script_range)
|
|
359
|
+
for (block_index, line_index), script_ranges in fallback_ranges.items():
|
|
360
|
+
line = lines[line_index]
|
|
361
|
+
assignments.setdefault(block_index, []).append(
|
|
362
|
+
PDFTextScriptLine(
|
|
363
|
+
bbox=line.bbox,
|
|
364
|
+
text=line.text,
|
|
365
|
+
script_ranges=tuple(script_ranges),
|
|
366
|
+
source_index=line.source_index,
|
|
367
|
+
angle=line.angle,
|
|
368
|
+
)
|
|
369
|
+
)
|
|
370
|
+
for block_lines in assignments.values():
|
|
371
|
+
block_lines.sort(key=lambda line: (line.source_index, line.bbox[1], line.bbox[0]))
|
|
372
|
+
return assignments
|
|
373
|
+
|
|
374
|
+
|
|
375
|
+
def _filter_line_styles_for_block(
|
|
376
|
+
lines: Sequence[PDFTextStyleLine],
|
|
377
|
+
block_type: Any,
|
|
378
|
+
) -> list[PDFTextStyleLine]:
|
|
379
|
+
"""按目标 block 类型过滤样式区间,同时保留无样式物理行用于顺序对齐。"""
|
|
380
|
+
|
|
381
|
+
output: list[PDFTextStyleLine] = []
|
|
382
|
+
for line in lines:
|
|
383
|
+
filtered_ranges: list[PDFTextStyleRange] = []
|
|
384
|
+
for style_range in line.style_ranges:
|
|
385
|
+
styles = _canonical_styles(
|
|
386
|
+
style
|
|
387
|
+
for style in style_range.styles
|
|
388
|
+
if block_type
|
|
389
|
+
in _PDF_TEXT_STYLE_TARGET_BLOCK_TYPES.get(
|
|
390
|
+
style,
|
|
391
|
+
frozenset(),
|
|
392
|
+
)
|
|
393
|
+
)
|
|
394
|
+
if styles:
|
|
395
|
+
filtered_ranges.append(
|
|
396
|
+
PDFTextStyleRange(
|
|
397
|
+
style_range.start,
|
|
398
|
+
style_range.end,
|
|
399
|
+
styles,
|
|
400
|
+
)
|
|
401
|
+
)
|
|
402
|
+
output.append(
|
|
403
|
+
PDFTextStyleLine(
|
|
404
|
+
bbox=line.bbox,
|
|
405
|
+
text=line.text,
|
|
406
|
+
style_ranges=tuple(filtered_ranges),
|
|
407
|
+
source_index=line.source_index,
|
|
408
|
+
)
|
|
409
|
+
)
|
|
410
|
+
return output
|
|
411
|
+
|
|
412
|
+
|
|
413
|
+
def _project_content_chars(content: str) -> list[_ProjectedChar]:
|
|
414
|
+
"""把原始文字投影为忽略空白和圆括号公式的可比较字符。"""
|
|
415
|
+
|
|
416
|
+
projected: list[_ProjectedChar] = []
|
|
417
|
+
pending_formula_gap = False
|
|
418
|
+
cursor = 0
|
|
419
|
+
while cursor < len(content):
|
|
420
|
+
if content.startswith(r"\(", cursor):
|
|
421
|
+
formula_end = content.find(r"\)", cursor + 2)
|
|
422
|
+
if formula_end >= 0:
|
|
423
|
+
cursor = formula_end + 2
|
|
424
|
+
pending_formula_gap = True
|
|
425
|
+
continue
|
|
426
|
+
raw_char = content[cursor]
|
|
427
|
+
fragment = _normalize_match_fragment(raw_char)
|
|
428
|
+
for fragment_index, value in enumerate(fragment):
|
|
429
|
+
projected.append(
|
|
430
|
+
_ProjectedChar(
|
|
431
|
+
value=value,
|
|
432
|
+
raw_start=cursor,
|
|
433
|
+
raw_end=cursor + 1,
|
|
434
|
+
existing_styles=frozenset(),
|
|
435
|
+
formula_gap_before=pending_formula_gap and fragment_index == 0,
|
|
436
|
+
inside_hyperlink=False,
|
|
437
|
+
)
|
|
438
|
+
)
|
|
439
|
+
if fragment:
|
|
440
|
+
pending_formula_gap = False
|
|
441
|
+
cursor += 1
|
|
442
|
+
return projected
|
|
443
|
+
|
|
444
|
+
|
|
445
|
+
def _all_occurrences(content: str, target: str, start: int) -> list[int]:
|
|
446
|
+
"""返回 target 在 content 指定位置后的全部精确匹配起点。"""
|
|
447
|
+
|
|
448
|
+
output: list[int] = []
|
|
449
|
+
cursor = start
|
|
450
|
+
while target and (match := content.find(target, cursor)) >= 0:
|
|
451
|
+
output.append(match)
|
|
452
|
+
cursor = match + 1
|
|
453
|
+
return output
|
|
454
|
+
|
|
455
|
+
|
|
456
|
+
def _resolve_fallback_occurrence(
|
|
457
|
+
content: str,
|
|
458
|
+
line: PDFTextStyleLine,
|
|
459
|
+
style_range: PDFTextStyleRange,
|
|
460
|
+
start: int,
|
|
461
|
+
) -> int | None:
|
|
462
|
+
"""在整行无法对齐时,用唯一样式片段及两侧精确上下文选择位置。"""
|
|
463
|
+
|
|
464
|
+
target = line.text[style_range.start : style_range.end]
|
|
465
|
+
occurrences = _all_occurrences(content, target, start)
|
|
466
|
+
if not occurrences:
|
|
467
|
+
return None
|
|
468
|
+
left_context = line.text[max(0, style_range.start - 12) : style_range.start]
|
|
469
|
+
right_context = line.text[style_range.end : style_range.end + 12]
|
|
470
|
+
scored = [
|
|
471
|
+
(
|
|
472
|
+
int(bool(left_context) and content[max(0, position - len(left_context)) : position] == left_context)
|
|
473
|
+
+ int(
|
|
474
|
+
bool(right_context)
|
|
475
|
+
and content[position + len(target) : position + len(target) + len(right_context)] == right_context
|
|
476
|
+
),
|
|
477
|
+
position,
|
|
478
|
+
)
|
|
479
|
+
for position in occurrences
|
|
480
|
+
]
|
|
481
|
+
has_geometric_style = bool(_PDF_GEOMETRIC_TEXT_STYLES.intersection(style_range.styles))
|
|
482
|
+
if len(occurrences) == 1 and (has_geometric_style or len(target) >= 3):
|
|
483
|
+
return occurrences[0]
|
|
484
|
+
best_score = max(score for score, _position in scored)
|
|
485
|
+
best_positions = [position for score, position in scored if score == best_score]
|
|
486
|
+
if has_geometric_style:
|
|
487
|
+
return best_positions[0] if best_score > 0 and len(best_positions) == 1 else None
|
|
488
|
+
required_context_score = int(bool(left_context)) + int(bool(right_context))
|
|
489
|
+
return (
|
|
490
|
+
best_positions[0]
|
|
491
|
+
if required_context_score > 0 and best_score == required_context_score and len(best_positions) == 1
|
|
492
|
+
else None
|
|
493
|
+
)
|
|
494
|
+
|
|
495
|
+
|
|
496
|
+
def _match_line_across_formula_gaps(
|
|
497
|
+
line_text: str,
|
|
498
|
+
projected: Sequence[_ProjectedChar],
|
|
499
|
+
start: int,
|
|
500
|
+
) -> _LineProjectionMatch | None:
|
|
501
|
+
"""用精确字符序列跨过公式空洞,将一个物理行对齐到 block 文本。"""
|
|
502
|
+
|
|
503
|
+
if not line_text or start >= len(projected) or not any(token.formula_gap_before for token in projected[start:]):
|
|
504
|
+
return None
|
|
505
|
+
for projected_start in range(max(0, start), len(projected)):
|
|
506
|
+
first_token = projected[projected_start]
|
|
507
|
+
if not first_token.formula_gap_before and first_token.value != line_text[0]:
|
|
508
|
+
continue
|
|
509
|
+
states: dict[int, tuple[int | None, ...]] = {0: ()}
|
|
510
|
+
for projected_index in range(projected_start, len(projected)):
|
|
511
|
+
token = projected[projected_index]
|
|
512
|
+
next_states: dict[int, tuple[int | None, ...]] = {}
|
|
513
|
+
for source_index, mapping in states.items():
|
|
514
|
+
if source_index >= len(line_text):
|
|
515
|
+
continue
|
|
516
|
+
if token.formula_gap_before:
|
|
517
|
+
candidate_source_indices = range(
|
|
518
|
+
source_index + 1,
|
|
519
|
+
len(line_text),
|
|
520
|
+
)
|
|
521
|
+
elif line_text[source_index] == token.value:
|
|
522
|
+
candidate_source_indices = (source_index,)
|
|
523
|
+
else:
|
|
524
|
+
continue
|
|
525
|
+
for matched_source_index in candidate_source_indices:
|
|
526
|
+
if line_text[matched_source_index] != token.value:
|
|
527
|
+
continue
|
|
528
|
+
next_source_index = matched_source_index + 1
|
|
529
|
+
next_mapping = (
|
|
530
|
+
*mapping,
|
|
531
|
+
*([None] * (matched_source_index - source_index)),
|
|
532
|
+
projected_index,
|
|
533
|
+
)
|
|
534
|
+
next_states.setdefault(next_source_index, next_mapping)
|
|
535
|
+
complete = next_states.get(len(line_text))
|
|
536
|
+
if complete is not None:
|
|
537
|
+
return _LineProjectionMatch(
|
|
538
|
+
start=projected_start,
|
|
539
|
+
end=projected_index + 1,
|
|
540
|
+
source_to_projected=complete,
|
|
541
|
+
)
|
|
542
|
+
if not next_states:
|
|
543
|
+
break
|
|
544
|
+
states = next_states
|
|
545
|
+
return None
|
|
546
|
+
|
|
547
|
+
|
|
548
|
+
def _ranges_from_line_projection(
|
|
549
|
+
line: PDFTextStyleLine,
|
|
550
|
+
match: _LineProjectionMatch,
|
|
551
|
+
) -> list[PDFTextStyleRange]:
|
|
552
|
+
"""把物理行样式区间投影为公式字符被跳过后的 block 文本区间。"""
|
|
553
|
+
|
|
554
|
+
output: list[PDFTextStyleRange] = []
|
|
555
|
+
for style_range in line.style_ranges:
|
|
556
|
+
current_start: int | None = None
|
|
557
|
+
previous_index: int | None = None
|
|
558
|
+
for projected_index in match.source_to_projected[style_range.start : style_range.end]:
|
|
559
|
+
if projected_index is None:
|
|
560
|
+
if current_start is not None and previous_index is not None:
|
|
561
|
+
output.append(
|
|
562
|
+
PDFTextStyleRange(
|
|
563
|
+
current_start,
|
|
564
|
+
previous_index + 1,
|
|
565
|
+
style_range.styles,
|
|
566
|
+
)
|
|
567
|
+
)
|
|
568
|
+
current_start = None
|
|
569
|
+
previous_index = None
|
|
570
|
+
continue
|
|
571
|
+
if current_start is not None and previous_index is not None and projected_index != previous_index + 1:
|
|
572
|
+
output.append(
|
|
573
|
+
PDFTextStyleRange(
|
|
574
|
+
current_start,
|
|
575
|
+
previous_index + 1,
|
|
576
|
+
style_range.styles,
|
|
577
|
+
)
|
|
578
|
+
)
|
|
579
|
+
current_start = None
|
|
580
|
+
if current_start is None:
|
|
581
|
+
current_start = projected_index
|
|
582
|
+
previous_index = projected_index
|
|
583
|
+
if current_start is not None and previous_index is not None:
|
|
584
|
+
output.append(
|
|
585
|
+
PDFTextStyleRange(
|
|
586
|
+
current_start,
|
|
587
|
+
previous_index + 1,
|
|
588
|
+
style_range.styles,
|
|
589
|
+
)
|
|
590
|
+
)
|
|
591
|
+
return output
|
|
592
|
+
|
|
593
|
+
|
|
594
|
+
def _lines_form_dehyphenated_continuation(
|
|
595
|
+
line: PDFTextEvidenceLine,
|
|
596
|
+
next_line: PDFTextEvidenceLine | None,
|
|
597
|
+
) -> bool:
|
|
598
|
+
"""判断相邻物理行是否符合正文回填使用的英文断词规则。"""
|
|
599
|
+
|
|
600
|
+
return bool(
|
|
601
|
+
next_line is not None
|
|
602
|
+
and next_line.source_index == line.source_index + 1
|
|
603
|
+
and line.text
|
|
604
|
+
and next_line.text
|
|
605
|
+
and is_hyphen_at_line_end(line.text)
|
|
606
|
+
and next_line.text[0].islower()
|
|
607
|
+
)
|
|
608
|
+
|
|
609
|
+
|
|
610
|
+
def _match_line_without_terminal_hyphen(
|
|
611
|
+
projected_text: str,
|
|
612
|
+
line: PDFTextEvidenceLine,
|
|
613
|
+
next_line: PDFTextEvidenceLine | None,
|
|
614
|
+
start: int,
|
|
615
|
+
) -> _LineProjectionMatch | None:
|
|
616
|
+
"""将 block 已删除的行末断词符映射为空洞,歧义时拒绝匹配。"""
|
|
617
|
+
|
|
618
|
+
if not _lines_form_dehyphenated_continuation(line, next_line):
|
|
619
|
+
return None
|
|
620
|
+
candidate = line.text[:-1]
|
|
621
|
+
next_first_char = next_line.text[0] if next_line is not None else ""
|
|
622
|
+
occurrences = [
|
|
623
|
+
position
|
|
624
|
+
for position in _all_occurrences(projected_text, candidate, start)
|
|
625
|
+
if (position + len(candidate) < len(projected_text) and projected_text[position + len(candidate)] == next_first_char)
|
|
626
|
+
]
|
|
627
|
+
if len(occurrences) != 1:
|
|
628
|
+
logger.debug(f"Skip ambiguous PDF text dehyphenation mapping: line={line.text!r}, occurrences={len(occurrences)}")
|
|
629
|
+
return None
|
|
630
|
+
position = occurrences[0]
|
|
631
|
+
return _LineProjectionMatch(
|
|
632
|
+
start=position,
|
|
633
|
+
end=position + len(candidate),
|
|
634
|
+
source_to_projected=(
|
|
635
|
+
*range(position, position + len(candidate)),
|
|
636
|
+
None,
|
|
637
|
+
),
|
|
638
|
+
)
|
|
639
|
+
|
|
640
|
+
|
|
641
|
+
def _match_style_ranges(
|
|
642
|
+
projected: Sequence[_ProjectedChar],
|
|
643
|
+
lines: Sequence[PDFTextStyleLine],
|
|
644
|
+
) -> list[PDFTextStyleRange]:
|
|
645
|
+
"""按物理行顺序把字体与装饰线证据确定性对齐到 block 文本。"""
|
|
646
|
+
|
|
647
|
+
projected_text = "".join(token.value for token in projected)
|
|
648
|
+
output: list[PDFTextStyleRange] = []
|
|
649
|
+
cursor = 0
|
|
650
|
+
for line_index, line in enumerate(lines):
|
|
651
|
+
next_line = lines[line_index + 1] if line_index + 1 < len(lines) else None
|
|
652
|
+
line_start = projected_text.find(line.text, cursor)
|
|
653
|
+
if line_start >= 0:
|
|
654
|
+
output.extend(
|
|
655
|
+
PDFTextStyleRange(
|
|
656
|
+
line_start + style_range.start,
|
|
657
|
+
line_start + style_range.end,
|
|
658
|
+
style_range.styles,
|
|
659
|
+
)
|
|
660
|
+
for style_range in line.style_ranges
|
|
661
|
+
)
|
|
662
|
+
cursor = line_start + len(line.text)
|
|
663
|
+
continue
|
|
664
|
+
formula_match = _match_line_across_formula_gaps(
|
|
665
|
+
line.text,
|
|
666
|
+
projected,
|
|
667
|
+
cursor,
|
|
668
|
+
)
|
|
669
|
+
if formula_match is not None:
|
|
670
|
+
output.extend(_ranges_from_line_projection(line, formula_match))
|
|
671
|
+
cursor = formula_match.end
|
|
672
|
+
continue
|
|
673
|
+
dehyphenated_match = _match_line_without_terminal_hyphen(
|
|
674
|
+
projected_text,
|
|
675
|
+
line,
|
|
676
|
+
next_line,
|
|
677
|
+
cursor,
|
|
678
|
+
)
|
|
679
|
+
if dehyphenated_match is not None:
|
|
680
|
+
output.extend(_ranges_from_line_projection(line, dehyphenated_match))
|
|
681
|
+
cursor = dehyphenated_match.end
|
|
682
|
+
continue
|
|
683
|
+
skipped_ranges: list[PDFTextStyleRange] = []
|
|
684
|
+
for style_range in line.style_ranges:
|
|
685
|
+
position = _resolve_fallback_occurrence(
|
|
686
|
+
projected_text,
|
|
687
|
+
line,
|
|
688
|
+
style_range,
|
|
689
|
+
cursor,
|
|
690
|
+
)
|
|
691
|
+
if position is None:
|
|
692
|
+
skipped_ranges.append(style_range)
|
|
693
|
+
continue
|
|
694
|
+
output.append(
|
|
695
|
+
PDFTextStyleRange(
|
|
696
|
+
position,
|
|
697
|
+
position + style_range.end - style_range.start,
|
|
698
|
+
style_range.styles,
|
|
699
|
+
)
|
|
700
|
+
)
|
|
701
|
+
cursor = position + style_range.end - style_range.start
|
|
702
|
+
if skipped_ranges:
|
|
703
|
+
skipped_samples = [
|
|
704
|
+
(
|
|
705
|
+
line.text[style_range.start : style_range.end],
|
|
706
|
+
style_range.styles,
|
|
707
|
+
)
|
|
708
|
+
for style_range in skipped_ranges[:3]
|
|
709
|
+
]
|
|
710
|
+
logger.debug(
|
|
711
|
+
"Skip ambiguous PDF text style mapping: "
|
|
712
|
+
f"line={line.text!r}, skipped={len(skipped_ranges)}, "
|
|
713
|
+
f"samples={skipped_samples!r}"
|
|
714
|
+
)
|
|
715
|
+
return _merge_style_ranges(output)
|
|
716
|
+
|
|
717
|
+
|
|
718
|
+
def _match_script_line_ranges(
|
|
719
|
+
projected: Sequence[_ProjectedChar],
|
|
720
|
+
line: PDFTextStyleLine,
|
|
721
|
+
) -> list[PDFTextStyleRange]:
|
|
722
|
+
"""独立投影单条脚本行,避免其它视觉行推进 cursor 后吞掉短脚本。"""
|
|
723
|
+
|
|
724
|
+
projected_text = "".join(token.value for token in projected)
|
|
725
|
+
exact_occurrences = _all_occurrences(projected_text, line.text, 0)
|
|
726
|
+
if len(exact_occurrences) == 1:
|
|
727
|
+
line_start = exact_occurrences[0]
|
|
728
|
+
return [
|
|
729
|
+
PDFTextStyleRange(
|
|
730
|
+
line_start + style_range.start,
|
|
731
|
+
line_start + style_range.end,
|
|
732
|
+
style_range.styles,
|
|
733
|
+
)
|
|
734
|
+
for style_range in line.style_ranges
|
|
735
|
+
]
|
|
736
|
+
formula_match = _match_line_across_formula_gaps(
|
|
737
|
+
line.text,
|
|
738
|
+
projected,
|
|
739
|
+
0,
|
|
740
|
+
)
|
|
741
|
+
if formula_match is not None:
|
|
742
|
+
return _ranges_from_line_projection(line, formula_match)
|
|
743
|
+
output: list[PDFTextStyleRange] = []
|
|
744
|
+
for style_range in line.style_ranges:
|
|
745
|
+
position = _resolve_fallback_occurrence(
|
|
746
|
+
projected_text,
|
|
747
|
+
line,
|
|
748
|
+
style_range,
|
|
749
|
+
0,
|
|
750
|
+
)
|
|
751
|
+
if position is None:
|
|
752
|
+
continue
|
|
753
|
+
output.append(
|
|
754
|
+
PDFTextStyleRange(
|
|
755
|
+
position,
|
|
756
|
+
position + style_range.end - style_range.start,
|
|
757
|
+
style_range.styles,
|
|
758
|
+
)
|
|
759
|
+
)
|
|
760
|
+
return _merge_style_ranges(output)
|
|
761
|
+
|
|
762
|
+
|
|
763
|
+
def _merge_style_ranges(ranges: Sequence[PDFTextStyleRange]) -> list[PDFTextStyleRange]:
|
|
764
|
+
"""把重叠样式取并集,并合并相邻且样式集合相同的区间。"""
|
|
765
|
+
|
|
766
|
+
events: dict[int, dict[PDFTextStyle, int]] = {}
|
|
767
|
+
for style_range in ranges:
|
|
768
|
+
styles = _canonical_styles(style_range.styles)
|
|
769
|
+
if style_range.start >= style_range.end or not styles:
|
|
770
|
+
continue
|
|
771
|
+
for position, delta in (
|
|
772
|
+
(style_range.start, 1),
|
|
773
|
+
(style_range.end, -1),
|
|
774
|
+
):
|
|
775
|
+
position_events = events.setdefault(position, {})
|
|
776
|
+
for style in styles:
|
|
777
|
+
position_events[style] = position_events.get(style, 0) + delta
|
|
778
|
+
|
|
779
|
+
active_counts: dict[PDFTextStyle, int] = {}
|
|
780
|
+
merged: list[PDFTextStyleRange] = []
|
|
781
|
+
previous_position: int | None = None
|
|
782
|
+
for position in sorted(events):
|
|
783
|
+
active_styles = _canonical_styles(style for style, count in active_counts.items() if count > 0)
|
|
784
|
+
if previous_position is not None and previous_position < position and active_styles:
|
|
785
|
+
if merged and merged[-1].end == previous_position and merged[-1].styles == active_styles:
|
|
786
|
+
merged[-1] = PDFTextStyleRange(
|
|
787
|
+
merged[-1].start,
|
|
788
|
+
position,
|
|
789
|
+
active_styles,
|
|
790
|
+
)
|
|
791
|
+
else:
|
|
792
|
+
merged.append(
|
|
793
|
+
PDFTextStyleRange(
|
|
794
|
+
previous_position,
|
|
795
|
+
position,
|
|
796
|
+
active_styles,
|
|
797
|
+
)
|
|
798
|
+
)
|
|
799
|
+
for style, delta in events[position].items():
|
|
800
|
+
active_counts[style] = active_counts.get(style, 0) + delta
|
|
801
|
+
previous_position = position
|
|
802
|
+
return merged
|
|
803
|
+
|
|
804
|
+
|
|
805
|
+
def _resolve_link_fallback_occurrence(
|
|
806
|
+
content: str,
|
|
807
|
+
line: PDFTextLinkLine,
|
|
808
|
+
link_range: PDFTextLinkRange,
|
|
809
|
+
start: int,
|
|
810
|
+
) -> int | None:
|
|
811
|
+
"""整行无法对齐时,用唯一标签或两侧精确上下文定位链接片段。"""
|
|
812
|
+
|
|
813
|
+
target_text = line.text[link_range.start : link_range.end]
|
|
814
|
+
occurrences = _all_occurrences(content, target_text, start)
|
|
815
|
+
if not occurrences:
|
|
816
|
+
return None
|
|
817
|
+
if len(occurrences) == 1:
|
|
818
|
+
return occurrences[0]
|
|
819
|
+
|
|
820
|
+
left_context = line.text[max(0, link_range.start - 12) : link_range.start]
|
|
821
|
+
right_context = line.text[link_range.end : link_range.end + 12]
|
|
822
|
+
required_context_score = int(bool(left_context)) + int(bool(right_context))
|
|
823
|
+
if required_context_score == 0:
|
|
824
|
+
return None
|
|
825
|
+
scored = [
|
|
826
|
+
(
|
|
827
|
+
int(bool(left_context) and content[max(0, position - len(left_context)) : position] == left_context)
|
|
828
|
+
+ int(
|
|
829
|
+
bool(right_context)
|
|
830
|
+
and content[position + len(target_text) : position + len(target_text) + len(right_context)] == right_context
|
|
831
|
+
),
|
|
832
|
+
position,
|
|
833
|
+
)
|
|
834
|
+
for position in occurrences
|
|
835
|
+
]
|
|
836
|
+
best_score = max(score for score, _position in scored)
|
|
837
|
+
best_positions = [position for score, position in scored if score == best_score]
|
|
838
|
+
return best_positions[0] if best_score == required_context_score and len(best_positions) == 1 else None
|
|
839
|
+
|
|
840
|
+
|
|
841
|
+
def _project_link_range_from_line_match(
|
|
842
|
+
link_range: PDFTextLinkRange,
|
|
843
|
+
match: _LineProjectionMatch,
|
|
844
|
+
source_index: int,
|
|
845
|
+
) -> list[_MatchedLinkRange]:
|
|
846
|
+
"""按行字符投影映射链接区间,并在缺失字符处安全分段。"""
|
|
847
|
+
|
|
848
|
+
output: list[_MatchedLinkRange] = []
|
|
849
|
+
current_start: int | None = None
|
|
850
|
+
previous_index: int | None = None
|
|
851
|
+
for projected_index in match.source_to_projected[link_range.start : link_range.end]:
|
|
852
|
+
if projected_index is None:
|
|
853
|
+
if current_start is not None and previous_index is not None:
|
|
854
|
+
output.append(
|
|
855
|
+
_MatchedLinkRange(
|
|
856
|
+
current_start,
|
|
857
|
+
previous_index + 1,
|
|
858
|
+
link_range.target,
|
|
859
|
+
source_index,
|
|
860
|
+
)
|
|
861
|
+
)
|
|
862
|
+
current_start = None
|
|
863
|
+
previous_index = None
|
|
864
|
+
continue
|
|
865
|
+
if current_start is not None and previous_index is not None and projected_index != previous_index + 1:
|
|
866
|
+
output.append(
|
|
867
|
+
_MatchedLinkRange(
|
|
868
|
+
current_start,
|
|
869
|
+
previous_index + 1,
|
|
870
|
+
link_range.target,
|
|
871
|
+
source_index,
|
|
872
|
+
)
|
|
873
|
+
)
|
|
874
|
+
current_start = None
|
|
875
|
+
if current_start is None:
|
|
876
|
+
current_start = projected_index
|
|
877
|
+
previous_index = projected_index
|
|
878
|
+
if current_start is not None and previous_index is not None:
|
|
879
|
+
output.append(
|
|
880
|
+
_MatchedLinkRange(
|
|
881
|
+
current_start,
|
|
882
|
+
previous_index + 1,
|
|
883
|
+
link_range.target,
|
|
884
|
+
source_index,
|
|
885
|
+
)
|
|
886
|
+
)
|
|
887
|
+
return output
|
|
888
|
+
|
|
889
|
+
|
|
890
|
+
def _link_lines_form_dehyphenated_continuation(
|
|
891
|
+
line: PDFTextLinkLine,
|
|
892
|
+
next_line: PDFTextLinkLine | None,
|
|
893
|
+
) -> bool:
|
|
894
|
+
"""判断相邻同 href 链接行是否符合文本回填的英文断词规则。"""
|
|
895
|
+
|
|
896
|
+
if not _lines_form_dehyphenated_continuation(line, next_line):
|
|
897
|
+
return False
|
|
898
|
+
tail_targets = {link_range.target for link_range in line.link_ranges if link_range.start < link_range.end == len(line.text)}
|
|
899
|
+
head_targets = {link_range.target for link_range in next_line.link_ranges if link_range.start == 0 < link_range.end}
|
|
900
|
+
return bool(tail_targets.intersection(head_targets))
|
|
901
|
+
|
|
902
|
+
|
|
903
|
+
def _match_link_line_without_terminal_hyphen(
|
|
904
|
+
projected_text: str,
|
|
905
|
+
line: PDFTextLinkLine,
|
|
906
|
+
next_line: PDFTextLinkLine | None,
|
|
907
|
+
start: int,
|
|
908
|
+
) -> _LineProjectionMatch | None:
|
|
909
|
+
"""在严格跨行条件下将已被 block 回填删除的行末断词符投影为空洞。"""
|
|
910
|
+
|
|
911
|
+
if not _link_lines_form_dehyphenated_continuation(line, next_line):
|
|
912
|
+
return None
|
|
913
|
+
return _match_line_without_terminal_hyphen(
|
|
914
|
+
projected_text,
|
|
915
|
+
line,
|
|
916
|
+
next_line,
|
|
917
|
+
start,
|
|
918
|
+
)
|
|
919
|
+
|
|
920
|
+
|
|
921
|
+
def _merge_matched_link_ranges(
|
|
922
|
+
ranges: Sequence[_MatchedLinkRange],
|
|
923
|
+
) -> list[_MatchedLinkRange]:
|
|
924
|
+
"""删除不同目标重叠区,并合并同一物理行内的同目标相邻区间。"""
|
|
925
|
+
|
|
926
|
+
valid_ranges = [link_range for link_range in ranges if link_range.start < link_range.end and link_range.target]
|
|
927
|
+
if not valid_ranges:
|
|
928
|
+
return []
|
|
929
|
+
boundaries = sorted({position for link_range in valid_ranges for position in (link_range.start, link_range.end)})
|
|
930
|
+
merged: list[_MatchedLinkRange] = []
|
|
931
|
+
for start, end in zip(boundaries, boundaries[1:]):
|
|
932
|
+
active = [link_range for link_range in valid_ranges if link_range.start < end and link_range.end > start]
|
|
933
|
+
targets = {link_range.target for link_range in active}
|
|
934
|
+
if len(targets) != 1:
|
|
935
|
+
continue
|
|
936
|
+
target = next(iter(targets))
|
|
937
|
+
source_index = min(link_range.source_index for link_range in active if link_range.target == target)
|
|
938
|
+
if merged and merged[-1].end == start and merged[-1].target == target and merged[-1].source_index == source_index:
|
|
939
|
+
merged[-1] = _MatchedLinkRange(
|
|
940
|
+
merged[-1].start,
|
|
941
|
+
end,
|
|
942
|
+
target,
|
|
943
|
+
source_index,
|
|
944
|
+
)
|
|
945
|
+
else:
|
|
946
|
+
merged.append(
|
|
947
|
+
_MatchedLinkRange(
|
|
948
|
+
start,
|
|
949
|
+
end,
|
|
950
|
+
target,
|
|
951
|
+
source_index,
|
|
952
|
+
)
|
|
953
|
+
)
|
|
954
|
+
return merged
|
|
955
|
+
|
|
956
|
+
|
|
957
|
+
def _match_link_ranges(
|
|
958
|
+
projected: Sequence[_ProjectedChar],
|
|
959
|
+
lines: Sequence[PDFTextLinkLine],
|
|
960
|
+
) -> list[_MatchedLinkRange]:
|
|
961
|
+
"""按物理行顺序把 Link 几何证据确定性对齐到 block 文本。"""
|
|
962
|
+
|
|
963
|
+
projected_text = "".join(token.value for token in projected)
|
|
964
|
+
output: list[_MatchedLinkRange] = []
|
|
965
|
+
cursor = 0
|
|
966
|
+
for line_index, line in enumerate(lines):
|
|
967
|
+
next_line = lines[line_index + 1] if line_index + 1 < len(lines) else None
|
|
968
|
+
line_start = projected_text.find(line.text, cursor)
|
|
969
|
+
if line_start >= 0:
|
|
970
|
+
output.extend(
|
|
971
|
+
_MatchedLinkRange(
|
|
972
|
+
line_start + link_range.start,
|
|
973
|
+
line_start + link_range.end,
|
|
974
|
+
link_range.target,
|
|
975
|
+
line.source_index,
|
|
976
|
+
)
|
|
977
|
+
for link_range in line.link_ranges
|
|
978
|
+
)
|
|
979
|
+
cursor = line_start + len(line.text)
|
|
980
|
+
continue
|
|
981
|
+
formula_match = _match_line_across_formula_gaps(
|
|
982
|
+
line.text,
|
|
983
|
+
projected,
|
|
984
|
+
cursor,
|
|
985
|
+
)
|
|
986
|
+
if formula_match is not None:
|
|
987
|
+
for link_range in line.link_ranges:
|
|
988
|
+
output.extend(
|
|
989
|
+
_project_link_range_from_line_match(
|
|
990
|
+
link_range,
|
|
991
|
+
formula_match,
|
|
992
|
+
line.source_index,
|
|
993
|
+
)
|
|
994
|
+
)
|
|
995
|
+
cursor = formula_match.end
|
|
996
|
+
continue
|
|
997
|
+
|
|
998
|
+
dehyphenated_match = _match_link_line_without_terminal_hyphen(
|
|
999
|
+
projected_text,
|
|
1000
|
+
line,
|
|
1001
|
+
next_line,
|
|
1002
|
+
cursor,
|
|
1003
|
+
)
|
|
1004
|
+
if dehyphenated_match is not None:
|
|
1005
|
+
for link_range in line.link_ranges:
|
|
1006
|
+
output.extend(
|
|
1007
|
+
_project_link_range_from_line_match(
|
|
1008
|
+
link_range,
|
|
1009
|
+
dehyphenated_match,
|
|
1010
|
+
line.source_index,
|
|
1011
|
+
)
|
|
1012
|
+
)
|
|
1013
|
+
cursor = dehyphenated_match.end
|
|
1014
|
+
continue
|
|
1015
|
+
|
|
1016
|
+
skipped_ranges: list[PDFTextLinkRange] = []
|
|
1017
|
+
for link_range in line.link_ranges:
|
|
1018
|
+
position = _resolve_link_fallback_occurrence(
|
|
1019
|
+
projected_text,
|
|
1020
|
+
line,
|
|
1021
|
+
link_range,
|
|
1022
|
+
cursor,
|
|
1023
|
+
)
|
|
1024
|
+
if position is None:
|
|
1025
|
+
skipped_ranges.append(link_range)
|
|
1026
|
+
continue
|
|
1027
|
+
output.append(
|
|
1028
|
+
_MatchedLinkRange(
|
|
1029
|
+
position,
|
|
1030
|
+
position + link_range.end - link_range.start,
|
|
1031
|
+
link_range.target,
|
|
1032
|
+
line.source_index,
|
|
1033
|
+
)
|
|
1034
|
+
)
|
|
1035
|
+
cursor = position + link_range.end - link_range.start
|
|
1036
|
+
if skipped_ranges:
|
|
1037
|
+
logger.debug(
|
|
1038
|
+
"Skip ambiguous PDF hyperlink mapping: "
|
|
1039
|
+
f"line={line.text!r}, skipped={len(skipped_ranges)}, "
|
|
1040
|
+
f"samples={[(line.text[item.start : item.end], item.target) for item in skipped_ranges[:3]]!r}"
|
|
1041
|
+
)
|
|
1042
|
+
return _merge_matched_link_ranges(output)
|
|
1043
|
+
|
|
1044
|
+
|
|
1045
|
+
def _append_raw_link_interval(
|
|
1046
|
+
intervals: list[_RawLinkInterval],
|
|
1047
|
+
start: int | None,
|
|
1048
|
+
end: int,
|
|
1049
|
+
target: str,
|
|
1050
|
+
source_index: int,
|
|
1051
|
+
) -> None:
|
|
1052
|
+
"""向结果追加一个合法原字符串链接区间。"""
|
|
1053
|
+
|
|
1054
|
+
if start is not None and start < end and target:
|
|
1055
|
+
intervals.append(
|
|
1056
|
+
_RawLinkInterval(
|
|
1057
|
+
start,
|
|
1058
|
+
end,
|
|
1059
|
+
target,
|
|
1060
|
+
source_index,
|
|
1061
|
+
)
|
|
1062
|
+
)
|
|
1063
|
+
|
|
1064
|
+
|
|
1065
|
+
def _raw_link_intervals(
|
|
1066
|
+
content: str,
|
|
1067
|
+
projected: Sequence[_ProjectedChar],
|
|
1068
|
+
ranges: Sequence[_MatchedLinkRange],
|
|
1069
|
+
) -> list[_RawLinkInterval]:
|
|
1070
|
+
"""把链接区间转换为不跨公式或已有 hyperlink 的原字符串区间。"""
|
|
1071
|
+
|
|
1072
|
+
intervals: list[_RawLinkInterval] = []
|
|
1073
|
+
for link_range in ranges:
|
|
1074
|
+
current_start: int | None = None
|
|
1075
|
+
current_end = 0
|
|
1076
|
+
for token in projected[link_range.start : link_range.end]:
|
|
1077
|
+
if token.inside_hyperlink or (token.formula_gap_before and current_start is not None):
|
|
1078
|
+
_append_raw_link_interval(
|
|
1079
|
+
intervals,
|
|
1080
|
+
current_start,
|
|
1081
|
+
current_end,
|
|
1082
|
+
link_range.target,
|
|
1083
|
+
link_range.source_index,
|
|
1084
|
+
)
|
|
1085
|
+
current_start = None
|
|
1086
|
+
if token.inside_hyperlink:
|
|
1087
|
+
continue
|
|
1088
|
+
if current_start is None:
|
|
1089
|
+
current_start = token.raw_start
|
|
1090
|
+
current_end = token.raw_end
|
|
1091
|
+
continue
|
|
1092
|
+
gap = content[current_end : token.raw_start]
|
|
1093
|
+
if token.raw_start <= current_end or not gap or gap.isspace():
|
|
1094
|
+
current_end = max(current_end, token.raw_end)
|
|
1095
|
+
else:
|
|
1096
|
+
_append_raw_link_interval(
|
|
1097
|
+
intervals,
|
|
1098
|
+
current_start,
|
|
1099
|
+
current_end,
|
|
1100
|
+
link_range.target,
|
|
1101
|
+
link_range.source_index,
|
|
1102
|
+
)
|
|
1103
|
+
current_start = token.raw_start
|
|
1104
|
+
current_end = token.raw_end
|
|
1105
|
+
_append_raw_link_interval(
|
|
1106
|
+
intervals,
|
|
1107
|
+
current_start,
|
|
1108
|
+
current_end,
|
|
1109
|
+
link_range.target,
|
|
1110
|
+
link_range.source_index,
|
|
1111
|
+
)
|
|
1112
|
+
return intervals
|
|
1113
|
+
|
|
1114
|
+
|
|
1115
|
+
def _raw_link_gap_is_boundary_only(gap: str) -> bool:
|
|
1116
|
+
"""判断两个跨行链接片段之间是否只包含空白或非正文边界符号。"""
|
|
1117
|
+
|
|
1118
|
+
if not gap:
|
|
1119
|
+
return True
|
|
1120
|
+
if r"\(" in gap or r"\)" in gap:
|
|
1121
|
+
return False
|
|
1122
|
+
return not any(char.isalnum() for char in html.unescape(gap))
|
|
1123
|
+
|
|
1124
|
+
|
|
1125
|
+
def _merge_raw_link_intervals(
|
|
1126
|
+
content: str,
|
|
1127
|
+
intervals: Sequence[_RawLinkInterval],
|
|
1128
|
+
) -> list[_RawLinkInterval]:
|
|
1129
|
+
"""合并相邻物理行中同 href 的首尾链接片段,不跨越正文或公式。"""
|
|
1130
|
+
|
|
1131
|
+
merged: list[_RawLinkInterval] = []
|
|
1132
|
+
for interval in sorted(
|
|
1133
|
+
intervals,
|
|
1134
|
+
key=lambda item: (item.start, item.end, item.source_index, item.target),
|
|
1135
|
+
):
|
|
1136
|
+
if interval.start >= interval.end or not interval.target:
|
|
1137
|
+
continue
|
|
1138
|
+
if (
|
|
1139
|
+
merged
|
|
1140
|
+
and merged[-1].target == interval.target
|
|
1141
|
+
and interval.source_index == merged[-1].source_index + 1
|
|
1142
|
+
and interval.start >= merged[-1].end
|
|
1143
|
+
and _raw_link_gap_is_boundary_only(content[merged[-1].end : interval.start])
|
|
1144
|
+
):
|
|
1145
|
+
merged[-1] = _RawLinkInterval(
|
|
1146
|
+
merged[-1].start,
|
|
1147
|
+
interval.end,
|
|
1148
|
+
interval.target,
|
|
1149
|
+
interval.source_index,
|
|
1150
|
+
)
|
|
1151
|
+
else:
|
|
1152
|
+
merged.append(interval)
|
|
1153
|
+
return merged
|
|
1154
|
+
|
|
1155
|
+
|
|
1156
|
+
__all__ = [
|
|
1157
|
+
"_resplit_evidence_segments",
|
|
1158
|
+
"_partition_resplit_text_evidence",
|
|
1159
|
+
"_realign_repaired_text_evidence",
|
|
1160
|
+
"_block_bbox_to_page_bbox",
|
|
1161
|
+
"_line_block_score",
|
|
1162
|
+
"_assign_lines_to_blocks",
|
|
1163
|
+
"_assign_script_lines_to_blocks",
|
|
1164
|
+
"_filter_line_styles_for_block",
|
|
1165
|
+
"_project_content_chars",
|
|
1166
|
+
"_all_occurrences",
|
|
1167
|
+
"_resolve_fallback_occurrence",
|
|
1168
|
+
"_match_line_across_formula_gaps",
|
|
1169
|
+
"_ranges_from_line_projection",
|
|
1170
|
+
"_lines_form_dehyphenated_continuation",
|
|
1171
|
+
"_match_line_without_terminal_hyphen",
|
|
1172
|
+
"_match_style_ranges",
|
|
1173
|
+
"_match_script_line_ranges",
|
|
1174
|
+
"_merge_style_ranges",
|
|
1175
|
+
"_resolve_link_fallback_occurrence",
|
|
1176
|
+
"_project_link_range_from_line_match",
|
|
1177
|
+
"_link_lines_form_dehyphenated_continuation",
|
|
1178
|
+
"_match_link_line_without_terminal_hyphen",
|
|
1179
|
+
"_merge_matched_link_ranges",
|
|
1180
|
+
"_match_link_ranges",
|
|
1181
|
+
"_append_raw_link_interval",
|
|
1182
|
+
"_raw_link_intervals",
|
|
1183
|
+
"_raw_link_gap_is_boundary_only",
|
|
1184
|
+
"_merge_raw_link_intervals",
|
|
1185
|
+
]
|