docvortex 0.2.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- docvortex/__init__.py +21 -0
- docvortex/analyzers/__init__.py +3 -0
- docvortex/analyzers/native/__init__.py +37 -0
- docvortex/analyzers/native/_shared/__init__.py +3 -0
- docvortex/analyzers/native/_shared/hyperlink.py +16 -0
- docvortex/analyzers/native/_shared/image.py +8 -0
- docvortex/analyzers/native/_shared/markup/__init__.py +47 -0
- docvortex/analyzers/native/_shared/markup/anchors.py +30 -0
- docvortex/analyzers/native/_shared/markup/formula.py +37 -0
- docvortex/analyzers/native/_shared/markup/projector.py +57 -0
- docvortex/analyzers/native/_shared/markup/styles.py +19 -0
- docvortex/analyzers/native/_shared/mathml.py +17 -0
- docvortex/analyzers/native/_shared/names.py +7 -0
- docvortex/analyzers/native/_shared/xycut.py +414 -0
- docvortex/analyzers/native/contracts.py +44 -0
- docvortex/analyzers/native/csv.py +351 -0
- docvortex/analyzers/native/epub/__init__.py +18 -0
- docvortex/analyzers/native/epub/constants.py +45 -0
- docvortex/analyzers/native/epub/converter.py +80 -0
- docvortex/analyzers/native/epub/errors.py +20 -0
- docvortex/analyzers/native/epub/metadata.py +31 -0
- docvortex/analyzers/native/epub/package.py +649 -0
- docvortex/analyzers/native/epub/xhtml.py +306 -0
- docvortex/analyzers/native/html/__init__.py +6 -0
- docvortex/analyzers/native/html/anchors.py +271 -0
- docvortex/analyzers/native/html/constants.py +25 -0
- docvortex/analyzers/native/html/contracts.py +7 -0
- docvortex/analyzers/native/html/converter.py +117 -0
- docvortex/analyzers/native/html/document.py +389 -0
- docvortex/analyzers/native/html/errors.py +12 -0
- docvortex/analyzers/native/html/resources.py +365 -0
- docvortex/analyzers/native/html/selector.py +421 -0
- docvortex/analyzers/native/models.py +209 -0
- docvortex/analyzers/native/ofd/__init__.py +14 -0
- docvortex/analyzers/native/ofd/constants.py +59 -0
- docvortex/analyzers/native/ofd/converter.py +31 -0
- docvortex/analyzers/native/ofd/errors.py +16 -0
- docvortex/analyzers/native/ofd/geometry.py +215 -0
- docvortex/analyzers/native/ofd/images.py +124 -0
- docvortex/analyzers/native/ofd/metadata.py +53 -0
- docvortex/analyzers/native/ofd/models.py +170 -0
- docvortex/analyzers/native/ofd/package.py +346 -0
- docvortex/analyzers/native/ofd/path.py +201 -0
- docvortex/analyzers/native/ofd/reading_order.py +293 -0
- docvortex/analyzers/native/ofd/resources.py +122 -0
- docvortex/analyzers/native/ofd/scene.py +347 -0
- docvortex/analyzers/native/ofd/table.py +240 -0
- docvortex/analyzers/native/ofd/text.py +521 -0
- docvortex/analyzers/native/office/__init__.py +3 -0
- docvortex/analyzers/native/office/doc/__init__.py +3 -0
- docvortex/analyzers/native/office/doc/bookmarks.py +81 -0
- docvortex/analyzers/native/office/doc/doc_converter.py +573 -0
- docvortex/analyzers/native/office/doc/fib.py +261 -0
- docvortex/analyzers/native/office/doc/fields.py +101 -0
- docvortex/analyzers/native/office/doc/formatting.py +171 -0
- docvortex/analyzers/native/office/doc/images.py +162 -0
- docvortex/analyzers/native/office/doc/lists.py +356 -0
- docvortex/analyzers/native/office/doc/models.py +164 -0
- docvortex/analyzers/native/office/doc/parser.py +851 -0
- docvortex/analyzers/native/office/doc/pieces.py +256 -0
- docvortex/analyzers/native/office/doc/records.py +40 -0
- docvortex/analyzers/native/office/doc/sprm.py +269 -0
- docvortex/analyzers/native/office/doc/styles.py +215 -0
- docvortex/analyzers/native/office/docx/__init__.py +3 -0
- docvortex/analyzers/native/office/docx/context.py +71 -0
- docvortex/analyzers/native/office/docx/docx_converter.py +614 -0
- docvortex/analyzers/native/office/docx/equationxml.py +119 -0
- docvortex/analyzers/native/office/docx/fields.py +816 -0
- docvortex/analyzers/native/office/docx/formatting_types.py +23 -0
- docvortex/analyzers/native/office/docx/main.py +50 -0
- docvortex/analyzers/native/office/docx/numbering.py +491 -0
- docvortex/analyzers/native/office/docx/office_xml.py +57 -0
- docvortex/analyzers/native/office/docx/package_normalizer.py +248 -0
- docvortex/analyzers/native/office/docx/resources.py +526 -0
- docvortex/analyzers/native/office/docx/styles.py +693 -0
- docvortex/analyzers/native/office/docx/tables.py +515 -0
- docvortex/analyzers/native/office/equation/__init__.py +3 -0
- docvortex/analyzers/native/office/equation/image.py +470 -0
- docvortex/analyzers/native/office/equation/latex_dict.py +324 -0
- docvortex/analyzers/native/office/equation/mtef.py +885 -0
- docvortex/analyzers/native/office/equation/mtef_v5.py +941 -0
- docvortex/analyzers/native/office/equation/omml.py +561 -0
- docvortex/analyzers/native/office/equation/ooxml.py +62 -0
- docvortex/analyzers/native/office/errors.py +33 -0
- docvortex/analyzers/native/office/image.py +307 -0
- docvortex/analyzers/native/office/legacy/__init__.py +3 -0
- docvortex/analyzers/native/office/legacy/binary.py +43 -0
- docvortex/analyzers/native/office/legacy/officeart.py +362 -0
- docvortex/analyzers/native/office/legacy/ole.py +110 -0
- docvortex/analyzers/native/office/limits.py +12 -0
- docvortex/analyzers/native/office/odf/__init__.py +3 -0
- docvortex/analyzers/native/office/odf/chart.py +76 -0
- docvortex/analyzers/native/office/odf/constants.py +70 -0
- docvortex/analyzers/native/office/odf/converters.py +403 -0
- docvortex/analyzers/native/office/odf/errors.py +18 -0
- docvortex/analyzers/native/office/odf/metadata.py +91 -0
- docvortex/analyzers/native/office/odf/models.py +176 -0
- docvortex/analyzers/native/office/odf/package.py +270 -0
- docvortex/analyzers/native/office/odf/styles.py +329 -0
- docvortex/analyzers/native/office/odf/table.py +469 -0
- docvortex/analyzers/native/office/odf/text.py +1002 -0
- docvortex/analyzers/native/office/ooxml_chart.py +1016 -0
- docvortex/analyzers/native/office/opc.py +38 -0
- docvortex/analyzers/native/office/ppt/__init__.py +3 -0
- docvortex/analyzers/native/office/ppt/models.py +118 -0
- docvortex/analyzers/native/office/ppt/parser.py +1895 -0
- docvortex/analyzers/native/office/ppt/ppt_converter.py +292 -0
- docvortex/analyzers/native/office/ppt/records.py +131 -0
- docvortex/analyzers/native/office/ppt/style_text.py +247 -0
- docvortex/analyzers/native/office/pptx/__init__.py +3 -0
- docvortex/analyzers/native/office/pptx/context.py +113 -0
- docvortex/analyzers/native/office/pptx/lists.py +558 -0
- docvortex/analyzers/native/office/pptx/main.py +20 -0
- docvortex/analyzers/native/office/pptx/package_normalizer.py +321 -0
- docvortex/analyzers/native/office/pptx/pptx_converter.py +323 -0
- docvortex/analyzers/native/office/pptx/resources.py +329 -0
- docvortex/analyzers/native/office/pptx/shapes.py +393 -0
- docvortex/analyzers/native/office/pptx/text_styles.py +625 -0
- docvortex/analyzers/native/office/pptx/titles.py +178 -0
- docvortex/analyzers/native/office/rich_text.py +420 -0
- docvortex/analyzers/native/office/rtf/__init__.py +3 -0
- docvortex/analyzers/native/office/rtf/converter.py +708 -0
- docvortex/analyzers/native/office/rtf/lexer.py +213 -0
- docvortex/analyzers/native/office/rtf/math.py +339 -0
- docvortex/analyzers/native/office/rtf/models.py +185 -0
- docvortex/analyzers/native/office/rtf/parser.py +1553 -0
- docvortex/analyzers/native/office/spreadsheet/__init__.py +3 -0
- docvortex/analyzers/native/office/spreadsheet/html.py +81 -0
- docvortex/analyzers/native/office/spreadsheet/models.py +79 -0
- docvortex/analyzers/native/office/spreadsheet/projector.py +928 -0
- docvortex/analyzers/native/office/streams.py +18 -0
- docvortex/analyzers/native/office/xls/__init__.py +3 -0
- docvortex/analyzers/native/office/xls/chart.py +132 -0
- docvortex/analyzers/native/office/xls/embedded_chart.py +299 -0
- docvortex/analyzers/native/office/xls/models.py +109 -0
- docvortex/analyzers/native/office/xls/number_format.py +521 -0
- docvortex/analyzers/native/office/xls/parser.py +1145 -0
- docvortex/analyzers/native/office/xls/records.py +201 -0
- docvortex/analyzers/native/office/xls/strings.py +205 -0
- docvortex/analyzers/native/office/xls/xls_converter.py +354 -0
- docvortex/analyzers/native/office/xlsx/__init__.py +3 -0
- docvortex/analyzers/native/office/xlsx/main.py +20 -0
- docvortex/analyzers/native/office/xlsx/ooxml_ole.py +522 -0
- docvortex/analyzers/native/office/xlsx/package_normalizer.py +310 -0
- docvortex/analyzers/native/office/xlsx/xlsx_converter.py +716 -0
- docvortex/analyzers/native/pdf/__init__.py +3 -0
- docvortex/analyzers/native/pdf/auxiliary_text.py +1671 -0
- docvortex/analyzers/native/pdf/char_geometry.py +1662 -0
- docvortex/analyzers/native/pdf/code_blocks.py +535 -0
- docvortex/analyzers/native/pdf/formulas.py +1985 -0
- docvortex/analyzers/native/pdf/geometry.py +281 -0
- docvortex/analyzers/native/pdf/graphics.py +1501 -0
- docvortex/analyzers/native/pdf/index_blocks.py +268 -0
- docvortex/analyzers/native/pdf/inline/__init__.py +3 -0
- docvortex/analyzers/native/pdf/inline/common.py +112 -0
- docvortex/analyzers/native/pdf/inline/detection.py +590 -0
- docvortex/analyzers/native/pdf/inline/matching.py +1185 -0
- docvortex/analyzers/native/pdf/inline/materialize.py +457 -0
- docvortex/analyzers/native/pdf/inline/scripts.py +975 -0
- docvortex/analyzers/native/pdf/inline/types.py +385 -0
- docvortex/analyzers/native/pdf/line_layout.py +1106 -0
- docvortex/analyzers/native/pdf/line_merging.py +1223 -0
- docvortex/analyzers/native/pdf/models.py +246 -0
- docvortex/analyzers/native/pdf/native_text.py +1004 -0
- docvortex/analyzers/native/pdf/pipeline.py +1390 -0
- docvortex/analyzers/native/pdf/script_geometry.py +636 -0
- docvortex/analyzers/native/pdf/shared.py +30 -0
- docvortex/analyzers/native/pdf/spatial_text.py +383 -0
- docvortex/analyzers/native/pdf/table_annotations.py +446 -0
- docvortex/analyzers/native/pdf/table_constants.py +48 -0
- docvortex/analyzers/native/pdf/table_detection.py +227 -0
- docvortex/analyzers/native/pdf/table_filled_grid.py +218 -0
- docvortex/analyzers/native/pdf/table_geometry.py +40 -0
- docvortex/analyzers/native/pdf/table_materialization.py +444 -0
- docvortex/analyzers/native/pdf/table_recovery/__init__.py +15 -0
- docvortex/analyzers/native/pdf/table_recovery/candidate.py +356 -0
- docvortex/analyzers/native/pdf/table_recovery/contracts.py +154 -0
- docvortex/analyzers/native/pdf/table_recovery/engine.py +609 -0
- docvortex/analyzers/native/pdf/table_recovery/geometry.py +164 -0
- docvortex/analyzers/native/pdf/table_recovery/sparse_common.py +85 -0
- docvortex/analyzers/native/pdf/table_recovery/sparse_hybrid.py +804 -0
- docvortex/analyzers/native/pdf/table_recovery/sparse_multiline.py +1122 -0
- docvortex/analyzers/native/pdf/table_recovery/text.py +414 -0
- docvortex/analyzers/native/pdf/table_recovery/text_grid.py +618 -0
- docvortex/analyzers/native/pdf/table_recovery/vector.py +1931 -0
- docvortex/analyzers/native/pdf/table_rows.py +34 -0
- docvortex/analyzers/native/pdf/table_rules.py +1129 -0
- docvortex/analyzers/native/pdf/table_text_styles.py +283 -0
- docvortex/analyzers/native/pdf/tables.py +147 -0
- docvortex/analyzers/native/pdf/text_assembly/__init__.py +3 -0
- docvortex/analyzers/native/pdf/text_assembly/annotations.py +581 -0
- docvortex/analyzers/native/pdf/text_assembly/assembly.py +292 -0
- docvortex/analyzers/native/pdf/text_assembly/common.py +477 -0
- docvortex/analyzers/native/pdf/text_assembly/footnotes.py +394 -0
- docvortex/analyzers/native/pdf/text_assembly/merging.py +1274 -0
- docvortex/analyzers/native/pdf/text_assembly/rows.py +692 -0
- docvortex/analyzers/native/pdf/text_blocks.py +82 -0
- docvortex/analyzers/native/pdf/text_styles.py +55 -0
- docvortex/analyzers/native/pdf/title_analysis/__init__.py +3 -0
- docvortex/analyzers/native/pdf/title_analysis/body_profile.py +215 -0
- docvortex/analyzers/native/pdf/title_analysis/common.py +117 -0
- docvortex/analyzers/native/pdf/title_analysis/document_profile.py +164 -0
- docvortex/analyzers/native/pdf/title_analysis/lane_titles.py +758 -0
- docvortex/analyzers/native/pdf/title_analysis/page_titles.py +1024 -0
- docvortex/analyzers/native/pdf/title_analysis/prototype.py +194 -0
- docvortex/analyzers/native/pdf/title_analysis/structural.py +1081 -0
- docvortex/analyzers/native/pdf/titles.py +75 -0
- docvortex/analyzers/native/pdf/typography.py +19 -0
- docvortex/analyzers/native/pdf/visual_annotations.py +1262 -0
- docvortex/api.py +180 -0
- docvortex/assets/__init__.py +5 -0
- docvortex/assets/store.py +51 -0
- docvortex/cli.py +54 -0
- docvortex/codecs/__init__.py +3 -0
- docvortex/codecs/html/__init__.py +22 -0
- docvortex/codecs/html/contracts.py +236 -0
- docvortex/codecs/html/materializer.py +331 -0
- docvortex/codecs/html/parser.py +763 -0
- docvortex/codecs/html/resources.py +34 -0
- docvortex/codecs/json.py +17 -0
- docvortex/content/__init__.py +5 -0
- docvortex/content/inline.py +248 -0
- docvortex/content/markup/__init__.py +44 -0
- docvortex/content/markup/anchors.py +188 -0
- docvortex/content/markup/formula.py +280 -0
- docvortex/content/markup/projector.py +1237 -0
- docvortex/content/markup/styles.py +327 -0
- docvortex/content/mathml.py +167 -0
- docvortex/content/normalization.py +188 -0
- docvortex/content/spans.py +183 -0
- docvortex/content/table/__init__.py +18 -0
- docvortex/content/table/blocks.py +152 -0
- docvortex/content/table/content.py +425 -0
- docvortex/content/table/document.py +104 -0
- docvortex/content/table/html.py +399 -0
- docvortex/content/table/models.py +76 -0
- docvortex/content/table/rules.py +42 -0
- docvortex/content/table/structure.py +221 -0
- docvortex/content/tree.py +5 -0
- docvortex/document/__init__.py +3 -0
- docvortex/document/contracts.py +22 -0
- docvortex/document/detection.py +389 -0
- docvortex/document/filetypes.py +175 -0
- docvortex/document/page_range.py +167 -0
- docvortex/document/pdf/__init__.py +19 -0
- docvortex/document/pdf/classify.py +1138 -0
- docvortex/document/pdf/constants.py +132 -0
- docvortex/document/pdf/diagnostics.py +350 -0
- docvortex/document/pdf/document.py +582 -0
- docvortex/document/pdf/font_runtime.py +335 -0
- docvortex/document/pdf/geometry.py +31 -0
- docvortex/document/pdf/images.py +654 -0
- docvortex/document/pdf/native_annotations.py +367 -0
- docvortex/document/pdf/native_contracts.py +169 -0
- docvortex/document/pdf/native_coordinates.py +216 -0
- docvortex/document/pdf/native_lifecycle.py +16 -0
- docvortex/document/pdf/native_objects.py +902 -0
- docvortex/document/pdf/native_text_geometry.py +315 -0
- docvortex/document/pdf/pdfium.py +325 -0
- docvortex/document/pdf/raster.py +46 -0
- docvortex/document/pdf/text/__init__.py +62 -0
- docvortex/document/pdf/text/contracts.py +211 -0
- docvortex/document/pdf/text/extract.py +165 -0
- docvortex/document/pdf/text/geometry.py +16 -0
- docvortex/document/pdf/text/groups.py +162 -0
- docvortex/document/pdf/visual_geometry.py +201 -0
- docvortex/document/pdf/visuals.py +343 -0
- docvortex/document/source.py +92 -0
- docvortex/errors.py +21 -0
- docvortex/export/__init__.py +3 -0
- docvortex/export/bundle.py +98 -0
- docvortex/export/files.py +66 -0
- docvortex/export/middle.py +208 -0
- docvortex/foundation/__init__.py +3 -0
- docvortex/foundation/geometry.py +125 -0
- docvortex/foundation/hyperlink.py +65 -0
- docvortex/foundation/image.py +48 -0
- docvortex/foundation/image_encoding.py +30 -0
- docvortex/foundation/image_payload.py +280 -0
- docvortex/foundation/language.py +92 -0
- docvortex/foundation/platform.py +38 -0
- docvortex/foundation/text.py +153 -0
- docvortex/foundation/type_identity.py +20 -0
- docvortex/foundation/xml_names.py +20 -0
- docvortex/options.py +30 -0
- docvortex/postprocess/__init__.py +3 -0
- docvortex/postprocess/content.py +53 -0
- docvortex/postprocess/document.py +19 -0
- docvortex/postprocess/lists.py +236 -0
- docvortex/postprocess/page_blocks.py +214 -0
- docvortex/postprocess/pages.py +95 -0
- docvortex/postprocess/paragraphs.py +580 -0
- docvortex/postprocess/visual.py +715 -0
- docvortex/render/__init__.py +48 -0
- docvortex/render/_internal/__init__.py +3 -0
- docvortex/render/_internal/common/__init__.py +3 -0
- docvortex/render/_internal/common/context.py +43 -0
- docvortex/render/_internal/common/html_table.py +178 -0
- docvortex/render/_internal/common/index.py +33 -0
- docvortex/render/_internal/common/list_items.py +158 -0
- docvortex/render/_internal/common/planner.py +140 -0
- docvortex/render/_internal/docx/__init__.py +3 -0
- docvortex/render/_internal/docx/assets.py +202 -0
- docvortex/render/_internal/docx/inline.py +434 -0
- docvortex/render/_internal/docx/math.py +220 -0
- docvortex/render/_internal/docx/renderer.py +905 -0
- docvortex/render/_internal/docx/styles.py +195 -0
- docvortex/render/_internal/docx/table.py +442 -0
- docvortex/render/_internal/epub/__init__.py +5 -0
- docvortex/render/_internal/epub/assets.py +173 -0
- docvortex/render/_internal/epub/package.py +249 -0
- docvortex/render/_internal/epub/renderer.py +1156 -0
- docvortex/render/_internal/html/__init__.py +3 -0
- docvortex/render/_internal/html/inline.py +346 -0
- docvortex/render/_internal/html/renderer.py +1041 -0
- docvortex/render/_internal/html/sanitizer.py +478 -0
- docvortex/render/_internal/html/table.py +121 -0
- docvortex/render/_internal/latex/__init__.py +1 -0
- docvortex/render/_internal/latex/assets.py +85 -0
- docvortex/render/_internal/latex/inline.py +145 -0
- docvortex/render/_internal/latex/renderer.py +506 -0
- docvortex/render/_internal/latex/table.py +347 -0
- docvortex/render/_internal/markdown/__init__.py +3 -0
- docvortex/render/_internal/markdown/assets.py +78 -0
- docvortex/render/_internal/markdown/blocks.py +635 -0
- docvortex/render/_internal/markdown/escaping.py +51 -0
- docvortex/render/_internal/markdown/inline.py +260 -0
- docvortex/render/_internal/markdown/renderer.py +93 -0
- docvortex/render/_internal/markdown/table.py +281 -0
- docvortex/render/_internal/pdf/__init__.py +3 -0
- docvortex/render/_internal/pdf/assets.py +197 -0
- docvortex/render/_internal/pdf/formula.py +417 -0
- docvortex/render/_internal/pdf/inline.py +343 -0
- docvortex/render/_internal/pdf/renderer.py +734 -0
- docvortex/render/_internal/pdf/styles.py +206 -0
- docvortex/render/_internal/pdf/table.py +272 -0
- docvortex/render/_internal/structured_content/__init__.py +3 -0
- docvortex/render/_internal/structured_content/renderer.py +193 -0
- docvortex/render/api.py +199 -0
- docvortex/render/contracts.py +205 -0
- docvortex/render/docx.py +38 -0
- docvortex/render/epub.py +35 -0
- docvortex/render/fragments.py +70 -0
- docvortex/render/html.py +29 -0
- docvortex/render/latex.py +24 -0
- docvortex/render/markdown.py +50 -0
- docvortex/render/pdf.py +25 -0
- docvortex/render/structured_content.py +23 -0
- docvortex/resources/epub/docvortex.css +91 -0
- docvortex/resources/fasttext-langdetect/lid.176.ftz +0 -0
- docvortex/resources/fonts/DroidSansFallbackFull.ttf +0 -0
- docvortex/resources/fonts/NOTICE +190 -0
- docvortex/resources/fonts/manifest.json +9 -0
- docvortex/resources/html/docvortex.css +601 -0
- docvortex/resources/html/docvortex.min.css +1 -0
- docvortex/result.py +91 -0
- docvortex/schema.py +1141 -0
- docvortex/version.py +3 -0
- docvortex-0.2.1.dist-info/METADATA +193 -0
- docvortex-0.2.1.dist-info/RECORD +364 -0
- docvortex-0.2.1.dist-info/WHEEL +5 -0
- docvortex-0.2.1.dist-info/entry_points.txt +2 -0
- docvortex-0.2.1.dist-info/licenses/LICENSE.md +21 -0
- docvortex-0.2.1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,1671 @@
|
|
|
1
|
+
"""分类页眉、页脚、页码、侧栏和页脚注。"""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
import statistics
|
|
7
|
+
import unicodedata
|
|
8
|
+
from difflib import SequenceMatcher
|
|
9
|
+
from typing import Literal
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
from ....schema import BBox
|
|
13
|
+
|
|
14
|
+
from .models import _AxisLine, _LineItem, _LocalAxisLine, _MarginalCandidate, _PageSource, _PreparedPage, _TextLane
|
|
15
|
+
from .geometry import (
|
|
16
|
+
_bbox_axis_overlap_ratio,
|
|
17
|
+
_bbox_center_x,
|
|
18
|
+
_bbox_center_y,
|
|
19
|
+
_bbox_intersects,
|
|
20
|
+
_bbox_union_many,
|
|
21
|
+
_clip_bbox,
|
|
22
|
+
_coerce_bbox,
|
|
23
|
+
_expand_bbox,
|
|
24
|
+
_horizontal_bbox_gap,
|
|
25
|
+
_rotate_bbox_to_upright,
|
|
26
|
+
_transform_axis_lines,
|
|
27
|
+
)
|
|
28
|
+
from .line_layout import _effective_text_row_gap, _infer_text_lanes, _line_effective_height
|
|
29
|
+
|
|
30
|
+
_PAGE_NUMBER_RE = re.compile(
|
|
31
|
+
r"^\s*(?:page\s*)?[\-\u2013\u2014\u00b7\u2022]*\s*(?:\u7b2c\s*)?"
|
|
32
|
+
r"(?P<value>\d{1,4}|[ivxlcdm]+|[\u3007\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e\u4e24]+)"
|
|
33
|
+
r"(?:\s*(?:/|of|\u5171)\s*(?:\d{1,4}|[ivxlcdm]+|[\u3007\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e\u4e24]+))?"
|
|
34
|
+
r"\s*(?:\u9875)?\s*[\-\u2013\u2014\u00b7\u2022]*\s*$",
|
|
35
|
+
re.IGNORECASE,
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _classify_page_auxiliary_text(prepared: _PreparedPage) -> None:
|
|
40
|
+
"""在容器认领后仅按空间关系标注侧栏文字和页脚注。"""
|
|
41
|
+
|
|
42
|
+
_classify_aside_text(prepared.remaining_lines, prepared.page_size)
|
|
43
|
+
_classify_image_footnotes(
|
|
44
|
+
prepared.remaining_lines,
|
|
45
|
+
[block["bbox"] for block in prepared.fixed_blocks if block.get("type") == "image"],
|
|
46
|
+
prepared.table_bboxes,
|
|
47
|
+
prepared.drawing_lines,
|
|
48
|
+
prepared.page_size,
|
|
49
|
+
)
|
|
50
|
+
prepared.page_footnote_groups = _classify_page_footnotes(
|
|
51
|
+
prepared.remaining_lines,
|
|
52
|
+
prepared.table_bboxes,
|
|
53
|
+
prepared.drawing_lines,
|
|
54
|
+
prepared.page_size,
|
|
55
|
+
visual_bboxes=[block["bbox"] for block in prepared.fixed_blocks if block.get("type") == "image"],
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _classify_aside_text(
|
|
60
|
+
lines: list[_LineItem],
|
|
61
|
+
page_size: tuple[float, float],
|
|
62
|
+
) -> None:
|
|
63
|
+
"""在横排正文占绝对多数时,以边缘带和物理尺寸识别垂直侧栏。"""
|
|
64
|
+
|
|
65
|
+
available = [line for line in lines if line.semantic_type is None]
|
|
66
|
+
upright_lines = [line for line in available if line.angle == 0]
|
|
67
|
+
if len(upright_lines) < 4:
|
|
68
|
+
return
|
|
69
|
+
|
|
70
|
+
support_by_angle = _geometric_text_support_by_angle(available, page_size)
|
|
71
|
+
total_support = sum(support_by_angle.values())
|
|
72
|
+
if total_support <= 0 or support_by_angle.get(0, 0.0) / total_support < 0.8:
|
|
73
|
+
return
|
|
74
|
+
|
|
75
|
+
page_width, page_height = page_size
|
|
76
|
+
if page_width <= 0 or page_height <= 0:
|
|
77
|
+
return
|
|
78
|
+
# 侧栏必须完整位于 12% 边缘带,且兼具不超过 8% 的窄宽和至少 15% 的物理高度。
|
|
79
|
+
aside_source_indices = {
|
|
80
|
+
line.source_index
|
|
81
|
+
for line in available
|
|
82
|
+
if line.angle in {90, 270}
|
|
83
|
+
and line.bbox[2] - line.bbox[0] <= 0.08 * page_width
|
|
84
|
+
and line.bbox[3] - line.bbox[1] >= 0.15 * page_height
|
|
85
|
+
and (line.bbox[2] <= 0.12 * page_width or line.bbox[0] >= 0.88 * page_width)
|
|
86
|
+
}
|
|
87
|
+
for line in available:
|
|
88
|
+
if line.source_index in aside_source_indices:
|
|
89
|
+
line.semantic_type = "aside_text"
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def _geometric_text_support_by_angle(
|
|
93
|
+
lines: list[_LineItem],
|
|
94
|
+
page_size: tuple[float, float],
|
|
95
|
+
) -> dict[int, float]:
|
|
96
|
+
"""按局部行宽乘有效行高累计各文字方向的纯几何支持度。"""
|
|
97
|
+
|
|
98
|
+
support_by_angle: dict[int, float] = {}
|
|
99
|
+
for line in lines:
|
|
100
|
+
local_bbox = _rotate_bbox_to_upright(line.bbox, page_size, line.angle)
|
|
101
|
+
local_width = max(0.1, local_bbox[2] - local_bbox[0])
|
|
102
|
+
support_by_angle[line.angle] = support_by_angle.get(line.angle, 0.0) + (
|
|
103
|
+
local_width * _line_effective_height(line, local_bbox)
|
|
104
|
+
)
|
|
105
|
+
return support_by_angle
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _classify_image_footnotes(
|
|
109
|
+
lines: list[_LineItem],
|
|
110
|
+
image_bboxes: list[BBox],
|
|
111
|
+
table_bboxes: list[BBox],
|
|
112
|
+
drawing_lines: list[_AxisLine],
|
|
113
|
+
page_size: tuple[float, float],
|
|
114
|
+
*,
|
|
115
|
+
reference_body_height: float | None = None,
|
|
116
|
+
) -> None:
|
|
117
|
+
"""用图片、下缘长横线和紧凑小字的联合关系识别图表脚注。"""
|
|
118
|
+
|
|
119
|
+
available = [line for line in lines if line.semantic_type is None]
|
|
120
|
+
if not available or not image_bboxes or not drawing_lines:
|
|
121
|
+
return
|
|
122
|
+
support_by_angle = _geometric_text_support_by_angle(available, page_size)
|
|
123
|
+
if not support_by_angle:
|
|
124
|
+
return
|
|
125
|
+
dominant_angle = max(
|
|
126
|
+
sorted(support_by_angle),
|
|
127
|
+
key=lambda angle: support_by_angle[angle],
|
|
128
|
+
)
|
|
129
|
+
local_page_size = (page_size[1], page_size[0]) if dominant_angle in {90, 270} else page_size
|
|
130
|
+
local_page_width, local_page_height = local_page_size
|
|
131
|
+
if local_page_width <= 0 or local_page_height <= 0:
|
|
132
|
+
return
|
|
133
|
+
|
|
134
|
+
line_geometry = sorted(
|
|
135
|
+
[
|
|
136
|
+
(line, _rotate_bbox_to_upright(line.bbox, page_size, dominant_angle))
|
|
137
|
+
for line in available
|
|
138
|
+
if line.angle == dominant_angle
|
|
139
|
+
],
|
|
140
|
+
key=lambda item: (item[1][1], item[1][0], item[0].source_index),
|
|
141
|
+
)
|
|
142
|
+
if not line_geometry:
|
|
143
|
+
return
|
|
144
|
+
if reference_body_height is not None and reference_body_height > 0:
|
|
145
|
+
# 图片占主导的稀疏页可能只剩图注和脚注,延迟复核时改用全文正文尺度。
|
|
146
|
+
body_height = max(0.1, reference_body_height)
|
|
147
|
+
else:
|
|
148
|
+
body_samples = [
|
|
149
|
+
_line_effective_height(line, bbox) for line, bbox in line_geometry if bbox[2] - bbox[0] >= 0.2 * local_page_width
|
|
150
|
+
]
|
|
151
|
+
if not body_samples:
|
|
152
|
+
body_samples = [_line_effective_height(line, bbox) for line, bbox in line_geometry]
|
|
153
|
+
body_height = max(0.1, statistics.median(body_samples))
|
|
154
|
+
local_images = [_rotate_bbox_to_upright(bbox, page_size, dominant_angle) for bbox in image_bboxes]
|
|
155
|
+
local_axis_lines = _transform_axis_lines(
|
|
156
|
+
drawing_lines,
|
|
157
|
+
page_size,
|
|
158
|
+
dominant_angle,
|
|
159
|
+
)
|
|
160
|
+
|
|
161
|
+
matched_source_indices: set[int] = set()
|
|
162
|
+
for image_bbox in local_images:
|
|
163
|
+
image_width = max(0.1, image_bbox[2] - image_bbox[0])
|
|
164
|
+
# 同一视觉行的并排图可能高度略有差异;共享较低下缘可避免把留白误作远距。
|
|
165
|
+
row_bottom = max(
|
|
166
|
+
peer_bbox[3] for peer_bbox in local_images if _bbox_axis_overlap_ratio(image_bbox, peer_bbox, axis="y") >= 0.5
|
|
167
|
+
)
|
|
168
|
+
candidate_rules = [
|
|
169
|
+
axis_line
|
|
170
|
+
for axis_line in local_axis_lines
|
|
171
|
+
if axis_line.orientation == "horizontal"
|
|
172
|
+
and 0.75 * image_width <= axis_line.bbox[2] - axis_line.bbox[0] <= 1.3 * image_width
|
|
173
|
+
and max(
|
|
174
|
+
0.0,
|
|
175
|
+
min(axis_line.bbox[2], image_bbox[2]) - max(axis_line.bbox[0], image_bbox[0]),
|
|
176
|
+
)
|
|
177
|
+
>= 0.85 * image_width
|
|
178
|
+
# 图片外框的底边属于图形本身,不能拿来证明下方文字是图表脚注。
|
|
179
|
+
and 0.0 <= axis_line.bbox[1] - row_bottom <= max(0.01 * local_page_height, 0.75 * body_height)
|
|
180
|
+
and not _rule_belongs_to_confirmed_table(
|
|
181
|
+
axis_line,
|
|
182
|
+
local_axis_lines,
|
|
183
|
+
table_bboxes,
|
|
184
|
+
local_page_width,
|
|
185
|
+
)
|
|
186
|
+
]
|
|
187
|
+
if not candidate_rules:
|
|
188
|
+
continue
|
|
189
|
+
rule = min(
|
|
190
|
+
candidate_rules,
|
|
191
|
+
key=lambda item: (max(0.0, item.bbox[1] - row_bottom), item.bbox[1]),
|
|
192
|
+
)
|
|
193
|
+
matched_source_indices.update(
|
|
194
|
+
_image_footnote_members(
|
|
195
|
+
line_geometry,
|
|
196
|
+
rule.bbox,
|
|
197
|
+
body_height,
|
|
198
|
+
local_page_height,
|
|
199
|
+
)
|
|
200
|
+
)
|
|
201
|
+
|
|
202
|
+
for line in available:
|
|
203
|
+
if line.source_index in matched_source_indices:
|
|
204
|
+
line.semantic_type = "footnote"
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def _classify_deferred_image_footnotes(
|
|
208
|
+
prepared_pages: list[_PreparedPage],
|
|
209
|
+
body_height: float,
|
|
210
|
+
) -> None:
|
|
211
|
+
"""在全文正文尺度确定后,仅重试仍未分类的图片脚注候选。"""
|
|
212
|
+
|
|
213
|
+
if body_height <= 0:
|
|
214
|
+
return
|
|
215
|
+
for prepared in prepared_pages:
|
|
216
|
+
_classify_image_footnotes(
|
|
217
|
+
prepared.remaining_lines,
|
|
218
|
+
[block["bbox"] for block in prepared.fixed_blocks if block.get("type") == "image"],
|
|
219
|
+
prepared.table_bboxes,
|
|
220
|
+
prepared.drawing_lines,
|
|
221
|
+
prepared.page_size,
|
|
222
|
+
reference_body_height=body_height,
|
|
223
|
+
)
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def _image_footnote_members(
|
|
227
|
+
line_geometry: list[tuple[_LineItem, BBox]],
|
|
228
|
+
rule_bbox: BBox,
|
|
229
|
+
body_height: float,
|
|
230
|
+
local_page_height: float,
|
|
231
|
+
) -> set[int]:
|
|
232
|
+
"""返回长横线下方、位于同一水平走廊内的连续小字号文本行。"""
|
|
233
|
+
|
|
234
|
+
first_gap_limit = max(0.025 * local_page_height, 2.0 * body_height)
|
|
235
|
+
horizontal_tolerance = 0.5 * body_height
|
|
236
|
+
candidates = [
|
|
237
|
+
item
|
|
238
|
+
for item in line_geometry
|
|
239
|
+
if -0.25 * body_height <= item[1][1] - rule_bbox[3] <= first_gap_limit
|
|
240
|
+
and item[1][0] >= rule_bbox[0] - horizontal_tolerance
|
|
241
|
+
and item[1][2] <= rule_bbox[2] + horizontal_tolerance
|
|
242
|
+
and _line_effective_height(*item) <= 0.9 * body_height
|
|
243
|
+
]
|
|
244
|
+
if not candidates:
|
|
245
|
+
return set()
|
|
246
|
+
first = min(candidates, key=lambda item: (item[1][1], item[1][0]))
|
|
247
|
+
members = [first]
|
|
248
|
+
continuation_gap_limit = max(1.25 * _line_effective_height(*first), 0.01 * local_page_height)
|
|
249
|
+
for current in line_geometry:
|
|
250
|
+
if current[0] is first[0] or current[1][1] < first[1][1]:
|
|
251
|
+
continue
|
|
252
|
+
if current[1][0] < rule_bbox[0] - horizontal_tolerance:
|
|
253
|
+
continue
|
|
254
|
+
if current[1][2] > rule_bbox[2] + horizontal_tolerance:
|
|
255
|
+
continue
|
|
256
|
+
if _line_effective_height(*current) > 0.95 * body_height:
|
|
257
|
+
continue
|
|
258
|
+
if _effective_text_row_gap(members[-1], current) > continuation_gap_limit:
|
|
259
|
+
break
|
|
260
|
+
members.append(current)
|
|
261
|
+
return {line.source_index for line, _bbox in members}
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def _classify_page_footnotes(
|
|
265
|
+
lines: list[_LineItem],
|
|
266
|
+
table_bboxes: list[BBox],
|
|
267
|
+
drawing_lines: list[_AxisLine],
|
|
268
|
+
page_size: tuple[float, float],
|
|
269
|
+
*,
|
|
270
|
+
visual_bboxes: list[BBox] | None = None,
|
|
271
|
+
) -> list[set[int]]:
|
|
272
|
+
"""识别主方向页脚注,并按触发分隔线返回来源编号分组。"""
|
|
273
|
+
|
|
274
|
+
available = [line for line in lines if line.semantic_type is None]
|
|
275
|
+
if not available or not drawing_lines:
|
|
276
|
+
return []
|
|
277
|
+
support_by_angle = _geometric_text_support_by_angle(available, page_size)
|
|
278
|
+
if not support_by_angle:
|
|
279
|
+
return []
|
|
280
|
+
dominant_angle = max(
|
|
281
|
+
sorted(support_by_angle),
|
|
282
|
+
key=lambda angle: support_by_angle[angle],
|
|
283
|
+
)
|
|
284
|
+
line_geometry = [
|
|
285
|
+
(line, _rotate_bbox_to_upright(line.bbox, page_size, dominant_angle))
|
|
286
|
+
for line in available
|
|
287
|
+
if line.angle == dominant_angle
|
|
288
|
+
]
|
|
289
|
+
if not line_geometry:
|
|
290
|
+
return []
|
|
291
|
+
|
|
292
|
+
local_page_size = (page_size[1], page_size[0]) if dominant_angle in {90, 270} else page_size
|
|
293
|
+
local_page_width, local_page_height = local_page_size
|
|
294
|
+
if local_page_width <= 0 or local_page_height <= 0:
|
|
295
|
+
return []
|
|
296
|
+
effective_heights = [_line_effective_height(line, bbox) for line, bbox in line_geometry]
|
|
297
|
+
median_height = statistics.median(effective_heights) if effective_heights else 1.0
|
|
298
|
+
lanes = _infer_text_lanes(
|
|
299
|
+
line_geometry,
|
|
300
|
+
local_page_width,
|
|
301
|
+
median_height,
|
|
302
|
+
# 脚注分隔线应对齐稳定栏锚点,不能被页眉或跨栏关键词的宽行扩张污染。
|
|
303
|
+
recalculate_intervals=False,
|
|
304
|
+
)
|
|
305
|
+
local_axis_lines = _transform_axis_lines(
|
|
306
|
+
drawing_lines,
|
|
307
|
+
page_size,
|
|
308
|
+
dominant_angle,
|
|
309
|
+
)
|
|
310
|
+
|
|
311
|
+
candidate_groups: list[set[int]] = []
|
|
312
|
+
visual_bboxes = visual_bboxes or []
|
|
313
|
+
for axis_line in local_axis_lines:
|
|
314
|
+
if axis_line.orientation != "horizontal":
|
|
315
|
+
continue
|
|
316
|
+
# 常规短分隔线仍要求进入页面下方 30%;栏宽分隔线可在下方 45% 内
|
|
317
|
+
# 依靠严格的单栏对齐和字号收缩证据提前触发。
|
|
318
|
+
rule_center_y = _bbox_center_y(axis_line.bbox)
|
|
319
|
+
if rule_center_y < 0.55 * local_page_height:
|
|
320
|
+
continue
|
|
321
|
+
# 表格边界会产生断裂横线;除框内线段外,也排除与其同高且近邻的框外线段。
|
|
322
|
+
if _rule_belongs_to_confirmed_table(
|
|
323
|
+
axis_line,
|
|
324
|
+
local_axis_lines,
|
|
325
|
+
table_bboxes,
|
|
326
|
+
local_page_width,
|
|
327
|
+
):
|
|
328
|
+
continue
|
|
329
|
+
if any(
|
|
330
|
+
_bbox_intersects(
|
|
331
|
+
_expand_bbox(axis_line.original_bbox, max(0.5, axis_line.width)),
|
|
332
|
+
visual_bbox,
|
|
333
|
+
)
|
|
334
|
+
for visual_bbox in visual_bboxes
|
|
335
|
+
):
|
|
336
|
+
# 图形坐标轴和外框不能充当页面脚注分隔线。
|
|
337
|
+
continue
|
|
338
|
+
rule_source_indices: set[int] = set()
|
|
339
|
+
for lane in lanes:
|
|
340
|
+
following_rule_tops = [
|
|
341
|
+
other.bbox[1]
|
|
342
|
+
for other in local_axis_lines
|
|
343
|
+
if other.orientation == "horizontal"
|
|
344
|
+
and other.bbox[1] - axis_line.bbox[3] > 0.5 * median_height
|
|
345
|
+
and _bbox_axis_overlap_ratio(
|
|
346
|
+
axis_line.bbox,
|
|
347
|
+
other.bbox,
|
|
348
|
+
axis="x",
|
|
349
|
+
)
|
|
350
|
+
>= 0.8
|
|
351
|
+
]
|
|
352
|
+
rule_source_indices.update(
|
|
353
|
+
_footnote_lane_members(
|
|
354
|
+
lane,
|
|
355
|
+
axis_line.bbox,
|
|
356
|
+
local_page_size,
|
|
357
|
+
page_median_height=median_height,
|
|
358
|
+
lane_width_reference=_footnote_lane_width_reference(
|
|
359
|
+
lane,
|
|
360
|
+
lanes,
|
|
361
|
+
median_height,
|
|
362
|
+
),
|
|
363
|
+
allow_column_width_rule=(rule_center_y >= 0.55 * local_page_height),
|
|
364
|
+
lower_barrier_y=(min(following_rule_tops) if following_rule_tops else None),
|
|
365
|
+
)
|
|
366
|
+
)
|
|
367
|
+
if rule_source_indices:
|
|
368
|
+
candidate_groups.append(rule_source_indices)
|
|
369
|
+
|
|
370
|
+
page_footnote_groups = _merge_overlapping_source_groups(candidate_groups)
|
|
371
|
+
_augment_footnote_groups_with_edge_markers(
|
|
372
|
+
page_footnote_groups,
|
|
373
|
+
line_geometry,
|
|
374
|
+
median_height,
|
|
375
|
+
)
|
|
376
|
+
footnote_source_indices = set().union(*page_footnote_groups) if page_footnote_groups else set()
|
|
377
|
+
for line in available:
|
|
378
|
+
if line.source_index in footnote_source_indices:
|
|
379
|
+
line.semantic_type = "page_footnote"
|
|
380
|
+
return page_footnote_groups
|
|
381
|
+
|
|
382
|
+
|
|
383
|
+
def _augment_footnote_groups_with_edge_markers(
|
|
384
|
+
groups: list[set[int]],
|
|
385
|
+
line_geometry: list[tuple[_LineItem, BBox]],
|
|
386
|
+
median_height: float,
|
|
387
|
+
) -> None:
|
|
388
|
+
"""把脚注正文左侧同高的窄编号标记补入对应分隔线分组。"""
|
|
389
|
+
|
|
390
|
+
geometry_by_source = {line.source_index: (line, bbox) for line, bbox in line_geometry}
|
|
391
|
+
for group in groups:
|
|
392
|
+
members = [geometry_by_source[source_index] for source_index in group if source_index in geometry_by_source]
|
|
393
|
+
if not members:
|
|
394
|
+
continue
|
|
395
|
+
group_top = min(bbox[1] for _line, bbox in members)
|
|
396
|
+
group_bottom = max(bbox[3] for _line, bbox in members)
|
|
397
|
+
content_left = min(bbox[0] for _line, bbox in members)
|
|
398
|
+
for line, bbox in line_geometry:
|
|
399
|
+
if line.source_index in group:
|
|
400
|
+
continue
|
|
401
|
+
line_width = bbox[2] - bbox[0]
|
|
402
|
+
center_y = _bbox_center_y(bbox)
|
|
403
|
+
if (
|
|
404
|
+
line_width <= 1.5 * median_height
|
|
405
|
+
and content_left - 2.0 * median_height <= bbox[0] <= content_left
|
|
406
|
+
and bbox[2] <= content_left + 0.5 * median_height
|
|
407
|
+
and group_top - median_height <= center_y <= group_bottom + median_height
|
|
408
|
+
):
|
|
409
|
+
group.add(line.source_index)
|
|
410
|
+
|
|
411
|
+
|
|
412
|
+
def _rule_belongs_to_confirmed_table(
|
|
413
|
+
candidate: _LocalAxisLine,
|
|
414
|
+
local_axis_lines: list[_LocalAxisLine],
|
|
415
|
+
table_bboxes: list[BBox],
|
|
416
|
+
local_page_width: float,
|
|
417
|
+
) -> bool:
|
|
418
|
+
"""把表格框内横线及其同高近邻断裂段一并排除,避免框外残段触发脚注。"""
|
|
419
|
+
|
|
420
|
+
if not table_bboxes:
|
|
421
|
+
return False
|
|
422
|
+
maximum_segment_gap = 0.04 * local_page_width
|
|
423
|
+
for table_line in local_axis_lines:
|
|
424
|
+
if table_line.orientation != "horizontal":
|
|
425
|
+
continue
|
|
426
|
+
table_margin = max(0.5, table_line.width)
|
|
427
|
+
if not any(
|
|
428
|
+
_bbox_intersects(
|
|
429
|
+
_expand_bbox(table_line.original_bbox, table_margin),
|
|
430
|
+
table_bbox,
|
|
431
|
+
)
|
|
432
|
+
for table_bbox in table_bboxes
|
|
433
|
+
):
|
|
434
|
+
continue
|
|
435
|
+
center_tolerance = max(1.0, candidate.width, table_line.width)
|
|
436
|
+
if abs(_bbox_center_y(candidate.bbox) - _bbox_center_y(table_line.bbox)) > center_tolerance:
|
|
437
|
+
continue
|
|
438
|
+
if _horizontal_bbox_gap(candidate.bbox, table_line.bbox) <= maximum_segment_gap:
|
|
439
|
+
return True
|
|
440
|
+
return False
|
|
441
|
+
|
|
442
|
+
|
|
443
|
+
def _merge_overlapping_source_groups(groups: list[set[int]]) -> list[set[int]]:
|
|
444
|
+
"""合并共享来源行的分隔线候选组,消除重复绘图线造成的重复分组。"""
|
|
445
|
+
|
|
446
|
+
merged: list[set[int]] = []
|
|
447
|
+
for group in groups:
|
|
448
|
+
combined = set(group)
|
|
449
|
+
index = 0
|
|
450
|
+
while index < len(merged):
|
|
451
|
+
if combined & merged[index]:
|
|
452
|
+
combined.update(merged.pop(index))
|
|
453
|
+
index = 0
|
|
454
|
+
continue
|
|
455
|
+
index += 1
|
|
456
|
+
merged.append(combined)
|
|
457
|
+
return sorted(merged, key=lambda group: min(group))
|
|
458
|
+
|
|
459
|
+
|
|
460
|
+
def _footnote_lane_members(
|
|
461
|
+
lane: _TextLane,
|
|
462
|
+
rule_bbox: BBox,
|
|
463
|
+
local_page_size: tuple[float, float],
|
|
464
|
+
*,
|
|
465
|
+
page_median_height: float | None = None,
|
|
466
|
+
lane_width_reference: float | None = None,
|
|
467
|
+
allow_column_width_rule: bool = False,
|
|
468
|
+
lower_barrier_y: float | None = None,
|
|
469
|
+
) -> set[int]:
|
|
470
|
+
"""验证横线与单个栏带的对齐关系,并返回其下连续脚注行的来源编号。"""
|
|
471
|
+
|
|
472
|
+
lane_lines = [item for item in lane.lines if item[0].semantic_type is None]
|
|
473
|
+
if not lane_lines:
|
|
474
|
+
return set()
|
|
475
|
+
lane_lines.sort(key=lambda item: (item[1][1], item[1][0], item[0].source_index))
|
|
476
|
+
local_page_width, local_page_height = local_page_size
|
|
477
|
+
lane_width = max(
|
|
478
|
+
0.1,
|
|
479
|
+
lane.right - lane.left,
|
|
480
|
+
lane_width_reference or 0.0,
|
|
481
|
+
)
|
|
482
|
+
lane_heights = [_line_effective_height(line, bbox) for line, bbox in lane_lines]
|
|
483
|
+
median_height = statistics.median(lane_heights) if lane_heights else 1.0
|
|
484
|
+
rule_width = max(0.0, rule_bbox[2] - rule_bbox[0])
|
|
485
|
+
# 同时限制绝对短线、相对长线和左缘偏移,排除图标、公式线及跨栏正文分隔线。
|
|
486
|
+
if rule_width < max(4.0 * median_height, 0.04 * local_page_width):
|
|
487
|
+
return set()
|
|
488
|
+
strict_left_tolerance = max(
|
|
489
|
+
2.0 * median_height,
|
|
490
|
+
0.04 * lane_width,
|
|
491
|
+
)
|
|
492
|
+
strict_left_alignment = abs(rule_bbox[0] - lane.left) <= strict_left_tolerance
|
|
493
|
+
relaxed_left_alignment = rule_bbox[0] < lane.left and lane.left - rule_bbox[0] <= 2.25 * median_height
|
|
494
|
+
rule_center_y = _bbox_center_y(rule_bbox)
|
|
495
|
+
centered_short_alignment = (
|
|
496
|
+
not lane.is_span
|
|
497
|
+
and rule_center_y >= 0.7 * local_page_height
|
|
498
|
+
and 0.35 * lane_width <= rule_width <= 0.7 * lane_width
|
|
499
|
+
and abs(_bbox_center_x(rule_bbox) - 0.5 * (lane.left + lane.right)) <= 0.08 * lane_width
|
|
500
|
+
)
|
|
501
|
+
if not strict_left_alignment and not relaxed_left_alignment and not centered_short_alignment:
|
|
502
|
+
return set()
|
|
503
|
+
|
|
504
|
+
is_regular_short_rule = rule_center_y >= 0.7 * local_page_height and rule_width <= 0.65 * lane_width
|
|
505
|
+
endpoint_tolerance = max(2.0 * median_height, 0.05 * lane_width)
|
|
506
|
+
is_column_width_rule = (
|
|
507
|
+
allow_column_width_rule
|
|
508
|
+
and not lane.is_span
|
|
509
|
+
and 0.65 * lane_width <= rule_width <= 1.05 * lane_width
|
|
510
|
+
and abs(rule_bbox[2] - lane.right) <= endpoint_tolerance
|
|
511
|
+
)
|
|
512
|
+
if not is_regular_short_rule and not is_column_width_rule and not centered_short_alignment:
|
|
513
|
+
return set()
|
|
514
|
+
|
|
515
|
+
# 首行采用较宽的 3.5% 页高窗口;命中后仅按紧凑的连续净空向下扩展。
|
|
516
|
+
first_gap_limit = max(3.0 * median_height, 0.035 * local_page_height)
|
|
517
|
+
first_index: int | None = None
|
|
518
|
+
for index, (_line, bbox) in enumerate(lane_lines):
|
|
519
|
+
rule_gap = bbox[1] - rule_bbox[3]
|
|
520
|
+
if rule_gap < -0.5 * median_height:
|
|
521
|
+
continue
|
|
522
|
+
if lower_barrier_y is not None and bbox[1] >= lower_barrier_y:
|
|
523
|
+
break
|
|
524
|
+
if rule_gap <= first_gap_limit:
|
|
525
|
+
first_index = index
|
|
526
|
+
break
|
|
527
|
+
if first_index is None:
|
|
528
|
+
return set()
|
|
529
|
+
|
|
530
|
+
if is_column_width_rule:
|
|
531
|
+
# 页面中段的栏宽横线只有在下方首行相对上方正文明显收缩时才可触发脚注,
|
|
532
|
+
# 避免把章节分隔线或普通栏内横线误当成脚注边界。
|
|
533
|
+
body_heights = [
|
|
534
|
+
_line_effective_height(line, bbox) for line, bbox in lane_lines if bbox[3] <= rule_bbox[1] + 0.5 * median_height
|
|
535
|
+
]
|
|
536
|
+
first_height = _line_effective_height(*lane_lines[first_index])
|
|
537
|
+
body_reference_height = statistics.median(body_heights) if body_heights else 0.0
|
|
538
|
+
if page_median_height is not None:
|
|
539
|
+
body_reference_height = max(
|
|
540
|
+
body_reference_height,
|
|
541
|
+
page_median_height,
|
|
542
|
+
)
|
|
543
|
+
if len(body_heights) < 3 or first_height > 0.95 * body_reference_height:
|
|
544
|
+
return set()
|
|
545
|
+
|
|
546
|
+
continuation_gap_limit = _page_footnote_continuation_gap_limit(
|
|
547
|
+
median_height,
|
|
548
|
+
local_page_height,
|
|
549
|
+
)
|
|
550
|
+
members = [lane_lines[first_index]]
|
|
551
|
+
for current in lane_lines[first_index + 1 :]:
|
|
552
|
+
if lower_barrier_y is not None and current[1][1] >= lower_barrier_y:
|
|
553
|
+
break
|
|
554
|
+
if _effective_text_row_gap(members[-1], current) > continuation_gap_limit:
|
|
555
|
+
break
|
|
556
|
+
members.append(current)
|
|
557
|
+
if relaxed_left_alignment and not strict_left_alignment:
|
|
558
|
+
first_bbox = members[0][1]
|
|
559
|
+
first_height = _line_effective_height(*members[0])
|
|
560
|
+
reference_height = max(
|
|
561
|
+
median_height,
|
|
562
|
+
page_median_height or 0.0,
|
|
563
|
+
)
|
|
564
|
+
horizontal_overlap = max(
|
|
565
|
+
0.0,
|
|
566
|
+
min(rule_bbox[2], first_bbox[2]) - max(rule_bbox[0], first_bbox[0]),
|
|
567
|
+
)
|
|
568
|
+
if len(members) < 2 or first_height > 0.9 * reference_height or horizontal_overlap / max(0.1, rule_width) < 0.8:
|
|
569
|
+
return set()
|
|
570
|
+
if centered_short_alignment:
|
|
571
|
+
reference_height = max(
|
|
572
|
+
median_height,
|
|
573
|
+
page_median_height or 0.0,
|
|
574
|
+
)
|
|
575
|
+
projecting_rows_above = []
|
|
576
|
+
for _line, bbox in lane_lines[:first_index]:
|
|
577
|
+
overlap = max(
|
|
578
|
+
0.0,
|
|
579
|
+
min(rule_bbox[2], bbox[2]) - max(rule_bbox[0], bbox[0]),
|
|
580
|
+
)
|
|
581
|
+
row_width = max(0.1, bbox[2] - bbox[0])
|
|
582
|
+
if bbox[1] < rule_bbox[1] and overlap >= 0.2 * min(rule_width, row_width):
|
|
583
|
+
projecting_rows_above.append(bbox)
|
|
584
|
+
if any(bbox[3] > rule_bbox[1] - 0.75 * reference_height for bbox in projecting_rows_above):
|
|
585
|
+
# 分式横线位于公式成员之间;真正的脚注分隔线上方应保留正文净空。
|
|
586
|
+
return set()
|
|
587
|
+
member_height = statistics.median(_line_effective_height(*member) for member in members)
|
|
588
|
+
if len(members) < 2 or member_height > 0.9 * reference_height:
|
|
589
|
+
return set()
|
|
590
|
+
return {line.source_index for line, _bbox in members}
|
|
591
|
+
|
|
592
|
+
|
|
593
|
+
def _page_footnote_continuation_gap_limit(
|
|
594
|
+
reference_height: float,
|
|
595
|
+
local_page_height: float,
|
|
596
|
+
) -> float:
|
|
597
|
+
"""统一返回页脚注连续扩展允许的最大有效净空。"""
|
|
598
|
+
|
|
599
|
+
return max(1.25 * reference_height, 0.01 * local_page_height)
|
|
600
|
+
|
|
601
|
+
|
|
602
|
+
def _footnote_lane_width_reference(
|
|
603
|
+
lane: _TextLane,
|
|
604
|
+
lanes: list[_TextLane],
|
|
605
|
+
median_height: float,
|
|
606
|
+
) -> float:
|
|
607
|
+
"""用下一稳定栏的左缘补偿当前栏因正文右缘参差造成的宽度低估。"""
|
|
608
|
+
|
|
609
|
+
lane_width = max(0.1, lane.right - lane.left)
|
|
610
|
+
stable_lanes = sorted(
|
|
611
|
+
[candidate for candidate in lanes if not candidate.is_span and len(candidate.lines) >= 3],
|
|
612
|
+
key=lambda candidate: candidate.left,
|
|
613
|
+
)
|
|
614
|
+
if lane not in stable_lanes:
|
|
615
|
+
return lane_width
|
|
616
|
+
lane_index = stable_lanes.index(lane)
|
|
617
|
+
if lane_index + 1 >= len(stable_lanes):
|
|
618
|
+
return lane_width
|
|
619
|
+
minimum_gutter = max(6.0, 0.75 * median_height)
|
|
620
|
+
next_lane = stable_lanes[lane_index + 1]
|
|
621
|
+
return max(
|
|
622
|
+
lane_width,
|
|
623
|
+
next_lane.left - lane.left - minimum_gutter,
|
|
624
|
+
)
|
|
625
|
+
|
|
626
|
+
|
|
627
|
+
def _classify_rule_delimited_headers(pages: list[_PreparedPage]) -> None:
|
|
628
|
+
"""在页码完成跨页判定后,用页首长横线补标其上方未分类文本。"""
|
|
629
|
+
|
|
630
|
+
for page in pages:
|
|
631
|
+
available = [line for line in page.remaining_lines if line.semantic_type is None]
|
|
632
|
+
if not available or not page.drawing_lines:
|
|
633
|
+
continue
|
|
634
|
+
support_by_angle = _geometric_text_support_by_angle(
|
|
635
|
+
page.remaining_lines,
|
|
636
|
+
page.page_size,
|
|
637
|
+
)
|
|
638
|
+
if not support_by_angle:
|
|
639
|
+
continue
|
|
640
|
+
dominant_angle = max(
|
|
641
|
+
sorted(support_by_angle),
|
|
642
|
+
key=lambda angle: support_by_angle[angle],
|
|
643
|
+
)
|
|
644
|
+
local_page_size = (page.page_size[1], page.page_size[0]) if dominant_angle in {90, 270} else page.page_size
|
|
645
|
+
local_page_width, local_page_height = local_page_size
|
|
646
|
+
if local_page_width <= 0 or local_page_height <= 0:
|
|
647
|
+
continue
|
|
648
|
+
local_lines = [
|
|
649
|
+
(
|
|
650
|
+
line,
|
|
651
|
+
_rotate_bbox_to_upright(
|
|
652
|
+
line.ink_bbox or line.bbox,
|
|
653
|
+
page.page_size,
|
|
654
|
+
dominant_angle,
|
|
655
|
+
),
|
|
656
|
+
)
|
|
657
|
+
for line in available
|
|
658
|
+
if line.angle == dominant_angle
|
|
659
|
+
]
|
|
660
|
+
header_evidence_bboxes = [
|
|
661
|
+
_rotate_bbox_to_upright(
|
|
662
|
+
line.ink_bbox or line.bbox,
|
|
663
|
+
page.page_size,
|
|
664
|
+
dominant_angle,
|
|
665
|
+
)
|
|
666
|
+
for line in page.remaining_lines
|
|
667
|
+
if line.angle == dominant_angle and line.semantic_type in {None, "header", "page_number"}
|
|
668
|
+
]
|
|
669
|
+
heights = [_line_effective_height(line, bbox) for line, bbox in local_lines]
|
|
670
|
+
median_height = statistics.median(heights) if heights else 1.0
|
|
671
|
+
local_axis_lines = _transform_axis_lines(
|
|
672
|
+
page.drawing_lines,
|
|
673
|
+
page.page_size,
|
|
674
|
+
dominant_angle,
|
|
675
|
+
)
|
|
676
|
+
candidates = [
|
|
677
|
+
axis_line
|
|
678
|
+
for axis_line in local_axis_lines
|
|
679
|
+
if axis_line.orientation == "horizontal"
|
|
680
|
+
and _bbox_center_y(axis_line.bbox) <= 0.15 * local_page_height
|
|
681
|
+
and axis_line.bbox[2] - axis_line.bbox[0] >= 0.6 * local_page_width
|
|
682
|
+
and any(bbox[3] <= _bbox_center_y(axis_line.bbox) for bbox in header_evidence_bboxes)
|
|
683
|
+
and not _rule_belongs_to_confirmed_table(
|
|
684
|
+
axis_line,
|
|
685
|
+
local_axis_lines,
|
|
686
|
+
page.table_bboxes,
|
|
687
|
+
local_page_width,
|
|
688
|
+
)
|
|
689
|
+
and not _rule_overlaps_fixed_container(
|
|
690
|
+
axis_line,
|
|
691
|
+
page.fixed_blocks,
|
|
692
|
+
page.page_size,
|
|
693
|
+
)
|
|
694
|
+
]
|
|
695
|
+
if not candidates:
|
|
696
|
+
continue
|
|
697
|
+
separator = min(candidates, key=lambda item: _bbox_center_y(item.bbox))
|
|
698
|
+
separator_y = _bbox_center_y(separator.bbox)
|
|
699
|
+
if not any(_bbox_center_y(bbox) >= separator_y + median_height for _line, bbox in local_lines):
|
|
700
|
+
continue
|
|
701
|
+
for line, bbox in local_lines:
|
|
702
|
+
if bbox[3] <= separator_y:
|
|
703
|
+
line.semantic_type = "header"
|
|
704
|
+
|
|
705
|
+
|
|
706
|
+
def _classify_rule_delimited_footers(pages: list[_PreparedPage]) -> None:
|
|
707
|
+
"""用页面底部横线确认双线间页脚或单线下方的小字号栏内页脚。"""
|
|
708
|
+
|
|
709
|
+
for page in pages:
|
|
710
|
+
available = [line for line in page.remaining_lines if line.semantic_type is None]
|
|
711
|
+
if not available or not page.drawing_lines:
|
|
712
|
+
continue
|
|
713
|
+
support_by_angle = _geometric_text_support_by_angle(
|
|
714
|
+
page.remaining_lines,
|
|
715
|
+
page.page_size,
|
|
716
|
+
)
|
|
717
|
+
if not support_by_angle:
|
|
718
|
+
continue
|
|
719
|
+
dominant_angle = max(
|
|
720
|
+
sorted(support_by_angle),
|
|
721
|
+
key=lambda angle: support_by_angle[angle],
|
|
722
|
+
)
|
|
723
|
+
local_page_size = (page.page_size[1], page.page_size[0]) if dominant_angle in {90, 270} else page.page_size
|
|
724
|
+
local_page_width, local_page_height = local_page_size
|
|
725
|
+
if local_page_width <= 0 or local_page_height <= 0:
|
|
726
|
+
continue
|
|
727
|
+
local_axis_lines = _transform_axis_lines(
|
|
728
|
+
page.drawing_lines,
|
|
729
|
+
page.page_size,
|
|
730
|
+
dominant_angle,
|
|
731
|
+
)
|
|
732
|
+
rules = [
|
|
733
|
+
rule
|
|
734
|
+
for rule in local_axis_lines
|
|
735
|
+
if rule.orientation == "horizontal"
|
|
736
|
+
and _bbox_center_y(rule.bbox) >= 0.85 * local_page_height
|
|
737
|
+
and rule.bbox[2] - rule.bbox[0] >= 0.2 * local_page_width
|
|
738
|
+
and not _rule_belongs_to_confirmed_table(
|
|
739
|
+
rule,
|
|
740
|
+
local_axis_lines,
|
|
741
|
+
page.table_bboxes,
|
|
742
|
+
local_page_width,
|
|
743
|
+
)
|
|
744
|
+
and not _rule_overlaps_fixed_container(
|
|
745
|
+
rule,
|
|
746
|
+
page.fixed_blocks,
|
|
747
|
+
page.page_size,
|
|
748
|
+
)
|
|
749
|
+
]
|
|
750
|
+
local_lines = [
|
|
751
|
+
(
|
|
752
|
+
line,
|
|
753
|
+
_rotate_bbox_to_upright(
|
|
754
|
+
line.bbox,
|
|
755
|
+
page.page_size,
|
|
756
|
+
dominant_angle,
|
|
757
|
+
),
|
|
758
|
+
)
|
|
759
|
+
for line in available
|
|
760
|
+
if line.angle == dominant_angle
|
|
761
|
+
]
|
|
762
|
+
if not local_lines:
|
|
763
|
+
continue
|
|
764
|
+
median_height = statistics.median(_line_effective_height(line, bbox) for line, bbox in local_lines)
|
|
765
|
+
lanes = [
|
|
766
|
+
lane
|
|
767
|
+
for lane in _infer_text_lanes(
|
|
768
|
+
local_lines,
|
|
769
|
+
local_page_width,
|
|
770
|
+
median_height,
|
|
771
|
+
recalculate_intervals=False,
|
|
772
|
+
)
|
|
773
|
+
if not lane.is_span
|
|
774
|
+
]
|
|
775
|
+
for upper_index, upper in enumerate(rules[:-1]):
|
|
776
|
+
for lower in rules[upper_index + 1 :]:
|
|
777
|
+
vertical_gap = lower.bbox[1] - upper.bbox[3]
|
|
778
|
+
if not 2.0 * median_height <= vertical_gap <= 6.0 * median_height:
|
|
779
|
+
continue
|
|
780
|
+
if _bbox_axis_overlap_ratio(upper.bbox, lower.bbox, axis="x") < 0.9:
|
|
781
|
+
continue
|
|
782
|
+
corridor_left = max(upper.bbox[0], lower.bbox[0])
|
|
783
|
+
corridor_right = min(upper.bbox[2], lower.bbox[2])
|
|
784
|
+
members = [
|
|
785
|
+
(line, bbox)
|
|
786
|
+
for line, bbox in local_lines
|
|
787
|
+
if bbox[1] >= upper.bbox[3]
|
|
788
|
+
and bbox[3] <= lower.bbox[1]
|
|
789
|
+
and bbox[0] >= corridor_left - 0.5 * median_height
|
|
790
|
+
and bbox[2] <= corridor_right + 0.5 * median_height
|
|
791
|
+
]
|
|
792
|
+
if not 1 <= len(members) <= 3:
|
|
793
|
+
continue
|
|
794
|
+
if any(
|
|
795
|
+
abs(_bbox_center_x(bbox) - 0.5 * (corridor_left + corridor_right))
|
|
796
|
+
> 0.15 * max(0.1, corridor_right - corridor_left)
|
|
797
|
+
for _line, bbox in members
|
|
798
|
+
):
|
|
799
|
+
continue
|
|
800
|
+
for line, _bbox in members:
|
|
801
|
+
line.semantic_type = "footer"
|
|
802
|
+
break
|
|
803
|
+
for rule in rules:
|
|
804
|
+
for line in _single_rule_footer_members(
|
|
805
|
+
rule,
|
|
806
|
+
local_lines,
|
|
807
|
+
lanes,
|
|
808
|
+
median_height,
|
|
809
|
+
):
|
|
810
|
+
line.semantic_type = "footer"
|
|
811
|
+
|
|
812
|
+
|
|
813
|
+
def _single_rule_footer_members(
|
|
814
|
+
rule: _LocalAxisLine,
|
|
815
|
+
local_lines: list[tuple[_LineItem, BBox]],
|
|
816
|
+
lanes: list[_TextLane],
|
|
817
|
+
body_height: float,
|
|
818
|
+
) -> list[_LineItem]:
|
|
819
|
+
"""返回底部单横线下方、唯一栏内连续的小字号页脚行。"""
|
|
820
|
+
|
|
821
|
+
rule_width = max(0.1, rule.bbox[2] - rule.bbox[0])
|
|
822
|
+
rule_center_x = _bbox_center_x(rule.bbox)
|
|
823
|
+
matching_lanes = []
|
|
824
|
+
for lane in lanes:
|
|
825
|
+
overlap = max(
|
|
826
|
+
0.0,
|
|
827
|
+
min(rule.bbox[2], lane.right) - max(rule.bbox[0], lane.left),
|
|
828
|
+
)
|
|
829
|
+
if overlap / rule_width >= 0.8 and lane.left <= rule_center_x <= lane.right:
|
|
830
|
+
matching_lanes.append(lane)
|
|
831
|
+
if len(matching_lanes) != 1:
|
|
832
|
+
return []
|
|
833
|
+
|
|
834
|
+
lane = matching_lanes[0]
|
|
835
|
+
if len(lanes) > 1 and lane.left < max(candidate_lane.left for candidate_lane in lanes) - body_height:
|
|
836
|
+
return []
|
|
837
|
+
tolerance = 0.5 * body_height
|
|
838
|
+
rows_below = sorted(
|
|
839
|
+
(
|
|
840
|
+
(line, bbox)
|
|
841
|
+
for line, bbox in local_lines
|
|
842
|
+
if line.semantic_type is None
|
|
843
|
+
and bbox[1] >= rule.bbox[3]
|
|
844
|
+
and bbox[0] >= lane.left - tolerance
|
|
845
|
+
and bbox[2] <= lane.right + tolerance
|
|
846
|
+
),
|
|
847
|
+
key=lambda item: (item[1][1], item[1][0], item[0].source_index),
|
|
848
|
+
)
|
|
849
|
+
if not rows_below:
|
|
850
|
+
return []
|
|
851
|
+
first_line, first_bbox = rows_below[0]
|
|
852
|
+
if first_bbox[1] - rule.bbox[3] > body_height or _line_effective_height(first_line, first_bbox) > 0.9 * body_height:
|
|
853
|
+
return []
|
|
854
|
+
|
|
855
|
+
members = [(first_line, first_bbox)]
|
|
856
|
+
for line, bbox in rows_below[1:]:
|
|
857
|
+
previous_bbox = members[-1][1]
|
|
858
|
+
if (
|
|
859
|
+
bbox[1] - previous_bbox[3] > body_height
|
|
860
|
+
or bbox[3] - first_bbox[1] > 5.0 * body_height
|
|
861
|
+
or _line_effective_height(line, bbox) > 0.9 * body_height
|
|
862
|
+
):
|
|
863
|
+
break
|
|
864
|
+
members.append((line, bbox))
|
|
865
|
+
if not 2 <= len(members) <= 8:
|
|
866
|
+
return []
|
|
867
|
+
member_left_edges = [bbox[0] for _line, bbox in members]
|
|
868
|
+
if max(member_left_edges) - min(member_left_edges) > 0.75 * body_height:
|
|
869
|
+
return []
|
|
870
|
+
return [line for line, _bbox in members]
|
|
871
|
+
|
|
872
|
+
|
|
873
|
+
def _rule_overlaps_fixed_container(
|
|
874
|
+
rule: _LocalAxisLine,
|
|
875
|
+
fixed_blocks: list[dict[str, object]],
|
|
876
|
+
page_size: tuple[float, float],
|
|
877
|
+
) -> bool:
|
|
878
|
+
"""排除落在表格、图片、公式或代码容器内的页首横线。"""
|
|
879
|
+
|
|
880
|
+
expanded_rule = _expand_bbox(
|
|
881
|
+
rule.original_bbox,
|
|
882
|
+
max(1.0, rule.width),
|
|
883
|
+
)
|
|
884
|
+
for block in fixed_blocks:
|
|
885
|
+
if block.get("type") not in {"table", "image", "equation", "code"}:
|
|
886
|
+
continue
|
|
887
|
+
bbox = _clip_bbox(_coerce_bbox(block.get("bbox")), page_size)
|
|
888
|
+
if bbox is not None and _bbox_intersects(expanded_rule, bbox):
|
|
889
|
+
return True
|
|
890
|
+
return False
|
|
891
|
+
|
|
892
|
+
|
|
893
|
+
def _classify_page_number_outer_companions(
|
|
894
|
+
pages: list[_PreparedPage],
|
|
895
|
+
) -> None:
|
|
896
|
+
"""把上下页码外侧的未分类文本和图片标为对应页眉或页脚。"""
|
|
897
|
+
|
|
898
|
+
for page in pages:
|
|
899
|
+
page_numbers = [line for line in page.remaining_lines if line.semantic_type == "page_number"]
|
|
900
|
+
for page_number in page_numbers:
|
|
901
|
+
angle = page_number.angle
|
|
902
|
+
local_page_size = (page.page_size[1], page.page_size[0]) if angle in {90, 270} else page.page_size
|
|
903
|
+
local_page_height = local_page_size[1]
|
|
904
|
+
if local_page_height <= 0:
|
|
905
|
+
continue
|
|
906
|
+
page_number_bbox = _rotate_bbox_to_upright(
|
|
907
|
+
page_number.bbox,
|
|
908
|
+
page.page_size,
|
|
909
|
+
angle,
|
|
910
|
+
)
|
|
911
|
+
normalized_center_y = _bbox_center_y(page_number_bbox) / local_page_height
|
|
912
|
+
if normalized_center_y <= 0.3:
|
|
913
|
+
target_type: Literal["header", "footer"] = "header"
|
|
914
|
+
outward_limit = page_number_bbox[1]
|
|
915
|
+
elif normalized_center_y >= 0.7:
|
|
916
|
+
target_type = "footer"
|
|
917
|
+
outward_limit = page_number_bbox[3]
|
|
918
|
+
else:
|
|
919
|
+
continue
|
|
920
|
+
for line in page.remaining_lines:
|
|
921
|
+
if line.semantic_type is not None or line.angle != angle:
|
|
922
|
+
continue
|
|
923
|
+
local_bbox = _rotate_bbox_to_upright(
|
|
924
|
+
line.bbox,
|
|
925
|
+
page.page_size,
|
|
926
|
+
angle,
|
|
927
|
+
)
|
|
928
|
+
is_outward = local_bbox[3] <= outward_limit if target_type == "header" else local_bbox[1] >= outward_limit
|
|
929
|
+
same_marginal_row = (
|
|
930
|
+
_bbox_axis_overlap_ratio(
|
|
931
|
+
local_bbox,
|
|
932
|
+
page_number_bbox,
|
|
933
|
+
axis="y",
|
|
934
|
+
)
|
|
935
|
+
>= 0.5
|
|
936
|
+
)
|
|
937
|
+
if is_outward or same_marginal_row:
|
|
938
|
+
line.semantic_type = target_type
|
|
939
|
+
for block in page.fixed_blocks:
|
|
940
|
+
if block.get("type") != "image":
|
|
941
|
+
continue
|
|
942
|
+
block_angle = int(block.get("angle", 0) or 0) % 360
|
|
943
|
+
if block_angle != angle:
|
|
944
|
+
continue
|
|
945
|
+
bbox = _clip_bbox(
|
|
946
|
+
_coerce_bbox(block.get("bbox")),
|
|
947
|
+
page.page_size,
|
|
948
|
+
)
|
|
949
|
+
if bbox is None:
|
|
950
|
+
continue
|
|
951
|
+
local_bbox = _rotate_bbox_to_upright(
|
|
952
|
+
bbox,
|
|
953
|
+
page.page_size,
|
|
954
|
+
angle,
|
|
955
|
+
)
|
|
956
|
+
is_outward = local_bbox[3] <= outward_limit if target_type == "header" else local_bbox[1] >= outward_limit
|
|
957
|
+
same_marginal_row = (
|
|
958
|
+
_bbox_axis_overlap_ratio(
|
|
959
|
+
local_bbox,
|
|
960
|
+
page_number_bbox,
|
|
961
|
+
axis="y",
|
|
962
|
+
)
|
|
963
|
+
>= 0.5
|
|
964
|
+
)
|
|
965
|
+
if is_outward or same_marginal_row:
|
|
966
|
+
block["type"] = target_type
|
|
967
|
+
|
|
968
|
+
|
|
969
|
+
def _classify_split_marginal_row_companions(
|
|
970
|
+
pages: list[_PreparedPage],
|
|
971
|
+
) -> None:
|
|
972
|
+
"""把页边缘同一拆分视觉行中的未分类碎片继承为页眉或页脚。"""
|
|
973
|
+
|
|
974
|
+
for page in pages:
|
|
975
|
+
row_groups: dict[tuple[int, int], list[_LineItem]] = {}
|
|
976
|
+
for line in page.remaining_lines:
|
|
977
|
+
if line.visual_row_id is None or not line.split_from_row:
|
|
978
|
+
continue
|
|
979
|
+
row_groups.setdefault((line.angle, line.visual_row_id), []).append(line)
|
|
980
|
+
for (angle, _row_id), members in row_groups.items():
|
|
981
|
+
local_page_height = page.page_size[0] if angle in {90, 270} else page.page_size[1]
|
|
982
|
+
local_bboxes = [_rotate_bbox_to_upright(line.bbox, page.page_size, angle) for line in members]
|
|
983
|
+
row_center = statistics.fmean(_bbox_center_y(bbox) for bbox in local_bboxes)
|
|
984
|
+
if row_center <= 0.1 * local_page_height:
|
|
985
|
+
target_type: Literal["header", "footer"] = "header"
|
|
986
|
+
elif row_center >= 0.9 * local_page_height:
|
|
987
|
+
target_type = "footer"
|
|
988
|
+
else:
|
|
989
|
+
continue
|
|
990
|
+
anchor_types = {line.semantic_type for line in members if line.semantic_type in {target_type, "page_number"}}
|
|
991
|
+
if not anchor_types:
|
|
992
|
+
continue
|
|
993
|
+
for line in members:
|
|
994
|
+
if line.semantic_type is None:
|
|
995
|
+
line.semantic_type = target_type
|
|
996
|
+
|
|
997
|
+
|
|
998
|
+
def _classify_raw_page_marginals(sources: list[_PageSource]) -> None:
|
|
999
|
+
"""在视觉容器认领前保护强跨页页码、页眉和页脚文本。"""
|
|
1000
|
+
|
|
1001
|
+
if len(sources) < 2:
|
|
1002
|
+
return
|
|
1003
|
+
candidates = [
|
|
1004
|
+
candidate
|
|
1005
|
+
for page_index, source in enumerate(sources)
|
|
1006
|
+
for line in source.lines
|
|
1007
|
+
if (
|
|
1008
|
+
candidate := _build_marginal_candidate(
|
|
1009
|
+
page_index,
|
|
1010
|
+
line,
|
|
1011
|
+
source.page_size,
|
|
1012
|
+
)
|
|
1013
|
+
)
|
|
1014
|
+
is not None
|
|
1015
|
+
and (
|
|
1016
|
+
_bbox_center_y(candidate.local_bbox) / candidate.local_page_size[1] <= 0.08
|
|
1017
|
+
or _bbox_center_y(candidate.local_bbox) / candidate.local_page_size[1] >= 0.92
|
|
1018
|
+
)
|
|
1019
|
+
]
|
|
1020
|
+
_classify_marginal_candidates(candidates)
|
|
1021
|
+
|
|
1022
|
+
|
|
1023
|
+
def _classify_repeated_page_marginals(pages: list[_PreparedPage]) -> None:
|
|
1024
|
+
"""仅用相邻或同奇偶页的重复证据标注页码、页眉和页脚。"""
|
|
1025
|
+
|
|
1026
|
+
if len(pages) < 2:
|
|
1027
|
+
return
|
|
1028
|
+
candidates = [
|
|
1029
|
+
candidate
|
|
1030
|
+
for page_index, page in enumerate(pages)
|
|
1031
|
+
for line in page.remaining_lines
|
|
1032
|
+
if (candidate := _build_marginal_candidate(page_index, line, page.page_size)) is not None
|
|
1033
|
+
]
|
|
1034
|
+
|
|
1035
|
+
_classify_marginal_candidates(candidates)
|
|
1036
|
+
|
|
1037
|
+
|
|
1038
|
+
def _classify_marginal_candidates(
|
|
1039
|
+
candidates: list[_MarginalCandidate],
|
|
1040
|
+
) -> None:
|
|
1041
|
+
"""复用跨页递增页码和稳定边缘文本的强证据匹配。"""
|
|
1042
|
+
|
|
1043
|
+
for left_index, left in enumerate(candidates):
|
|
1044
|
+
left_value = _parse_page_number_value(left.line.text)
|
|
1045
|
+
if left_value is None:
|
|
1046
|
+
continue
|
|
1047
|
+
for right in candidates[left_index + 1 :]:
|
|
1048
|
+
page_delta = right.page_index - left.page_index
|
|
1049
|
+
if page_delta > 2:
|
|
1050
|
+
break
|
|
1051
|
+
right_value = _parse_page_number_value(right.line.text)
|
|
1052
|
+
if (
|
|
1053
|
+
page_delta > 0
|
|
1054
|
+
and right_value is not None
|
|
1055
|
+
and right_value - left_value == page_delta
|
|
1056
|
+
and _page_number_candidates_match(left, right)
|
|
1057
|
+
):
|
|
1058
|
+
left.line.semantic_type = "page_number"
|
|
1059
|
+
right.line.semantic_type = "page_number"
|
|
1060
|
+
|
|
1061
|
+
for left_index, left in enumerate(candidates):
|
|
1062
|
+
if left.line.semantic_type == "page_number":
|
|
1063
|
+
continue
|
|
1064
|
+
for right in candidates[left_index + 1 :]:
|
|
1065
|
+
page_delta = right.page_index - left.page_index
|
|
1066
|
+
if page_delta > 2:
|
|
1067
|
+
break
|
|
1068
|
+
if (
|
|
1069
|
+
page_delta > 0
|
|
1070
|
+
and left.region != "side"
|
|
1071
|
+
and right.region != "side"
|
|
1072
|
+
and right.line.semantic_type != "page_number"
|
|
1073
|
+
and _marginal_geometry_matches(left, right)
|
|
1074
|
+
and _marginal_text_matches(left.line.text, right.line.text)
|
|
1075
|
+
):
|
|
1076
|
+
left.line.semantic_type = left.region
|
|
1077
|
+
right.line.semantic_type = right.region
|
|
1078
|
+
|
|
1079
|
+
|
|
1080
|
+
def _classify_single_page_compound_headers(pages: list[_PreparedPage]) -> None:
|
|
1081
|
+
"""以拆分同行、字号收缩和正文栏右缘共同确认单页复合页眉。"""
|
|
1082
|
+
|
|
1083
|
+
if len(pages) != 1:
|
|
1084
|
+
return
|
|
1085
|
+
page = pages[0]
|
|
1086
|
+
page_width, page_height = page.page_size
|
|
1087
|
+
if page_width <= 0 or page_height <= 0:
|
|
1088
|
+
return
|
|
1089
|
+
|
|
1090
|
+
row_groups: dict[tuple[int, int], list[_LineItem]] = {}
|
|
1091
|
+
for line in page.remaining_lines:
|
|
1092
|
+
if line.semantic_type is None and line.visual_row_id is not None and line.split_from_row:
|
|
1093
|
+
row_groups.setdefault((line.angle, line.visual_row_id), []).append(line)
|
|
1094
|
+
|
|
1095
|
+
for (angle, _row_id), members in row_groups.items():
|
|
1096
|
+
if len(members) < 2:
|
|
1097
|
+
continue
|
|
1098
|
+
local_page_width = page_height if angle in {90, 270} else page_width
|
|
1099
|
+
local_page_height = page_width if angle in {90, 270} else page_height
|
|
1100
|
+
local_members = [(line, _rotate_bbox_to_upright(line.bbox, page.page_size, angle)) for line in members]
|
|
1101
|
+
row_top = min(bbox[1] for _line, bbox in local_members)
|
|
1102
|
+
row_bottom = max(bbox[3] for _line, bbox in local_members)
|
|
1103
|
+
if row_top < 0 or row_bottom > 0.05 * local_page_height:
|
|
1104
|
+
continue
|
|
1105
|
+
|
|
1106
|
+
row_left = min(bbox[0] for _line, bbox in local_members)
|
|
1107
|
+
row_right = max(bbox[2] for _line, bbox in local_members)
|
|
1108
|
+
related_body = [
|
|
1109
|
+
(line, local_bbox)
|
|
1110
|
+
for line in page.remaining_lines
|
|
1111
|
+
if line.semantic_type is None
|
|
1112
|
+
and line.angle == angle
|
|
1113
|
+
and line not in members
|
|
1114
|
+
and (
|
|
1115
|
+
local_bbox := _rotate_bbox_to_upright(
|
|
1116
|
+
line.bbox,
|
|
1117
|
+
page.page_size,
|
|
1118
|
+
angle,
|
|
1119
|
+
)
|
|
1120
|
+
)[1]
|
|
1121
|
+
>= 0.05 * local_page_height
|
|
1122
|
+
and local_bbox[2] - local_bbox[0] >= 0.3 * local_page_width
|
|
1123
|
+
and _bbox_axis_overlap_ratio(
|
|
1124
|
+
(row_left, row_top, row_right, row_bottom),
|
|
1125
|
+
local_bbox,
|
|
1126
|
+
axis="x",
|
|
1127
|
+
)
|
|
1128
|
+
>= 0.2
|
|
1129
|
+
]
|
|
1130
|
+
if len(related_body) < 3:
|
|
1131
|
+
continue
|
|
1132
|
+
body_height = statistics.median(_line_effective_height(line, bbox) for line, bbox in related_body)
|
|
1133
|
+
row_height = max(_line_effective_height(line, bbox) for line, bbox in local_members)
|
|
1134
|
+
if row_height > 0.85 * body_height:
|
|
1135
|
+
continue
|
|
1136
|
+
body_right = statistics.median(bbox[2] for _line, bbox in related_body)
|
|
1137
|
+
has_right_sidecar = any(
|
|
1138
|
+
bbox[2] - bbox[0] <= max(4.0 * row_height, 0.12 * local_page_width)
|
|
1139
|
+
and abs(bbox[2] - body_right) <= max(3.0, 0.02 * local_page_width)
|
|
1140
|
+
for _line, bbox in local_members
|
|
1141
|
+
)
|
|
1142
|
+
if has_right_sidecar:
|
|
1143
|
+
for line, _bbox in local_members:
|
|
1144
|
+
line.semantic_type = "header"
|
|
1145
|
+
|
|
1146
|
+
|
|
1147
|
+
def _classify_page_footnote_trailing_footers(
|
|
1148
|
+
pages: list[_PreparedPage],
|
|
1149
|
+
) -> None:
|
|
1150
|
+
"""把任意页脚注投影下方、已越过续行边界的紧凑尾段标为页脚。"""
|
|
1151
|
+
|
|
1152
|
+
for page in pages:
|
|
1153
|
+
line_by_source = {line.source_index: line for line in page.remaining_lines}
|
|
1154
|
+
ranked_groups: list[tuple[float, set[int]]] = []
|
|
1155
|
+
for source_indices in page.page_footnote_groups:
|
|
1156
|
+
anchor_lines = [
|
|
1157
|
+
line_by_source[source_index]
|
|
1158
|
+
for source_index in source_indices
|
|
1159
|
+
if source_index in line_by_source and line_by_source[source_index].semantic_type == "page_footnote"
|
|
1160
|
+
]
|
|
1161
|
+
angles = {line.angle for line in anchor_lines}
|
|
1162
|
+
if len(angles) != 1:
|
|
1163
|
+
continue
|
|
1164
|
+
angle = next(iter(angles))
|
|
1165
|
+
local_bottom = max(
|
|
1166
|
+
_rotate_bbox_to_upright(
|
|
1167
|
+
line.bbox,
|
|
1168
|
+
page.page_size,
|
|
1169
|
+
angle,
|
|
1170
|
+
)[3]
|
|
1171
|
+
for line in anchor_lines
|
|
1172
|
+
)
|
|
1173
|
+
ranked_groups.append((local_bottom, source_indices))
|
|
1174
|
+
|
|
1175
|
+
# 优先处理页面最下方的脚注组,避免上方脚注跨过下方脚注寻找页脚。
|
|
1176
|
+
for _local_bottom, source_indices in sorted(
|
|
1177
|
+
ranked_groups,
|
|
1178
|
+
key=lambda item: item[0],
|
|
1179
|
+
reverse=True,
|
|
1180
|
+
):
|
|
1181
|
+
for line in _page_footnote_trailing_footer_members(
|
|
1182
|
+
page,
|
|
1183
|
+
source_indices,
|
|
1184
|
+
):
|
|
1185
|
+
line.semantic_type = "footer"
|
|
1186
|
+
|
|
1187
|
+
|
|
1188
|
+
def _page_footnote_trailing_footer_members(
|
|
1189
|
+
page: _PreparedPage,
|
|
1190
|
+
source_indices: set[int],
|
|
1191
|
+
) -> list[_LineItem]:
|
|
1192
|
+
"""返回脚注水平投影下方唯一、紧凑且小于正文尺度的页脚行。"""
|
|
1193
|
+
|
|
1194
|
+
line_by_source = {line.source_index: line for line in page.remaining_lines}
|
|
1195
|
+
anchor_lines = [
|
|
1196
|
+
line_by_source[source_index]
|
|
1197
|
+
for source_index in source_indices
|
|
1198
|
+
if source_index in line_by_source and line_by_source[source_index].semantic_type == "page_footnote"
|
|
1199
|
+
]
|
|
1200
|
+
angles = {line.angle for line in anchor_lines}
|
|
1201
|
+
if len(angles) != 1:
|
|
1202
|
+
return []
|
|
1203
|
+
angle = next(iter(angles))
|
|
1204
|
+
local_page_size = (page.page_size[1], page.page_size[0]) if angle in {90, 270} else page.page_size
|
|
1205
|
+
local_page_width, local_page_height = local_page_size
|
|
1206
|
+
if local_page_width <= 0 or local_page_height <= 0:
|
|
1207
|
+
return []
|
|
1208
|
+
|
|
1209
|
+
anchor_geometry = [
|
|
1210
|
+
(
|
|
1211
|
+
line,
|
|
1212
|
+
_rotate_bbox_to_upright(
|
|
1213
|
+
line.bbox,
|
|
1214
|
+
page.page_size,
|
|
1215
|
+
angle,
|
|
1216
|
+
),
|
|
1217
|
+
)
|
|
1218
|
+
for line in anchor_lines
|
|
1219
|
+
]
|
|
1220
|
+
anchor_bbox = _bbox_union_many([bbox for _line, bbox in anchor_geometry])
|
|
1221
|
+
if anchor_bbox[3] < 0.75 * local_page_height:
|
|
1222
|
+
return []
|
|
1223
|
+
|
|
1224
|
+
unresolved_geometry = [
|
|
1225
|
+
(
|
|
1226
|
+
line,
|
|
1227
|
+
_rotate_bbox_to_upright(
|
|
1228
|
+
line.bbox,
|
|
1229
|
+
page.page_size,
|
|
1230
|
+
angle,
|
|
1231
|
+
),
|
|
1232
|
+
)
|
|
1233
|
+
for line in page.remaining_lines
|
|
1234
|
+
if line.semantic_type is None and line.angle == angle
|
|
1235
|
+
]
|
|
1236
|
+
body_geometry = [
|
|
1237
|
+
(line, bbox)
|
|
1238
|
+
for line, bbox in unresolved_geometry
|
|
1239
|
+
if bbox[3] <= anchor_bbox[1]
|
|
1240
|
+
and bbox[2] - bbox[0] >= 0.3 * local_page_width
|
|
1241
|
+
and 0.1 * local_page_height <= _bbox_center_y(bbox) <= 0.94 * local_page_height
|
|
1242
|
+
and _bbox_axis_overlap_ratio(
|
|
1243
|
+
anchor_bbox,
|
|
1244
|
+
bbox,
|
|
1245
|
+
axis="x",
|
|
1246
|
+
)
|
|
1247
|
+
>= 0.2
|
|
1248
|
+
]
|
|
1249
|
+
if len(body_geometry) < 3:
|
|
1250
|
+
return []
|
|
1251
|
+
body_height = statistics.median(_line_effective_height(line, bbox) for line, bbox in body_geometry)
|
|
1252
|
+
if body_height <= 0:
|
|
1253
|
+
return []
|
|
1254
|
+
|
|
1255
|
+
if any(bbox[1] <= anchor_bbox[3] < bbox[3] for _line, bbox in unresolved_geometry):
|
|
1256
|
+
# 另一栏正文仍跨过脚注底边时,不能把其下方局部文本猜成全页页脚。
|
|
1257
|
+
return []
|
|
1258
|
+
trailing_geometry = sorted(
|
|
1259
|
+
((line, bbox) for line, bbox in unresolved_geometry if bbox[1] > anchor_bbox[3]),
|
|
1260
|
+
key=lambda item: (item[1][1], item[1][0], item[0].source_index),
|
|
1261
|
+
)
|
|
1262
|
+
if not 1 <= len(trailing_geometry) <= 3:
|
|
1263
|
+
return []
|
|
1264
|
+
|
|
1265
|
+
first = trailing_geometry[0]
|
|
1266
|
+
candidate_bbox = _bbox_union_many([bbox for _line, bbox in trailing_geometry])
|
|
1267
|
+
if first[1][1] < 0.82 * local_page_height:
|
|
1268
|
+
return []
|
|
1269
|
+
projection_tolerance = 0.5 * body_height
|
|
1270
|
+
if (
|
|
1271
|
+
candidate_bbox[0] < anchor_bbox[0] - projection_tolerance
|
|
1272
|
+
or candidate_bbox[2] > anchor_bbox[2] + projection_tolerance
|
|
1273
|
+
or abs(candidate_bbox[0] - anchor_bbox[0]) > body_height
|
|
1274
|
+
):
|
|
1275
|
+
return []
|
|
1276
|
+
left_edges = [bbox[0] for _line, bbox in trailing_geometry]
|
|
1277
|
+
if max(left_edges) - min(left_edges) > 0.75 * body_height:
|
|
1278
|
+
return []
|
|
1279
|
+
if candidate_bbox[2] - candidate_bbox[0] > 0.6 * local_page_width:
|
|
1280
|
+
return []
|
|
1281
|
+
if candidate_bbox[3] - candidate_bbox[1] > 4.0 * body_height:
|
|
1282
|
+
return []
|
|
1283
|
+
if any(_line_effective_height(line, bbox) > 0.95 * body_height for line, bbox in trailing_geometry):
|
|
1284
|
+
return []
|
|
1285
|
+
|
|
1286
|
+
projecting_anchor_rows = [
|
|
1287
|
+
item
|
|
1288
|
+
for item in anchor_geometry
|
|
1289
|
+
if _bbox_axis_overlap_ratio(
|
|
1290
|
+
item[1],
|
|
1291
|
+
candidate_bbox,
|
|
1292
|
+
axis="x",
|
|
1293
|
+
)
|
|
1294
|
+
>= 0.2
|
|
1295
|
+
]
|
|
1296
|
+
if not projecting_anchor_rows:
|
|
1297
|
+
return []
|
|
1298
|
+
anchor_last = max(
|
|
1299
|
+
projecting_anchor_rows,
|
|
1300
|
+
key=lambda item: (item[1][1], item[1][0], item[0].source_index),
|
|
1301
|
+
)
|
|
1302
|
+
continuation_gap_limit = _page_footnote_continuation_gap_limit(
|
|
1303
|
+
body_height,
|
|
1304
|
+
local_page_height,
|
|
1305
|
+
)
|
|
1306
|
+
first_gap = _effective_text_row_gap(anchor_last, first)
|
|
1307
|
+
if not continuation_gap_limit < first_gap <= 3.0 * body_height:
|
|
1308
|
+
return []
|
|
1309
|
+
if any(
|
|
1310
|
+
_effective_text_row_gap(previous, current) > continuation_gap_limit
|
|
1311
|
+
for previous, current in zip(
|
|
1312
|
+
trailing_geometry,
|
|
1313
|
+
trailing_geometry[1:],
|
|
1314
|
+
)
|
|
1315
|
+
):
|
|
1316
|
+
return []
|
|
1317
|
+
|
|
1318
|
+
original_candidate_bbox = _bbox_union_many([line.bbox for line, _bbox in trailing_geometry])
|
|
1319
|
+
if any(
|
|
1320
|
+
container_bbox is not None and _bbox_intersects(original_candidate_bbox, container_bbox)
|
|
1321
|
+
for block in page.fixed_blocks
|
|
1322
|
+
if (
|
|
1323
|
+
container_bbox := _clip_bbox(
|
|
1324
|
+
_coerce_bbox(block.get("bbox")),
|
|
1325
|
+
page.page_size,
|
|
1326
|
+
)
|
|
1327
|
+
)
|
|
1328
|
+
is not None
|
|
1329
|
+
):
|
|
1330
|
+
return []
|
|
1331
|
+
return [line for line, _bbox in trailing_geometry]
|
|
1332
|
+
|
|
1333
|
+
|
|
1334
|
+
def _classify_isolated_first_page_footer(pages: list[_PreparedPage]) -> None:
|
|
1335
|
+
"""用多页首页的极底位置、正文尺度和孤立净空补标唯一页脚。"""
|
|
1336
|
+
|
|
1337
|
+
if len(pages) < 2:
|
|
1338
|
+
return
|
|
1339
|
+
page = pages[0]
|
|
1340
|
+
page_width, page_height = page.page_size
|
|
1341
|
+
if page_width <= 0 or page_height <= 0:
|
|
1342
|
+
return
|
|
1343
|
+
|
|
1344
|
+
body_lines = [
|
|
1345
|
+
line
|
|
1346
|
+
for line in page.remaining_lines
|
|
1347
|
+
if line.semantic_type is None
|
|
1348
|
+
and line.angle == 0
|
|
1349
|
+
and line.bbox[2] - line.bbox[0] >= 0.3 * page_width
|
|
1350
|
+
and 0.1 * page_height <= _bbox_center_y(line.bbox) <= 0.94 * page_height
|
|
1351
|
+
]
|
|
1352
|
+
if len(body_lines) < 4:
|
|
1353
|
+
return
|
|
1354
|
+
body_height = statistics.median(_line_effective_height(line, line.bbox) for line in body_lines)
|
|
1355
|
+
body_bottom = max(line.bbox[3] for line in body_lines)
|
|
1356
|
+
if body_bottom < 0.7 * page_height:
|
|
1357
|
+
return
|
|
1358
|
+
container_bboxes = [bbox for block in page.fixed_blocks if (bbox := _coerce_bbox(block.get("bbox"))) is not None]
|
|
1359
|
+
candidates = [
|
|
1360
|
+
line
|
|
1361
|
+
for line in page.remaining_lines
|
|
1362
|
+
if line.semantic_type is None
|
|
1363
|
+
and line.angle == 0
|
|
1364
|
+
and line.bbox[1] >= 0.94 * page_height
|
|
1365
|
+
and line.bbox[2] - line.bbox[0] <= 0.6 * page_width
|
|
1366
|
+
and abs(_bbox_center_x(line.bbox) - 0.5 * page_width) <= 0.08 * page_width
|
|
1367
|
+
and _line_effective_height(line, line.bbox) <= 0.95 * body_height
|
|
1368
|
+
and line.bbox[1] - body_bottom >= 1.5 * body_height
|
|
1369
|
+
and not any(_bbox_intersects(line.bbox, container_bbox) for container_bbox in container_bboxes)
|
|
1370
|
+
]
|
|
1371
|
+
if len(candidates) == 1:
|
|
1372
|
+
candidates[0].semantic_type = "footer"
|
|
1373
|
+
|
|
1374
|
+
|
|
1375
|
+
def _classify_repeated_visual_headers(pages: list[_PreparedPage]) -> None:
|
|
1376
|
+
"""仅按页首位置与跨页重复几何,把整体图片重标为视觉页眉。"""
|
|
1377
|
+
|
|
1378
|
+
candidates: list[tuple[int, dict[str, object], BBox, int]] = []
|
|
1379
|
+
for page_index, page in enumerate(pages):
|
|
1380
|
+
# 首页常使用独立封面版式,不参与正文页视觉页眉聚类。
|
|
1381
|
+
if page_index == 0:
|
|
1382
|
+
continue
|
|
1383
|
+
page_width, page_height = page.page_size
|
|
1384
|
+
if page_width <= 0 or page_height <= 0:
|
|
1385
|
+
continue
|
|
1386
|
+
for block in page.fixed_blocks:
|
|
1387
|
+
if block.get("type") != "image":
|
|
1388
|
+
continue
|
|
1389
|
+
bbox = _clip_bbox(_coerce_bbox(block.get("bbox")), page.page_size)
|
|
1390
|
+
if bbox is None or bbox[3] > 0.12 * page_height:
|
|
1391
|
+
continue
|
|
1392
|
+
normalized_bbox = (
|
|
1393
|
+
bbox[0] / page_width,
|
|
1394
|
+
bbox[1] / page_height,
|
|
1395
|
+
bbox[2] / page_width,
|
|
1396
|
+
bbox[3] / page_height,
|
|
1397
|
+
)
|
|
1398
|
+
angle = int(block.get("angle", 0) or 0) % 360
|
|
1399
|
+
candidates.append((page_index, block, normalized_bbox, angle))
|
|
1400
|
+
|
|
1401
|
+
if len(candidates) < 3:
|
|
1402
|
+
return
|
|
1403
|
+
|
|
1404
|
+
parents = list(range(len(candidates)))
|
|
1405
|
+
|
|
1406
|
+
def find(index: int) -> int:
|
|
1407
|
+
"""查找视觉页眉候选所属几何簇的根节点。"""
|
|
1408
|
+
|
|
1409
|
+
while parents[index] != index:
|
|
1410
|
+
parents[index] = parents[parents[index]]
|
|
1411
|
+
index = parents[index]
|
|
1412
|
+
return index
|
|
1413
|
+
|
|
1414
|
+
def union(first_index: int, second_index: int) -> None:
|
|
1415
|
+
"""合并跨页距离和归一化几何均匹配的两个候选。"""
|
|
1416
|
+
|
|
1417
|
+
first_root = find(first_index)
|
|
1418
|
+
second_root = find(second_index)
|
|
1419
|
+
if first_root != second_root:
|
|
1420
|
+
parents[second_root] = first_root
|
|
1421
|
+
|
|
1422
|
+
for first_index, (
|
|
1423
|
+
first_page,
|
|
1424
|
+
_first_block,
|
|
1425
|
+
first_bbox,
|
|
1426
|
+
first_angle,
|
|
1427
|
+
) in enumerate(candidates):
|
|
1428
|
+
for second_index in range(first_index + 1, len(candidates)):
|
|
1429
|
+
second_page, _second_block, second_bbox, second_angle = candidates[second_index]
|
|
1430
|
+
page_delta = second_page - first_page
|
|
1431
|
+
if page_delta > 2:
|
|
1432
|
+
break
|
|
1433
|
+
if page_delta > 0 and first_angle == second_angle and _visual_header_geometry_matches(first_bbox, second_bbox):
|
|
1434
|
+
union(first_index, second_index)
|
|
1435
|
+
|
|
1436
|
+
clusters: dict[int, list[int]] = {}
|
|
1437
|
+
for candidate_index in range(len(candidates)):
|
|
1438
|
+
clusters.setdefault(find(candidate_index), []).append(candidate_index)
|
|
1439
|
+
for member_indices in clusters.values():
|
|
1440
|
+
page_indices = {candidates[index][0] for index in member_indices}
|
|
1441
|
+
if len(page_indices) < 3:
|
|
1442
|
+
continue
|
|
1443
|
+
for index in member_indices:
|
|
1444
|
+
candidates[index][1]["type"] = "header"
|
|
1445
|
+
|
|
1446
|
+
|
|
1447
|
+
def _visual_header_geometry_matches(first: BBox, second: BBox) -> bool:
|
|
1448
|
+
"""比较两个归一化页首图片的 IoU 与宽高尺度。"""
|
|
1449
|
+
|
|
1450
|
+
first_width = first[2] - first[0]
|
|
1451
|
+
first_height = first[3] - first[1]
|
|
1452
|
+
second_width = second[2] - second[0]
|
|
1453
|
+
second_height = second[3] - second[1]
|
|
1454
|
+
if min(first_width, first_height, second_width, second_height) <= 0:
|
|
1455
|
+
return False
|
|
1456
|
+
if max(first_width, second_width) / min(first_width, second_width) > 1.1:
|
|
1457
|
+
return False
|
|
1458
|
+
if max(first_height, second_height) / min(first_height, second_height) > 1.1:
|
|
1459
|
+
return False
|
|
1460
|
+
|
|
1461
|
+
intersection_width = max(0.0, min(first[2], second[2]) - max(first[0], second[0]))
|
|
1462
|
+
intersection_height = max(0.0, min(first[3], second[3]) - max(first[1], second[1]))
|
|
1463
|
+
intersection = intersection_width * intersection_height
|
|
1464
|
+
union_area = first_width * first_height + second_width * second_height - intersection
|
|
1465
|
+
return union_area > 0 and intersection / union_area >= 0.9
|
|
1466
|
+
|
|
1467
|
+
|
|
1468
|
+
def _build_marginal_candidate(
|
|
1469
|
+
page_index: int,
|
|
1470
|
+
line: _LineItem,
|
|
1471
|
+
page_size: tuple[float, float],
|
|
1472
|
+
) -> _MarginalCandidate | None:
|
|
1473
|
+
"""把页面上下百分之十五内的常规小行转换成跨页比较候选。"""
|
|
1474
|
+
|
|
1475
|
+
if line.semantic_type not in {None, "page_footnote"}:
|
|
1476
|
+
return None
|
|
1477
|
+
local_bbox = _rotate_bbox_to_upright(line.bbox, page_size, line.angle)
|
|
1478
|
+
local_page_size = (page_size[1], page_size[0]) if line.angle in {90, 270} else page_size
|
|
1479
|
+
local_page_width, local_page_height = local_page_size
|
|
1480
|
+
if local_page_width <= 0 or local_page_height <= 0:
|
|
1481
|
+
return None
|
|
1482
|
+
normalized_center_y = _bbox_center_y(local_bbox) / local_page_height
|
|
1483
|
+
normalized_center_x = _bbox_center_x(local_bbox) / local_page_width
|
|
1484
|
+
if line.semantic_type == "page_footnote" and normalized_center_y < 0.94:
|
|
1485
|
+
# 只允许极底部脚注重新参加跨页强证据匹配,正文脚注继续保留原类型。
|
|
1486
|
+
return None
|
|
1487
|
+
if normalized_center_y <= 0.15:
|
|
1488
|
+
region: Literal["header", "footer", "side"] = "header"
|
|
1489
|
+
elif normalized_center_y >= 0.9:
|
|
1490
|
+
region = "footer"
|
|
1491
|
+
elif (
|
|
1492
|
+
normalized_center_y <= 0.18
|
|
1493
|
+
or normalized_center_y >= 0.82
|
|
1494
|
+
or (
|
|
1495
|
+
(normalized_center_x <= 0.15 or normalized_center_x >= 0.85)
|
|
1496
|
+
and (normalized_center_y <= 0.3 or normalized_center_y >= 0.7)
|
|
1497
|
+
)
|
|
1498
|
+
):
|
|
1499
|
+
# 仅页码递增逻辑会消费 side;稳定文本不会被侧栏位置猜成页眉页脚。
|
|
1500
|
+
region = "side"
|
|
1501
|
+
else:
|
|
1502
|
+
return None
|
|
1503
|
+
if _line_effective_height(line, local_bbox) > 0.06 * local_page_height:
|
|
1504
|
+
return None
|
|
1505
|
+
return _MarginalCandidate(
|
|
1506
|
+
page_index=page_index,
|
|
1507
|
+
line=line,
|
|
1508
|
+
local_bbox=local_bbox,
|
|
1509
|
+
local_page_size=local_page_size,
|
|
1510
|
+
region=region,
|
|
1511
|
+
)
|
|
1512
|
+
|
|
1513
|
+
|
|
1514
|
+
def _page_number_candidates_match(
|
|
1515
|
+
first: _MarginalCandidate,
|
|
1516
|
+
second: _MarginalCandidate,
|
|
1517
|
+
) -> bool:
|
|
1518
|
+
"""校验连续页码的同边缘几何,横竖版切换时允许边缘位置随版面改变。"""
|
|
1519
|
+
|
|
1520
|
+
if _marginal_geometry_matches(first, second):
|
|
1521
|
+
return True
|
|
1522
|
+
first_landscape = first.local_page_size[0] > first.local_page_size[1]
|
|
1523
|
+
second_landscape = second.local_page_size[0] > second.local_page_size[1]
|
|
1524
|
+
if first_landscape == second_landscape or first.line.angle != second.line.angle:
|
|
1525
|
+
return False
|
|
1526
|
+
first_height = _line_effective_height(first.line, first.local_bbox) / first.local_page_size[1]
|
|
1527
|
+
second_height = _line_effective_height(second.line, second.local_bbox) / second.local_page_size[1]
|
|
1528
|
+
return (
|
|
1529
|
+
min(first_height, second_height) > 0
|
|
1530
|
+
and max(first_height, second_height)
|
|
1531
|
+
/ min(
|
|
1532
|
+
first_height,
|
|
1533
|
+
second_height,
|
|
1534
|
+
)
|
|
1535
|
+
<= 1.5
|
|
1536
|
+
)
|
|
1537
|
+
|
|
1538
|
+
|
|
1539
|
+
def _marginal_geometry_matches(
|
|
1540
|
+
first: _MarginalCandidate,
|
|
1541
|
+
second: _MarginalCandidate,
|
|
1542
|
+
) -> bool:
|
|
1543
|
+
"""比较边缘候选的方向、纵向带、字号以及同侧或镜像横向位置。"""
|
|
1544
|
+
|
|
1545
|
+
if first.region != second.region or first.line.angle != second.line.angle:
|
|
1546
|
+
return False
|
|
1547
|
+
first_width, first_height = first.local_page_size
|
|
1548
|
+
second_width, second_height = second.local_page_size
|
|
1549
|
+
first_y = _bbox_center_y(first.local_bbox) / first_height
|
|
1550
|
+
second_y = _bbox_center_y(second.local_bbox) / second_height
|
|
1551
|
+
if abs(first_y - second_y) > 0.025:
|
|
1552
|
+
return False
|
|
1553
|
+
first_line_height = _line_effective_height(first.line, first.local_bbox) / first_height
|
|
1554
|
+
second_line_height = _line_effective_height(second.line, second.local_bbox) / second_height
|
|
1555
|
+
if (
|
|
1556
|
+
min(first_line_height, second_line_height) <= 0
|
|
1557
|
+
or max(first_line_height, second_line_height)
|
|
1558
|
+
/ min(
|
|
1559
|
+
first_line_height,
|
|
1560
|
+
second_line_height,
|
|
1561
|
+
)
|
|
1562
|
+
> 1.35
|
|
1563
|
+
):
|
|
1564
|
+
return False
|
|
1565
|
+
if (
|
|
1566
|
+
first.line.font_signature is not None
|
|
1567
|
+
and second.line.font_signature is not None
|
|
1568
|
+
and first.line.font_coverage >= 0.75
|
|
1569
|
+
and second.line.font_coverage >= 0.75
|
|
1570
|
+
and first.line.font_signature != second.line.font_signature
|
|
1571
|
+
):
|
|
1572
|
+
return False
|
|
1573
|
+
|
|
1574
|
+
first_normalized_bbox = (
|
|
1575
|
+
first.local_bbox[0] / first_width,
|
|
1576
|
+
first.local_bbox[1] / first_height,
|
|
1577
|
+
first.local_bbox[2] / first_width,
|
|
1578
|
+
first.local_bbox[3] / first_height,
|
|
1579
|
+
)
|
|
1580
|
+
second_normalized_bbox = (
|
|
1581
|
+
second.local_bbox[0] / second_width,
|
|
1582
|
+
second.local_bbox[1] / second_height,
|
|
1583
|
+
second.local_bbox[2] / second_width,
|
|
1584
|
+
second.local_bbox[3] / second_height,
|
|
1585
|
+
)
|
|
1586
|
+
same_side = (
|
|
1587
|
+
_bbox_axis_overlap_ratio(first_normalized_bbox, second_normalized_bbox, axis="x") >= 0.4
|
|
1588
|
+
or abs(_bbox_center_x(first_normalized_bbox) - _bbox_center_x(second_normalized_bbox)) <= 0.08
|
|
1589
|
+
)
|
|
1590
|
+
mirrored = abs(_bbox_center_x(first_normalized_bbox) + _bbox_center_x(second_normalized_bbox) - 1.0) <= 0.12
|
|
1591
|
+
return same_side or mirrored
|
|
1592
|
+
|
|
1593
|
+
|
|
1594
|
+
def _parse_page_number_value(text: str) -> int | None:
|
|
1595
|
+
"""解析整行阿拉伯、罗马或中文页码;混有稳定正文的行不作为纯页码。"""
|
|
1596
|
+
|
|
1597
|
+
normalized = unicodedata.normalize("NFKC", str(text or ""))
|
|
1598
|
+
match = _PAGE_NUMBER_RE.fullmatch(normalized)
|
|
1599
|
+
if match is None:
|
|
1600
|
+
return None
|
|
1601
|
+
value = match.group("value")
|
|
1602
|
+
if value.isdecimal():
|
|
1603
|
+
return int(value)
|
|
1604
|
+
if re.fullmatch(r"[ivxlcdm]+", value, re.IGNORECASE):
|
|
1605
|
+
return _roman_number_to_int(value)
|
|
1606
|
+
return _chinese_page_number_to_int(value)
|
|
1607
|
+
|
|
1608
|
+
|
|
1609
|
+
def _roman_number_to_int(value: str) -> int | None:
|
|
1610
|
+
"""把页码中的规范罗马数字转换成整数,非法组合返回空。"""
|
|
1611
|
+
|
|
1612
|
+
roman_values = {"I": 1, "V": 5, "X": 10, "L": 50, "C": 100, "D": 500, "M": 1000}
|
|
1613
|
+
normalized = value.upper()
|
|
1614
|
+
total = 0
|
|
1615
|
+
previous = 0
|
|
1616
|
+
for char in reversed(normalized):
|
|
1617
|
+
current = roman_values.get(char)
|
|
1618
|
+
if current is None:
|
|
1619
|
+
return None
|
|
1620
|
+
total += -current if current < previous else current
|
|
1621
|
+
previous = max(previous, current)
|
|
1622
|
+
if total <= 0 or total > 4999:
|
|
1623
|
+
return None
|
|
1624
|
+
return total
|
|
1625
|
+
|
|
1626
|
+
|
|
1627
|
+
def _chinese_page_number_to_int(value: str) -> int | None:
|
|
1628
|
+
"""把常见百位以内中文页码转换成整数,供跨页递增校验使用。"""
|
|
1629
|
+
|
|
1630
|
+
digits = {"〇": 0, "零": 0, "一": 1, "二": 2, "两": 2, "三": 3, "四": 4, "五": 5, "六": 6, "七": 7, "八": 8, "九": 9}
|
|
1631
|
+
if all(char in digits for char in value):
|
|
1632
|
+
try:
|
|
1633
|
+
return int("".join(str(digits[char]) for char in value))
|
|
1634
|
+
except ValueError:
|
|
1635
|
+
return None
|
|
1636
|
+
total = 0
|
|
1637
|
+
current_digit = 0
|
|
1638
|
+
for char in value:
|
|
1639
|
+
if char in digits:
|
|
1640
|
+
current_digit = digits[char]
|
|
1641
|
+
elif char == "十":
|
|
1642
|
+
total += (current_digit or 1) * 10
|
|
1643
|
+
current_digit = 0
|
|
1644
|
+
elif char == "百":
|
|
1645
|
+
total += (current_digit or 1) * 100
|
|
1646
|
+
current_digit = 0
|
|
1647
|
+
else:
|
|
1648
|
+
return None
|
|
1649
|
+
return total + current_digit if total + current_digit > 0 else None
|
|
1650
|
+
|
|
1651
|
+
|
|
1652
|
+
def _marginal_text_matches(first_text: str, second_text: str) -> bool:
|
|
1653
|
+
"""在屏蔽变化数字后比较边缘稳定文本,短文本只接受完全一致。"""
|
|
1654
|
+
|
|
1655
|
+
first = _normalize_marginal_text(first_text)
|
|
1656
|
+
second = _normalize_marginal_text(second_text)
|
|
1657
|
+
if not first or not second:
|
|
1658
|
+
return False
|
|
1659
|
+
if first == second:
|
|
1660
|
+
return True
|
|
1661
|
+
if min(len(first), len(second)) < 8:
|
|
1662
|
+
return False
|
|
1663
|
+
return SequenceMatcher(a=first, b=second, autojunk=False).ratio() >= 0.92
|
|
1664
|
+
|
|
1665
|
+
|
|
1666
|
+
def _normalize_marginal_text(text: str) -> str:
|
|
1667
|
+
"""统一边缘重复文本的宽窄字符、大小写、空白和可变数字。"""
|
|
1668
|
+
|
|
1669
|
+
normalized = unicodedata.normalize("NFKC", str(text or "")).casefold()
|
|
1670
|
+
normalized = re.sub(r"\d+", "#", normalized)
|
|
1671
|
+
return re.sub(r"\s+", "", normalized).strip()
|