docvortex 0.2.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- docvortex/__init__.py +21 -0
- docvortex/analyzers/__init__.py +3 -0
- docvortex/analyzers/native/__init__.py +37 -0
- docvortex/analyzers/native/_shared/__init__.py +3 -0
- docvortex/analyzers/native/_shared/hyperlink.py +16 -0
- docvortex/analyzers/native/_shared/image.py +8 -0
- docvortex/analyzers/native/_shared/markup/__init__.py +47 -0
- docvortex/analyzers/native/_shared/markup/anchors.py +30 -0
- docvortex/analyzers/native/_shared/markup/formula.py +37 -0
- docvortex/analyzers/native/_shared/markup/projector.py +57 -0
- docvortex/analyzers/native/_shared/markup/styles.py +19 -0
- docvortex/analyzers/native/_shared/mathml.py +17 -0
- docvortex/analyzers/native/_shared/names.py +7 -0
- docvortex/analyzers/native/_shared/xycut.py +414 -0
- docvortex/analyzers/native/contracts.py +44 -0
- docvortex/analyzers/native/csv.py +351 -0
- docvortex/analyzers/native/epub/__init__.py +18 -0
- docvortex/analyzers/native/epub/constants.py +45 -0
- docvortex/analyzers/native/epub/converter.py +80 -0
- docvortex/analyzers/native/epub/errors.py +20 -0
- docvortex/analyzers/native/epub/metadata.py +31 -0
- docvortex/analyzers/native/epub/package.py +649 -0
- docvortex/analyzers/native/epub/xhtml.py +306 -0
- docvortex/analyzers/native/html/__init__.py +6 -0
- docvortex/analyzers/native/html/anchors.py +271 -0
- docvortex/analyzers/native/html/constants.py +25 -0
- docvortex/analyzers/native/html/contracts.py +7 -0
- docvortex/analyzers/native/html/converter.py +117 -0
- docvortex/analyzers/native/html/document.py +389 -0
- docvortex/analyzers/native/html/errors.py +12 -0
- docvortex/analyzers/native/html/resources.py +365 -0
- docvortex/analyzers/native/html/selector.py +421 -0
- docvortex/analyzers/native/models.py +209 -0
- docvortex/analyzers/native/ofd/__init__.py +14 -0
- docvortex/analyzers/native/ofd/constants.py +59 -0
- docvortex/analyzers/native/ofd/converter.py +31 -0
- docvortex/analyzers/native/ofd/errors.py +16 -0
- docvortex/analyzers/native/ofd/geometry.py +215 -0
- docvortex/analyzers/native/ofd/images.py +124 -0
- docvortex/analyzers/native/ofd/metadata.py +53 -0
- docvortex/analyzers/native/ofd/models.py +170 -0
- docvortex/analyzers/native/ofd/package.py +346 -0
- docvortex/analyzers/native/ofd/path.py +201 -0
- docvortex/analyzers/native/ofd/reading_order.py +293 -0
- docvortex/analyzers/native/ofd/resources.py +122 -0
- docvortex/analyzers/native/ofd/scene.py +347 -0
- docvortex/analyzers/native/ofd/table.py +240 -0
- docvortex/analyzers/native/ofd/text.py +521 -0
- docvortex/analyzers/native/office/__init__.py +3 -0
- docvortex/analyzers/native/office/doc/__init__.py +3 -0
- docvortex/analyzers/native/office/doc/bookmarks.py +81 -0
- docvortex/analyzers/native/office/doc/doc_converter.py +573 -0
- docvortex/analyzers/native/office/doc/fib.py +261 -0
- docvortex/analyzers/native/office/doc/fields.py +101 -0
- docvortex/analyzers/native/office/doc/formatting.py +171 -0
- docvortex/analyzers/native/office/doc/images.py +162 -0
- docvortex/analyzers/native/office/doc/lists.py +356 -0
- docvortex/analyzers/native/office/doc/models.py +164 -0
- docvortex/analyzers/native/office/doc/parser.py +851 -0
- docvortex/analyzers/native/office/doc/pieces.py +256 -0
- docvortex/analyzers/native/office/doc/records.py +40 -0
- docvortex/analyzers/native/office/doc/sprm.py +269 -0
- docvortex/analyzers/native/office/doc/styles.py +215 -0
- docvortex/analyzers/native/office/docx/__init__.py +3 -0
- docvortex/analyzers/native/office/docx/context.py +71 -0
- docvortex/analyzers/native/office/docx/docx_converter.py +614 -0
- docvortex/analyzers/native/office/docx/equationxml.py +119 -0
- docvortex/analyzers/native/office/docx/fields.py +816 -0
- docvortex/analyzers/native/office/docx/formatting_types.py +23 -0
- docvortex/analyzers/native/office/docx/main.py +50 -0
- docvortex/analyzers/native/office/docx/numbering.py +491 -0
- docvortex/analyzers/native/office/docx/office_xml.py +57 -0
- docvortex/analyzers/native/office/docx/package_normalizer.py +248 -0
- docvortex/analyzers/native/office/docx/resources.py +526 -0
- docvortex/analyzers/native/office/docx/styles.py +693 -0
- docvortex/analyzers/native/office/docx/tables.py +515 -0
- docvortex/analyzers/native/office/equation/__init__.py +3 -0
- docvortex/analyzers/native/office/equation/image.py +470 -0
- docvortex/analyzers/native/office/equation/latex_dict.py +324 -0
- docvortex/analyzers/native/office/equation/mtef.py +885 -0
- docvortex/analyzers/native/office/equation/mtef_v5.py +941 -0
- docvortex/analyzers/native/office/equation/omml.py +561 -0
- docvortex/analyzers/native/office/equation/ooxml.py +62 -0
- docvortex/analyzers/native/office/errors.py +33 -0
- docvortex/analyzers/native/office/image.py +307 -0
- docvortex/analyzers/native/office/legacy/__init__.py +3 -0
- docvortex/analyzers/native/office/legacy/binary.py +43 -0
- docvortex/analyzers/native/office/legacy/officeart.py +362 -0
- docvortex/analyzers/native/office/legacy/ole.py +110 -0
- docvortex/analyzers/native/office/limits.py +12 -0
- docvortex/analyzers/native/office/odf/__init__.py +3 -0
- docvortex/analyzers/native/office/odf/chart.py +76 -0
- docvortex/analyzers/native/office/odf/constants.py +70 -0
- docvortex/analyzers/native/office/odf/converters.py +403 -0
- docvortex/analyzers/native/office/odf/errors.py +18 -0
- docvortex/analyzers/native/office/odf/metadata.py +91 -0
- docvortex/analyzers/native/office/odf/models.py +176 -0
- docvortex/analyzers/native/office/odf/package.py +270 -0
- docvortex/analyzers/native/office/odf/styles.py +329 -0
- docvortex/analyzers/native/office/odf/table.py +469 -0
- docvortex/analyzers/native/office/odf/text.py +1002 -0
- docvortex/analyzers/native/office/ooxml_chart.py +1016 -0
- docvortex/analyzers/native/office/opc.py +38 -0
- docvortex/analyzers/native/office/ppt/__init__.py +3 -0
- docvortex/analyzers/native/office/ppt/models.py +118 -0
- docvortex/analyzers/native/office/ppt/parser.py +1895 -0
- docvortex/analyzers/native/office/ppt/ppt_converter.py +292 -0
- docvortex/analyzers/native/office/ppt/records.py +131 -0
- docvortex/analyzers/native/office/ppt/style_text.py +247 -0
- docvortex/analyzers/native/office/pptx/__init__.py +3 -0
- docvortex/analyzers/native/office/pptx/context.py +113 -0
- docvortex/analyzers/native/office/pptx/lists.py +558 -0
- docvortex/analyzers/native/office/pptx/main.py +20 -0
- docvortex/analyzers/native/office/pptx/package_normalizer.py +321 -0
- docvortex/analyzers/native/office/pptx/pptx_converter.py +323 -0
- docvortex/analyzers/native/office/pptx/resources.py +329 -0
- docvortex/analyzers/native/office/pptx/shapes.py +393 -0
- docvortex/analyzers/native/office/pptx/text_styles.py +625 -0
- docvortex/analyzers/native/office/pptx/titles.py +178 -0
- docvortex/analyzers/native/office/rich_text.py +420 -0
- docvortex/analyzers/native/office/rtf/__init__.py +3 -0
- docvortex/analyzers/native/office/rtf/converter.py +708 -0
- docvortex/analyzers/native/office/rtf/lexer.py +213 -0
- docvortex/analyzers/native/office/rtf/math.py +339 -0
- docvortex/analyzers/native/office/rtf/models.py +185 -0
- docvortex/analyzers/native/office/rtf/parser.py +1553 -0
- docvortex/analyzers/native/office/spreadsheet/__init__.py +3 -0
- docvortex/analyzers/native/office/spreadsheet/html.py +81 -0
- docvortex/analyzers/native/office/spreadsheet/models.py +79 -0
- docvortex/analyzers/native/office/spreadsheet/projector.py +928 -0
- docvortex/analyzers/native/office/streams.py +18 -0
- docvortex/analyzers/native/office/xls/__init__.py +3 -0
- docvortex/analyzers/native/office/xls/chart.py +132 -0
- docvortex/analyzers/native/office/xls/embedded_chart.py +299 -0
- docvortex/analyzers/native/office/xls/models.py +109 -0
- docvortex/analyzers/native/office/xls/number_format.py +521 -0
- docvortex/analyzers/native/office/xls/parser.py +1145 -0
- docvortex/analyzers/native/office/xls/records.py +201 -0
- docvortex/analyzers/native/office/xls/strings.py +205 -0
- docvortex/analyzers/native/office/xls/xls_converter.py +354 -0
- docvortex/analyzers/native/office/xlsx/__init__.py +3 -0
- docvortex/analyzers/native/office/xlsx/main.py +20 -0
- docvortex/analyzers/native/office/xlsx/ooxml_ole.py +522 -0
- docvortex/analyzers/native/office/xlsx/package_normalizer.py +310 -0
- docvortex/analyzers/native/office/xlsx/xlsx_converter.py +716 -0
- docvortex/analyzers/native/pdf/__init__.py +3 -0
- docvortex/analyzers/native/pdf/auxiliary_text.py +1671 -0
- docvortex/analyzers/native/pdf/char_geometry.py +1662 -0
- docvortex/analyzers/native/pdf/code_blocks.py +535 -0
- docvortex/analyzers/native/pdf/formulas.py +1985 -0
- docvortex/analyzers/native/pdf/geometry.py +281 -0
- docvortex/analyzers/native/pdf/graphics.py +1501 -0
- docvortex/analyzers/native/pdf/index_blocks.py +268 -0
- docvortex/analyzers/native/pdf/inline/__init__.py +3 -0
- docvortex/analyzers/native/pdf/inline/common.py +112 -0
- docvortex/analyzers/native/pdf/inline/detection.py +590 -0
- docvortex/analyzers/native/pdf/inline/matching.py +1185 -0
- docvortex/analyzers/native/pdf/inline/materialize.py +457 -0
- docvortex/analyzers/native/pdf/inline/scripts.py +975 -0
- docvortex/analyzers/native/pdf/inline/types.py +385 -0
- docvortex/analyzers/native/pdf/line_layout.py +1106 -0
- docvortex/analyzers/native/pdf/line_merging.py +1223 -0
- docvortex/analyzers/native/pdf/models.py +246 -0
- docvortex/analyzers/native/pdf/native_text.py +1004 -0
- docvortex/analyzers/native/pdf/pipeline.py +1390 -0
- docvortex/analyzers/native/pdf/script_geometry.py +636 -0
- docvortex/analyzers/native/pdf/shared.py +30 -0
- docvortex/analyzers/native/pdf/spatial_text.py +383 -0
- docvortex/analyzers/native/pdf/table_annotations.py +446 -0
- docvortex/analyzers/native/pdf/table_constants.py +48 -0
- docvortex/analyzers/native/pdf/table_detection.py +227 -0
- docvortex/analyzers/native/pdf/table_filled_grid.py +218 -0
- docvortex/analyzers/native/pdf/table_geometry.py +40 -0
- docvortex/analyzers/native/pdf/table_materialization.py +444 -0
- docvortex/analyzers/native/pdf/table_recovery/__init__.py +15 -0
- docvortex/analyzers/native/pdf/table_recovery/candidate.py +356 -0
- docvortex/analyzers/native/pdf/table_recovery/contracts.py +154 -0
- docvortex/analyzers/native/pdf/table_recovery/engine.py +609 -0
- docvortex/analyzers/native/pdf/table_recovery/geometry.py +164 -0
- docvortex/analyzers/native/pdf/table_recovery/sparse_common.py +85 -0
- docvortex/analyzers/native/pdf/table_recovery/sparse_hybrid.py +804 -0
- docvortex/analyzers/native/pdf/table_recovery/sparse_multiline.py +1122 -0
- docvortex/analyzers/native/pdf/table_recovery/text.py +414 -0
- docvortex/analyzers/native/pdf/table_recovery/text_grid.py +618 -0
- docvortex/analyzers/native/pdf/table_recovery/vector.py +1931 -0
- docvortex/analyzers/native/pdf/table_rows.py +34 -0
- docvortex/analyzers/native/pdf/table_rules.py +1129 -0
- docvortex/analyzers/native/pdf/table_text_styles.py +283 -0
- docvortex/analyzers/native/pdf/tables.py +147 -0
- docvortex/analyzers/native/pdf/text_assembly/__init__.py +3 -0
- docvortex/analyzers/native/pdf/text_assembly/annotations.py +581 -0
- docvortex/analyzers/native/pdf/text_assembly/assembly.py +292 -0
- docvortex/analyzers/native/pdf/text_assembly/common.py +477 -0
- docvortex/analyzers/native/pdf/text_assembly/footnotes.py +394 -0
- docvortex/analyzers/native/pdf/text_assembly/merging.py +1274 -0
- docvortex/analyzers/native/pdf/text_assembly/rows.py +692 -0
- docvortex/analyzers/native/pdf/text_blocks.py +82 -0
- docvortex/analyzers/native/pdf/text_styles.py +55 -0
- docvortex/analyzers/native/pdf/title_analysis/__init__.py +3 -0
- docvortex/analyzers/native/pdf/title_analysis/body_profile.py +215 -0
- docvortex/analyzers/native/pdf/title_analysis/common.py +117 -0
- docvortex/analyzers/native/pdf/title_analysis/document_profile.py +164 -0
- docvortex/analyzers/native/pdf/title_analysis/lane_titles.py +758 -0
- docvortex/analyzers/native/pdf/title_analysis/page_titles.py +1024 -0
- docvortex/analyzers/native/pdf/title_analysis/prototype.py +194 -0
- docvortex/analyzers/native/pdf/title_analysis/structural.py +1081 -0
- docvortex/analyzers/native/pdf/titles.py +75 -0
- docvortex/analyzers/native/pdf/typography.py +19 -0
- docvortex/analyzers/native/pdf/visual_annotations.py +1262 -0
- docvortex/api.py +180 -0
- docvortex/assets/__init__.py +5 -0
- docvortex/assets/store.py +51 -0
- docvortex/cli.py +54 -0
- docvortex/codecs/__init__.py +3 -0
- docvortex/codecs/html/__init__.py +22 -0
- docvortex/codecs/html/contracts.py +236 -0
- docvortex/codecs/html/materializer.py +331 -0
- docvortex/codecs/html/parser.py +763 -0
- docvortex/codecs/html/resources.py +34 -0
- docvortex/codecs/json.py +17 -0
- docvortex/content/__init__.py +5 -0
- docvortex/content/inline.py +248 -0
- docvortex/content/markup/__init__.py +44 -0
- docvortex/content/markup/anchors.py +188 -0
- docvortex/content/markup/formula.py +280 -0
- docvortex/content/markup/projector.py +1237 -0
- docvortex/content/markup/styles.py +327 -0
- docvortex/content/mathml.py +167 -0
- docvortex/content/normalization.py +188 -0
- docvortex/content/spans.py +183 -0
- docvortex/content/table/__init__.py +18 -0
- docvortex/content/table/blocks.py +152 -0
- docvortex/content/table/content.py +425 -0
- docvortex/content/table/document.py +104 -0
- docvortex/content/table/html.py +399 -0
- docvortex/content/table/models.py +76 -0
- docvortex/content/table/rules.py +42 -0
- docvortex/content/table/structure.py +221 -0
- docvortex/content/tree.py +5 -0
- docvortex/document/__init__.py +3 -0
- docvortex/document/contracts.py +22 -0
- docvortex/document/detection.py +389 -0
- docvortex/document/filetypes.py +175 -0
- docvortex/document/page_range.py +167 -0
- docvortex/document/pdf/__init__.py +19 -0
- docvortex/document/pdf/classify.py +1138 -0
- docvortex/document/pdf/constants.py +132 -0
- docvortex/document/pdf/diagnostics.py +350 -0
- docvortex/document/pdf/document.py +582 -0
- docvortex/document/pdf/font_runtime.py +335 -0
- docvortex/document/pdf/geometry.py +31 -0
- docvortex/document/pdf/images.py +654 -0
- docvortex/document/pdf/native_annotations.py +367 -0
- docvortex/document/pdf/native_contracts.py +169 -0
- docvortex/document/pdf/native_coordinates.py +216 -0
- docvortex/document/pdf/native_lifecycle.py +16 -0
- docvortex/document/pdf/native_objects.py +902 -0
- docvortex/document/pdf/native_text_geometry.py +315 -0
- docvortex/document/pdf/pdfium.py +325 -0
- docvortex/document/pdf/raster.py +46 -0
- docvortex/document/pdf/text/__init__.py +62 -0
- docvortex/document/pdf/text/contracts.py +211 -0
- docvortex/document/pdf/text/extract.py +165 -0
- docvortex/document/pdf/text/geometry.py +16 -0
- docvortex/document/pdf/text/groups.py +162 -0
- docvortex/document/pdf/visual_geometry.py +201 -0
- docvortex/document/pdf/visuals.py +343 -0
- docvortex/document/source.py +92 -0
- docvortex/errors.py +21 -0
- docvortex/export/__init__.py +3 -0
- docvortex/export/bundle.py +98 -0
- docvortex/export/files.py +66 -0
- docvortex/export/middle.py +208 -0
- docvortex/foundation/__init__.py +3 -0
- docvortex/foundation/geometry.py +125 -0
- docvortex/foundation/hyperlink.py +65 -0
- docvortex/foundation/image.py +48 -0
- docvortex/foundation/image_encoding.py +30 -0
- docvortex/foundation/image_payload.py +280 -0
- docvortex/foundation/language.py +92 -0
- docvortex/foundation/platform.py +38 -0
- docvortex/foundation/text.py +153 -0
- docvortex/foundation/type_identity.py +20 -0
- docvortex/foundation/xml_names.py +20 -0
- docvortex/options.py +30 -0
- docvortex/postprocess/__init__.py +3 -0
- docvortex/postprocess/content.py +53 -0
- docvortex/postprocess/document.py +19 -0
- docvortex/postprocess/lists.py +236 -0
- docvortex/postprocess/page_blocks.py +214 -0
- docvortex/postprocess/pages.py +95 -0
- docvortex/postprocess/paragraphs.py +580 -0
- docvortex/postprocess/visual.py +715 -0
- docvortex/render/__init__.py +48 -0
- docvortex/render/_internal/__init__.py +3 -0
- docvortex/render/_internal/common/__init__.py +3 -0
- docvortex/render/_internal/common/context.py +43 -0
- docvortex/render/_internal/common/html_table.py +178 -0
- docvortex/render/_internal/common/index.py +33 -0
- docvortex/render/_internal/common/list_items.py +158 -0
- docvortex/render/_internal/common/planner.py +140 -0
- docvortex/render/_internal/docx/__init__.py +3 -0
- docvortex/render/_internal/docx/assets.py +202 -0
- docvortex/render/_internal/docx/inline.py +434 -0
- docvortex/render/_internal/docx/math.py +220 -0
- docvortex/render/_internal/docx/renderer.py +905 -0
- docvortex/render/_internal/docx/styles.py +195 -0
- docvortex/render/_internal/docx/table.py +442 -0
- docvortex/render/_internal/epub/__init__.py +5 -0
- docvortex/render/_internal/epub/assets.py +173 -0
- docvortex/render/_internal/epub/package.py +249 -0
- docvortex/render/_internal/epub/renderer.py +1156 -0
- docvortex/render/_internal/html/__init__.py +3 -0
- docvortex/render/_internal/html/inline.py +346 -0
- docvortex/render/_internal/html/renderer.py +1041 -0
- docvortex/render/_internal/html/sanitizer.py +478 -0
- docvortex/render/_internal/html/table.py +121 -0
- docvortex/render/_internal/latex/__init__.py +1 -0
- docvortex/render/_internal/latex/assets.py +85 -0
- docvortex/render/_internal/latex/inline.py +145 -0
- docvortex/render/_internal/latex/renderer.py +506 -0
- docvortex/render/_internal/latex/table.py +347 -0
- docvortex/render/_internal/markdown/__init__.py +3 -0
- docvortex/render/_internal/markdown/assets.py +78 -0
- docvortex/render/_internal/markdown/blocks.py +635 -0
- docvortex/render/_internal/markdown/escaping.py +51 -0
- docvortex/render/_internal/markdown/inline.py +260 -0
- docvortex/render/_internal/markdown/renderer.py +93 -0
- docvortex/render/_internal/markdown/table.py +281 -0
- docvortex/render/_internal/pdf/__init__.py +3 -0
- docvortex/render/_internal/pdf/assets.py +197 -0
- docvortex/render/_internal/pdf/formula.py +417 -0
- docvortex/render/_internal/pdf/inline.py +343 -0
- docvortex/render/_internal/pdf/renderer.py +734 -0
- docvortex/render/_internal/pdf/styles.py +206 -0
- docvortex/render/_internal/pdf/table.py +272 -0
- docvortex/render/_internal/structured_content/__init__.py +3 -0
- docvortex/render/_internal/structured_content/renderer.py +193 -0
- docvortex/render/api.py +199 -0
- docvortex/render/contracts.py +205 -0
- docvortex/render/docx.py +38 -0
- docvortex/render/epub.py +35 -0
- docvortex/render/fragments.py +70 -0
- docvortex/render/html.py +29 -0
- docvortex/render/latex.py +24 -0
- docvortex/render/markdown.py +50 -0
- docvortex/render/pdf.py +25 -0
- docvortex/render/structured_content.py +23 -0
- docvortex/resources/epub/docvortex.css +91 -0
- docvortex/resources/fasttext-langdetect/lid.176.ftz +0 -0
- docvortex/resources/fonts/DroidSansFallbackFull.ttf +0 -0
- docvortex/resources/fonts/NOTICE +190 -0
- docvortex/resources/fonts/manifest.json +9 -0
- docvortex/resources/html/docvortex.css +601 -0
- docvortex/resources/html/docvortex.min.css +1 -0
- docvortex/result.py +91 -0
- docvortex/schema.py +1141 -0
- docvortex/version.py +3 -0
- docvortex-0.2.1.dist-info/METADATA +193 -0
- docvortex-0.2.1.dist-info/RECORD +364 -0
- docvortex-0.2.1.dist-info/WHEEL +5 -0
- docvortex-0.2.1.dist-info/entry_points.txt +2 -0
- docvortex-0.2.1.dist-info/licenses/LICENSE.md +21 -0
- docvortex-0.2.1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,421 @@
|
|
|
1
|
+
"""基于静态 DOM 指标执行保守的 HTML 正文自动选择。"""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from copy import deepcopy
|
|
6
|
+
from dataclasses import dataclass
|
|
7
|
+
import re
|
|
8
|
+
|
|
9
|
+
from lxml import etree # type: ignore[reportMissingImports]
|
|
10
|
+
|
|
11
|
+
from docvortex.content.markup import MarkupStylesheet, TextStyle
|
|
12
|
+
from docvortex.content.markup.projector import local_name
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
_CANDIDATE_TAGS = frozenset({"article", "div", "main", "section"})
|
|
16
|
+
_SEMANTIC_TAGS = frozenset({"figure", "img", "math", "pre", "svg", "table"})
|
|
17
|
+
_PARAGRAPH_TAGS = frozenset({"dd", "dt", "li", "p", "pre"})
|
|
18
|
+
_BOILERPLATE_TAGS = frozenset({"footer", "form", "nav"})
|
|
19
|
+
_POSITIVE_TOKENS = frozenset({"article", "body", "content", "entry", "main", "post", "story", "text"})
|
|
20
|
+
_NEGATIVE_TOKENS = frozenset(
|
|
21
|
+
{
|
|
22
|
+
"advert",
|
|
23
|
+
"banner",
|
|
24
|
+
"comment",
|
|
25
|
+
"cookie",
|
|
26
|
+
"footer",
|
|
27
|
+
"menu",
|
|
28
|
+
"nav",
|
|
29
|
+
"newsletter",
|
|
30
|
+
"related",
|
|
31
|
+
"share",
|
|
32
|
+
"sidebar",
|
|
33
|
+
"social",
|
|
34
|
+
"widget",
|
|
35
|
+
}
|
|
36
|
+
)
|
|
37
|
+
_TOKEN_RE = re.compile(r"[^a-z0-9]+")
|
|
38
|
+
_MIN_TEXT_CHARS = 200
|
|
39
|
+
_MIN_SEMANTIC_OBJECTS = 2
|
|
40
|
+
_MIN_RETAINED_RATIO = 1 / 7
|
|
41
|
+
_MIN_SCORE_MARGIN = 1.25
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
@dataclass(frozen=True, slots=True)
|
|
45
|
+
class CandidateMetrics:
|
|
46
|
+
"""保存一个 DOM 子树的可见正文与噪声统计。"""
|
|
47
|
+
|
|
48
|
+
text_chars: int = 0
|
|
49
|
+
link_chars: int = 0
|
|
50
|
+
paragraph_chars: int = 0
|
|
51
|
+
heading_count: int = 0
|
|
52
|
+
semantic_count: int = 0
|
|
53
|
+
boilerplate_count: int = 0
|
|
54
|
+
|
|
55
|
+
def __add__(self, other: CandidateMetrics) -> CandidateMetrics:
|
|
56
|
+
"""合并两个子树指标。"""
|
|
57
|
+
return CandidateMetrics(
|
|
58
|
+
text_chars=self.text_chars + other.text_chars,
|
|
59
|
+
link_chars=self.link_chars + other.link_chars,
|
|
60
|
+
paragraph_chars=self.paragraph_chars + other.paragraph_chars,
|
|
61
|
+
heading_count=self.heading_count + other.heading_count,
|
|
62
|
+
semantic_count=self.semantic_count + other.semantic_count,
|
|
63
|
+
boilerplate_count=self.boilerplate_count + other.boilerplate_count,
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
@dataclass(frozen=True, slots=True)
|
|
68
|
+
class ContentSelection:
|
|
69
|
+
"""保存最终内容根、正文命中状态和可诊断的保留率。"""
|
|
70
|
+
|
|
71
|
+
root: etree._Element
|
|
72
|
+
mode_used: str
|
|
73
|
+
confidence: float
|
|
74
|
+
retained_text_ratio: float
|
|
75
|
+
reason: str
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
@dataclass(frozen=True, slots=True)
|
|
79
|
+
class _ScoredCandidate:
|
|
80
|
+
"""绑定候选元素、指标、分数和显式语义标记。"""
|
|
81
|
+
|
|
82
|
+
element: etree._Element
|
|
83
|
+
metrics: CandidateMetrics
|
|
84
|
+
score: float
|
|
85
|
+
explicit: bool
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
@dataclass(frozen=True, slots=True)
|
|
89
|
+
class _SoftPruneMetrics:
|
|
90
|
+
"""保存 soft prune 所需的规范文本段、链接文本和语义对象统计。"""
|
|
91
|
+
|
|
92
|
+
text_chars: int = 0
|
|
93
|
+
text_segments: int = 0
|
|
94
|
+
link_chars: int = 0
|
|
95
|
+
link_count: int = 0
|
|
96
|
+
semantic_count: int = 0
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def select_auto_content(body: etree._Element, stylesheet: MarkupStylesheet) -> ContentSelection:
|
|
100
|
+
"""高置信选择正文候选,任一保守门槛失败时回退完整 body。"""
|
|
101
|
+
metrics_by_element: dict[etree._Element, CandidateMetrics] = {}
|
|
102
|
+
body_metrics = _collect_metrics(body, stylesheet, metrics_by_element, TextStyle(), False, False)
|
|
103
|
+
repeated_penalties = _repeated_short_sibling_penalties(body)
|
|
104
|
+
candidates: list[_ScoredCandidate] = []
|
|
105
|
+
for element in body.iter():
|
|
106
|
+
if not isinstance(element.tag, str) or element is body:
|
|
107
|
+
continue
|
|
108
|
+
name = local_name(element)
|
|
109
|
+
metrics = metrics_by_element.get(element, CandidateMetrics())
|
|
110
|
+
explicit = _is_explicit_candidate(element)
|
|
111
|
+
if name not in _CANDIDATE_TAGS and not explicit:
|
|
112
|
+
continue
|
|
113
|
+
if not explicit and metrics.text_chars < _MIN_TEXT_CHARS and metrics.semantic_count < _MIN_SEMANTIC_OBJECTS:
|
|
114
|
+
continue
|
|
115
|
+
candidates.append(
|
|
116
|
+
_ScoredCandidate(
|
|
117
|
+
element,
|
|
118
|
+
metrics,
|
|
119
|
+
_candidate_score(element, metrics, repeated_penalties.get(element, 0)),
|
|
120
|
+
explicit,
|
|
121
|
+
)
|
|
122
|
+
)
|
|
123
|
+
candidates.sort(key=lambda item: (-item.score, _document_order(item.element)))
|
|
124
|
+
repeated_candidate_items = _repeated_candidate_items(candidates, metrics_by_element)
|
|
125
|
+
|
|
126
|
+
for candidate in candidates:
|
|
127
|
+
if candidate.score <= 0:
|
|
128
|
+
continue
|
|
129
|
+
if candidate.element in repeated_candidate_items:
|
|
130
|
+
continue
|
|
131
|
+
second = next(
|
|
132
|
+
(
|
|
133
|
+
item
|
|
134
|
+
for item in candidates
|
|
135
|
+
if item is not candidate and item.score > 0 and not _is_containment_equivalent(candidate, item)
|
|
136
|
+
),
|
|
137
|
+
None,
|
|
138
|
+
)
|
|
139
|
+
if not candidate.explicit and second is not None and candidate.score < second.score * _MIN_SCORE_MARGIN:
|
|
140
|
+
continue
|
|
141
|
+
selected = _copy_candidate_with_ancestors(candidate.element, body)
|
|
142
|
+
selected_metrics: dict[etree._Element, CandidateMetrics] = {}
|
|
143
|
+
final_metrics = _collect_metrics(selected, stylesheet, selected_metrics, TextStyle(), False, False)
|
|
144
|
+
retained_ratio = final_metrics.text_chars / max(1, body_metrics.text_chars)
|
|
145
|
+
if final_metrics.text_chars < _MIN_TEXT_CHARS and final_metrics.semantic_count < _MIN_SEMANTIC_OBJECTS:
|
|
146
|
+
continue
|
|
147
|
+
if body_metrics.text_chars >= _MIN_TEXT_CHARS and retained_ratio < _MIN_RETAINED_RATIO:
|
|
148
|
+
continue
|
|
149
|
+
if candidate.metrics.semantic_count and final_metrics.semantic_count == 0:
|
|
150
|
+
continue
|
|
151
|
+
confidence = candidate.score / max(candidate.score + (second.score if second is not None else 0), 1)
|
|
152
|
+
return ContentSelection(selected, "main", confidence, retained_ratio, "high_confidence_candidate")
|
|
153
|
+
|
|
154
|
+
return ContentSelection(deepcopy(body), "document", 0.0, 1.0, "body_fallback")
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def _copy_candidate_with_ancestors(candidate: etree._Element, body: etree._Element) -> etree._Element:
|
|
158
|
+
"""复制正文候选及其到 body 的空祖先链,保留继承样式但不带入周边正文。"""
|
|
159
|
+
selected = deepcopy(candidate)
|
|
160
|
+
selected.tail = None
|
|
161
|
+
_soft_prune(selected)
|
|
162
|
+
root = selected
|
|
163
|
+
for ancestor in candidate.iterancestors():
|
|
164
|
+
if not isinstance(ancestor.tag, str):
|
|
165
|
+
continue
|
|
166
|
+
wrapper = _empty_ancestor_wrapper(ancestor)
|
|
167
|
+
wrapper.append(root)
|
|
168
|
+
root = wrapper
|
|
169
|
+
if ancestor is body:
|
|
170
|
+
return root
|
|
171
|
+
return root
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def _empty_ancestor_wrapper(ancestor: etree._Element) -> etree._Element:
|
|
175
|
+
"""复制祖先标签和属性;非法 HTML QName 使用安全 div,避免复制整棵兄弟子树。"""
|
|
176
|
+
try:
|
|
177
|
+
wrapper = etree.Element(ancestor.tag, nsmap=ancestor.nsmap)
|
|
178
|
+
except ValueError:
|
|
179
|
+
wrapper = etree.Element("div", nsmap=ancestor.nsmap)
|
|
180
|
+
for name, value in ancestor.attrib.items():
|
|
181
|
+
try:
|
|
182
|
+
wrapper.set(name, value)
|
|
183
|
+
except ValueError:
|
|
184
|
+
continue
|
|
185
|
+
return wrapper
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def _collect_metrics(
|
|
189
|
+
element: etree._Element,
|
|
190
|
+
stylesheet: MarkupStylesheet,
|
|
191
|
+
output: dict[etree._Element, CandidateMetrics],
|
|
192
|
+
inherited: TextStyle,
|
|
193
|
+
inherited_visibility_hidden: bool,
|
|
194
|
+
inside_link: bool,
|
|
195
|
+
) -> CandidateMetrics:
|
|
196
|
+
"""单次深度优先遍历计算所有元素的可见指标,避免候选间重复扫描。"""
|
|
197
|
+
resolved = stylesheet.resolve(element, inherited, inherited_visibility_hidden)
|
|
198
|
+
if resolved.subtree_hidden:
|
|
199
|
+
output[element] = CandidateMetrics()
|
|
200
|
+
return output[element]
|
|
201
|
+
name = local_name(element)
|
|
202
|
+
current_inside_link = inside_link or name == "a"
|
|
203
|
+
own_text = 0 if resolved.visibility_hidden else len(_normalized_text(element.text))
|
|
204
|
+
metrics = CandidateMetrics(
|
|
205
|
+
text_chars=own_text,
|
|
206
|
+
link_chars=own_text if current_inside_link else 0,
|
|
207
|
+
paragraph_chars=own_text if name in _PARAGRAPH_TAGS else 0,
|
|
208
|
+
heading_count=1 if name in {"h1", "h2", "h3", "h4", "h5", "h6"} else 0,
|
|
209
|
+
semantic_count=1 if name in _SEMANTIC_TAGS else 0,
|
|
210
|
+
boilerplate_count=1 if name in _BOILERPLATE_TAGS or bool(_tokens(element) & _NEGATIVE_TOKENS) else 0,
|
|
211
|
+
)
|
|
212
|
+
for child in element:
|
|
213
|
+
if isinstance(child.tag, str):
|
|
214
|
+
metrics += _collect_metrics(
|
|
215
|
+
child,
|
|
216
|
+
stylesheet,
|
|
217
|
+
output,
|
|
218
|
+
resolved.text,
|
|
219
|
+
resolved.visibility_hidden,
|
|
220
|
+
current_inside_link,
|
|
221
|
+
)
|
|
222
|
+
if not resolved.visibility_hidden:
|
|
223
|
+
tail_chars = len(_normalized_text(child.tail))
|
|
224
|
+
metrics += CandidateMetrics(
|
|
225
|
+
text_chars=tail_chars,
|
|
226
|
+
link_chars=tail_chars if current_inside_link else 0,
|
|
227
|
+
paragraph_chars=tail_chars if name in _PARAGRAPH_TAGS else 0,
|
|
228
|
+
)
|
|
229
|
+
output[element] = metrics
|
|
230
|
+
return metrics
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def _candidate_score(element: etree._Element, metrics: CandidateMetrics, repeated_penalty: int) -> float:
|
|
234
|
+
"""按正文、结构对象、链接与模板噪声计算确定性候选分数。"""
|
|
235
|
+
tokens = _tokens(element)
|
|
236
|
+
token_bonus = 200 if tokens & _POSITIVE_TOKENS else 0
|
|
237
|
+
token_penalty = 240 if tokens & _NEGATIVE_TOKENS else 0
|
|
238
|
+
link_density = metrics.link_chars / max(1, metrics.text_chars)
|
|
239
|
+
return (
|
|
240
|
+
metrics.text_chars
|
|
241
|
+
+ metrics.paragraph_chars
|
|
242
|
+
+ metrics.heading_count * 40
|
|
243
|
+
+ metrics.semantic_count * 120
|
|
244
|
+
+ token_bonus
|
|
245
|
+
- metrics.link_chars * (1.0 + link_density)
|
|
246
|
+
- metrics.boilerplate_count * 120
|
|
247
|
+
- repeated_penalty * 80
|
|
248
|
+
- token_penalty
|
|
249
|
+
)
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def _is_explicit_candidate(element: etree._Element) -> bool:
|
|
253
|
+
"""识别标准语义 main/article/role/itemprop 正文候选。"""
|
|
254
|
+
name = local_name(element)
|
|
255
|
+
roles = frozenset((element.get("role") or "").casefold().split())
|
|
256
|
+
itemprop = frozenset((element.get("itemprop") or "").casefold().split())
|
|
257
|
+
return name in {"main", "article"} or "main" in roles or "articlebody" in itemprop
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def _tokens(element: etree._Element) -> frozenset[str]:
|
|
261
|
+
"""把 class/id 拆成完整小写 token,避免任意 substring 误判。"""
|
|
262
|
+
value = f"{element.get('id') or ''} {element.get('class') or ''}".casefold()
|
|
263
|
+
return frozenset(token for token in _TOKEN_RE.split(value) if token)
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
def _repeated_short_sibling_penalties(root: etree._Element) -> dict[etree._Element, int]:
|
|
267
|
+
"""单次后序遍历预计算各子树的重复短同级惩罚,供嵌套候选共享。"""
|
|
268
|
+
elements = [element for element in root.iter() if isinstance(element.tag, str)]
|
|
269
|
+
short_text_lengths: dict[etree._Element, int] = {}
|
|
270
|
+
penalties: dict[etree._Element, int] = {}
|
|
271
|
+
for parent in reversed(elements):
|
|
272
|
+
text_length = min(len(_normalized_text(parent.text)), 161)
|
|
273
|
+
subtree_penalty = 0
|
|
274
|
+
groups: dict[tuple[str, tuple[str, ...]], int] = {}
|
|
275
|
+
for child in parent:
|
|
276
|
+
if isinstance(child.tag, str):
|
|
277
|
+
child_text_length = short_text_lengths[child]
|
|
278
|
+
if child_text_length:
|
|
279
|
+
text_length = min(text_length + (1 if text_length else 0) + child_text_length, 161)
|
|
280
|
+
subtree_penalty += penalties[child]
|
|
281
|
+
if child_text_length <= 160 and child_text_length > 0:
|
|
282
|
+
signature = (local_name(child), tuple(sorted(_tokens(child))))
|
|
283
|
+
groups[signature] = groups.get(signature, 0) + 1
|
|
284
|
+
tail_length = len(_normalized_text(child.tail))
|
|
285
|
+
if tail_length:
|
|
286
|
+
text_length = min(text_length + (1 if text_length else 0) + tail_length, 161)
|
|
287
|
+
short_text_lengths[parent] = text_length
|
|
288
|
+
penalties[parent] = subtree_penalty + sum(count - 2 for count in groups.values() if count > 2)
|
|
289
|
+
return penalties
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
def _repeated_candidate_items(
|
|
293
|
+
candidates: list[_ScoredCandidate],
|
|
294
|
+
metrics_by_element: dict[etree._Element, CandidateMetrics],
|
|
295
|
+
) -> frozenset[etree._Element]:
|
|
296
|
+
"""一次预计算论坛、文档页或聚合页中的重复同级候选,避免逐候选重扫兄弟节点。"""
|
|
297
|
+
semantic_tokens = frozenset({"article", "content", "entry", "post", "section"})
|
|
298
|
+
parents = {parent for candidate in candidates if (parent := candidate.element.getparent()) is not None}
|
|
299
|
+
repeated: set[etree._Element] = set()
|
|
300
|
+
for parent in parents:
|
|
301
|
+
eligible = [
|
|
302
|
+
child
|
|
303
|
+
for child in parent
|
|
304
|
+
if isinstance(child.tag, str) and metrics_by_element.get(child, CandidateMetrics()).text_chars >= 100
|
|
305
|
+
]
|
|
306
|
+
names = {child: local_name(child) for child in eligible}
|
|
307
|
+
tokens = {child: _tokens(child) & semantic_tokens for child in eligible}
|
|
308
|
+
name_counts: dict[str, int] = {}
|
|
309
|
+
token_counts: dict[str, int] = {}
|
|
310
|
+
for child in eligible:
|
|
311
|
+
name_counts[names[child]] = name_counts.get(names[child], 0) + 1
|
|
312
|
+
for token in tokens[child]:
|
|
313
|
+
token_counts[token] = token_counts.get(token, 0) + 1
|
|
314
|
+
repeated.update(
|
|
315
|
+
child
|
|
316
|
+
for child in eligible
|
|
317
|
+
if name_counts[names[child]] >= 2 or any(token_counts[token] >= 2 for token in tokens[child])
|
|
318
|
+
)
|
|
319
|
+
return frozenset(repeated)
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
def _is_containment_equivalent(first: _ScoredCandidate, second: _ScoredCandidate) -> bool:
|
|
323
|
+
"""忽略高度重叠的祖先/后代候选,避免嵌套 main/article 互相压低置信度。"""
|
|
324
|
+
contains = first.element in second.element.iterancestors() or second.element in first.element.iterancestors()
|
|
325
|
+
if not contains:
|
|
326
|
+
return False
|
|
327
|
+
smaller = min(first.metrics.text_chars, second.metrics.text_chars)
|
|
328
|
+
larger = max(first.metrics.text_chars, second.metrics.text_chars, 1)
|
|
329
|
+
return smaller / larger >= 0.8
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
def _soft_prune(root: etree._Element) -> None:
|
|
333
|
+
"""在候选副本中删除确定的导航/表单和高噪声 token 子树。"""
|
|
334
|
+
metrics_by_element = _collect_soft_prune_metrics(root)
|
|
335
|
+
for element in list(root.iterdescendants()):
|
|
336
|
+
if not isinstance(element.tag, str):
|
|
337
|
+
continue
|
|
338
|
+
name = local_name(element)
|
|
339
|
+
tokens = _tokens(element)
|
|
340
|
+
metrics = metrics_by_element[element]
|
|
341
|
+
valuable = metrics.semantic_count > (1 if name in _SEMANTIC_TAGS else 0)
|
|
342
|
+
text_chars = metrics.text_chars + max(0, metrics.text_segments - 1)
|
|
343
|
+
link_chars = metrics.link_chars + max(0, metrics.link_count - 1)
|
|
344
|
+
link_density = link_chars / max(1, text_chars)
|
|
345
|
+
should_remove = name in _BOILERPLATE_TAGS or (
|
|
346
|
+
bool(tokens & _NEGATIVE_TOKENS) and not valuable and (text_chars < 200 or link_density > 0.5)
|
|
347
|
+
)
|
|
348
|
+
if should_remove:
|
|
349
|
+
_drop_tree_preserve_tail(element)
|
|
350
|
+
|
|
351
|
+
|
|
352
|
+
def _collect_soft_prune_metrics(root: etree._Element) -> dict[etree._Element, _SoftPruneMetrics]:
|
|
353
|
+
"""单次后序遍历预计算每个元素的完整子树文本、链接和语义对象统计。"""
|
|
354
|
+
elements = [element for element in root.iter() if isinstance(element.tag, str)]
|
|
355
|
+
output: dict[etree._Element, _SoftPruneMetrics] = {}
|
|
356
|
+
for element in reversed(elements):
|
|
357
|
+
own_text = _normalized_text(element.text)
|
|
358
|
+
text_chars = len(own_text)
|
|
359
|
+
text_segments = 1 if own_text else 0
|
|
360
|
+
link_chars = 0
|
|
361
|
+
link_count = 0
|
|
362
|
+
semantic_count = 1 if local_name(element) in _SEMANTIC_TAGS else 0
|
|
363
|
+
for child in element:
|
|
364
|
+
if isinstance(child.tag, str):
|
|
365
|
+
child_metrics = output[child]
|
|
366
|
+
text_chars += child_metrics.text_chars
|
|
367
|
+
text_segments += child_metrics.text_segments
|
|
368
|
+
link_chars += child_metrics.link_chars
|
|
369
|
+
link_count += child_metrics.link_count
|
|
370
|
+
semantic_count += child_metrics.semantic_count
|
|
371
|
+
if local_name(child) == "a":
|
|
372
|
+
child_text_chars = child_metrics.text_chars + max(0, child_metrics.text_segments - 1)
|
|
373
|
+
if child_text_chars:
|
|
374
|
+
link_chars += child_text_chars
|
|
375
|
+
link_count += 1
|
|
376
|
+
tail = _normalized_text(child.tail)
|
|
377
|
+
if tail:
|
|
378
|
+
text_chars += len(tail)
|
|
379
|
+
text_segments += 1
|
|
380
|
+
output[element] = _SoftPruneMetrics(
|
|
381
|
+
text_chars=text_chars,
|
|
382
|
+
text_segments=text_segments,
|
|
383
|
+
link_chars=link_chars,
|
|
384
|
+
link_count=link_count,
|
|
385
|
+
semantic_count=semantic_count,
|
|
386
|
+
)
|
|
387
|
+
return output
|
|
388
|
+
|
|
389
|
+
|
|
390
|
+
def _drop_tree_preserve_tail(element: etree._Element) -> None:
|
|
391
|
+
"""删除噪声子树,同时把 tail 归还到相邻文本位置。"""
|
|
392
|
+
parent = element.getparent()
|
|
393
|
+
if parent is None:
|
|
394
|
+
return
|
|
395
|
+
tail = element.tail or ""
|
|
396
|
+
previous = element.getprevious()
|
|
397
|
+
if tail:
|
|
398
|
+
if previous is not None:
|
|
399
|
+
previous.tail = (previous.tail or "") + tail
|
|
400
|
+
else:
|
|
401
|
+
parent.text = (parent.text or "") + tail
|
|
402
|
+
parent.remove(element)
|
|
403
|
+
|
|
404
|
+
|
|
405
|
+
def _normalized_text(value: str | None) -> str:
|
|
406
|
+
"""折叠文本空白,供字符计数和候选比较使用。"""
|
|
407
|
+
return re.sub(r"\s+", " ", value or "").strip()
|
|
408
|
+
|
|
409
|
+
|
|
410
|
+
def _document_order(element: etree._Element) -> tuple[int, ...]:
|
|
411
|
+
"""返回元素从根到自身的逐层索引,作为稳定排序键。"""
|
|
412
|
+
path: list[int] = []
|
|
413
|
+
current: etree._Element | None = element
|
|
414
|
+
while current is not None and current.getparent() is not None:
|
|
415
|
+
parent = current.getparent()
|
|
416
|
+
path.append(parent.index(current))
|
|
417
|
+
current = parent
|
|
418
|
+
return tuple(reversed(path))
|
|
419
|
+
|
|
420
|
+
|
|
421
|
+
__all__ = ["CandidateMetrics", "ContentSelection", "select_auto_content"]
|
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
"""Flash PDF、EPUB、HTML、OFD、CSV 与 Office/RTF 文档模型。"""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import TYPE_CHECKING, Any, BinaryIO
|
|
6
|
+
|
|
7
|
+
from docvortex.document.contracts import HtmlSourceContext
|
|
8
|
+
|
|
9
|
+
if TYPE_CHECKING:
|
|
10
|
+
from ...document.pdf.document import PDFDocument
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class PdfModel:
|
|
14
|
+
"""将 Flash 原生 PDF 流水线包装为无状态模型。"""
|
|
15
|
+
|
|
16
|
+
def predict(self, pdf_doc: PDFDocument) -> list[list[dict[str, Any]]]:
|
|
17
|
+
"""分析调用方持有的 PDFDocument,在所有文字匹配结束后统一输出可见英数。"""
|
|
18
|
+
from .pdf import pipeline
|
|
19
|
+
from ...content import normalize_pdf_model_text
|
|
20
|
+
|
|
21
|
+
pages = pipeline._analyze_native_document(pdf_doc)
|
|
22
|
+
normalize_pdf_model_text(pages)
|
|
23
|
+
return pages
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class CsvModel:
|
|
27
|
+
"""将 CSV 分隔符文本包装为无状态 Flash 模型。"""
|
|
28
|
+
|
|
29
|
+
def predict(self, file_binary: BinaryIO) -> list[list[dict[str, Any]]]:
|
|
30
|
+
"""转换调用方持有的 CSV 二进制流,并返回单逻辑页 model_list。"""
|
|
31
|
+
from .csv import convert_csv
|
|
32
|
+
|
|
33
|
+
return convert_csv(file_binary)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class EpubModel:
|
|
37
|
+
"""将 EPUB OCF/OPF 文档包装为无状态 Flash 模型。"""
|
|
38
|
+
|
|
39
|
+
def predict(
|
|
40
|
+
self,
|
|
41
|
+
file_binary: BinaryIO,
|
|
42
|
+
) -> list[list[dict[str, Any]]]:
|
|
43
|
+
"""转换调用方持有的整本 EPUB 流,并返回目录页和全部正文逻辑页。"""
|
|
44
|
+
from .epub.converter import EpubConverter
|
|
45
|
+
|
|
46
|
+
converter = EpubConverter()
|
|
47
|
+
converter.convert(file_binary)
|
|
48
|
+
return converter.pages
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
class HtmlModel:
|
|
52
|
+
"""将 standalone HTML 文档包装为无状态 Flash 模型。"""
|
|
53
|
+
|
|
54
|
+
def predict(
|
|
55
|
+
self,
|
|
56
|
+
file_binary: BinaryIO,
|
|
57
|
+
*,
|
|
58
|
+
source_context: HtmlSourceContext | None = None,
|
|
59
|
+
) -> list[list[dict[str, Any]]]:
|
|
60
|
+
"""转换静态 HTML 流,并返回单逻辑页 model_list。"""
|
|
61
|
+
from .html.converter import HtmlConverter
|
|
62
|
+
|
|
63
|
+
converter = HtmlConverter()
|
|
64
|
+
converter.convert(file_binary, source_context=source_context)
|
|
65
|
+
return converter.pages
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
class OfdModel:
|
|
69
|
+
"""将 OFD 固定版式文档包装为无状态 Flash 模型。"""
|
|
70
|
+
|
|
71
|
+
def predict(self, file_binary: BinaryIO) -> list[list[dict[str, Any]]]:
|
|
72
|
+
"""转换调用方持有的整份 OFD 流,并返回逐物理页 model-list。"""
|
|
73
|
+
from .ofd.converter import OfdConverter
|
|
74
|
+
|
|
75
|
+
converter = OfdConverter()
|
|
76
|
+
converter.convert(file_binary)
|
|
77
|
+
return converter.pages
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
class RtfModel:
|
|
81
|
+
"""将 Rich Text Format 文档包装为无状态 Flash 模型。"""
|
|
82
|
+
|
|
83
|
+
def predict(self, file_binary: BinaryIO) -> list[list[dict[str, Any]]]:
|
|
84
|
+
"""转换调用方持有的 RTF 二进制流,并返回单逻辑页 model_list。"""
|
|
85
|
+
from .office.rtf.converter import RtfConverter
|
|
86
|
+
|
|
87
|
+
converter = RtfConverter()
|
|
88
|
+
converter.convert(file_binary)
|
|
89
|
+
return converter.pages
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
class DocxModel:
|
|
93
|
+
"""将 DOCX Converter 包装为无状态模型。"""
|
|
94
|
+
|
|
95
|
+
def predict(self, file_binary: BinaryIO) -> list[list[dict[str, Any]]]:
|
|
96
|
+
"""转换调用方持有的 DOCX 二进制流,并返回分页 model_list。"""
|
|
97
|
+
|
|
98
|
+
# 延迟加载 Converter,避免纯 PDF 路径提前加载 Office 依赖。
|
|
99
|
+
from .office.docx.docx_converter import DocxConverter
|
|
100
|
+
|
|
101
|
+
converter = DocxConverter()
|
|
102
|
+
converter.convert(file_binary)
|
|
103
|
+
return converter.pages
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
class DocModel:
|
|
107
|
+
"""将 Word 97–2003 Converter 包装为无状态模型。"""
|
|
108
|
+
|
|
109
|
+
def predict(self, file_binary: BinaryIO) -> list[list[dict[str, Any]]]:
|
|
110
|
+
"""转换调用方持有的 DOC 二进制流,并返回逐 section model-list。"""
|
|
111
|
+
|
|
112
|
+
# 延迟加载旧版 DOC 解析器,避免其他格式提前加载 olefile。
|
|
113
|
+
from .office.doc.doc_converter import DocConverter
|
|
114
|
+
|
|
115
|
+
converter = DocConverter()
|
|
116
|
+
converter.convert(file_binary)
|
|
117
|
+
return converter.pages
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
class PptxModel:
|
|
121
|
+
"""将 PPTX Converter 包装为无状态模型。"""
|
|
122
|
+
|
|
123
|
+
def predict(self, file_binary: BinaryIO) -> list[list[dict[str, Any]]]:
|
|
124
|
+
"""转换调用方持有的 PPTX 二进制流,并返回分页 model_list。"""
|
|
125
|
+
|
|
126
|
+
# 延迟加载 Converter,避免纯 PDF 路径提前加载 Office 依赖。
|
|
127
|
+
from .office.pptx.pptx_converter import PptxConverter
|
|
128
|
+
|
|
129
|
+
converter = PptxConverter()
|
|
130
|
+
converter.convert(file_binary)
|
|
131
|
+
return converter.pages
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
class PptModel:
|
|
135
|
+
"""将 PowerPoint 97–2003 Converter 包装为无状态模型。"""
|
|
136
|
+
|
|
137
|
+
def predict(self, file_binary: BinaryIO) -> list[list[dict[str, Any]]]:
|
|
138
|
+
"""转换调用方持有的 PPT 二进制流,并返回逐幻灯片 model-list。"""
|
|
139
|
+
|
|
140
|
+
# 延迟加载旧版 PPT 解析器,避免其他格式提前加载 olefile。
|
|
141
|
+
from .office.ppt.ppt_converter import PptConverter
|
|
142
|
+
|
|
143
|
+
converter = PptConverter()
|
|
144
|
+
converter.convert(file_binary)
|
|
145
|
+
return converter.pages
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
class XlsModel:
|
|
149
|
+
"""将 Excel 97–2003 Converter 包装为无状态模型。"""
|
|
150
|
+
|
|
151
|
+
def predict(self, file_binary: BinaryIO) -> list[list[dict[str, Any]]]:
|
|
152
|
+
"""转换调用方持有的 XLS 二进制流,并返回逐工作表 model-list。"""
|
|
153
|
+
|
|
154
|
+
# 延迟加载旧版 XLS 解析器,避免其他格式提前加载 olefile/openpyxl。
|
|
155
|
+
from .office.xls.xls_converter import XlsConverter
|
|
156
|
+
|
|
157
|
+
converter = XlsConverter()
|
|
158
|
+
converter.convert(file_binary)
|
|
159
|
+
return converter.pages
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
class XlsxModel:
|
|
163
|
+
"""将 XLSX Converter 包装为无状态模型。"""
|
|
164
|
+
|
|
165
|
+
def predict(self, file_binary: BinaryIO) -> list[list[dict[str, Any]]]:
|
|
166
|
+
"""转换调用方持有的 XLSX 二进制流,并返回分页 model_list。"""
|
|
167
|
+
|
|
168
|
+
# 延迟加载 Converter,避免纯 PDF 路径提前加载 Office 依赖。
|
|
169
|
+
from .office.xlsx.xlsx_converter import XlsxConverter
|
|
170
|
+
|
|
171
|
+
converter = XlsxConverter()
|
|
172
|
+
converter.convert(file_binary)
|
|
173
|
+
return converter.pages
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
class OdtModel:
|
|
177
|
+
"""将 OpenDocument Text 包装为无状态 Flash 模型。"""
|
|
178
|
+
|
|
179
|
+
def predict(self, file_binary: BinaryIO) -> list[list[dict[str, Any]]]:
|
|
180
|
+
"""转换调用方持有的 ODT 二进制流,并返回分页 model_list。"""
|
|
181
|
+
from .office.odf.converters import OdtConverter
|
|
182
|
+
|
|
183
|
+
converter = OdtConverter()
|
|
184
|
+
converter.convert(file_binary)
|
|
185
|
+
return converter.pages
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
class OdsModel:
|
|
189
|
+
"""将 OpenDocument Spreadsheet 包装为无状态 Flash 模型。"""
|
|
190
|
+
|
|
191
|
+
def predict(self, file_binary: BinaryIO) -> list[list[dict[str, Any]]]:
|
|
192
|
+
"""转换调用方持有的 ODS 二进制流,并返回逐工作表 model_list。"""
|
|
193
|
+
from .office.odf.converters import OdsConverter
|
|
194
|
+
|
|
195
|
+
converter = OdsConverter()
|
|
196
|
+
converter.convert(file_binary)
|
|
197
|
+
return converter.pages
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
class OdpModel:
|
|
201
|
+
"""将 OpenDocument Presentation 包装为无状态 Flash 模型。"""
|
|
202
|
+
|
|
203
|
+
def predict(self, file_binary: BinaryIO) -> list[list[dict[str, Any]]]:
|
|
204
|
+
"""转换调用方持有的 ODP 二进制流,并返回逐幻灯片 model_list。"""
|
|
205
|
+
from .office.odf.converters import OdpConverter
|
|
206
|
+
|
|
207
|
+
converter = OdpConverter()
|
|
208
|
+
converter.convert(file_binary)
|
|
209
|
+
return converter.pages
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
"""OFD 固定版式 Flash 解析入口。"""
|
|
2
|
+
|
|
3
|
+
from .errors import OfdEncryptedError, OfdParseError, OfdResourceLimitError
|
|
4
|
+
from .metadata import extract_ofd_metadata
|
|
5
|
+
from .package import detect_ofd, detect_ofd_path
|
|
6
|
+
|
|
7
|
+
__all__ = [
|
|
8
|
+
"OfdEncryptedError",
|
|
9
|
+
"OfdParseError",
|
|
10
|
+
"OfdResourceLimitError",
|
|
11
|
+
"detect_ofd",
|
|
12
|
+
"detect_ofd_path",
|
|
13
|
+
"extract_ofd_metadata",
|
|
14
|
+
]
|