docvortex 0.4.22__tar.gz → 0.4.23__tar.gz

{docvortex-0.4.22/src/docvortex.egg-info → docvortex-0.4.23}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: docvortex
-Version: 0.4.22
+Version: 0.4.23
 Summary: A fast, multi-format document parsing and conversion engine
 License-Expression: MIT
 Project-URL: Repository, https://github.com/myhloli/DocVortex
@@ -127,17 +127,19 @@ use `overwrite=True` in Python or `--overwrite` in the CLI to replace them.
 
 ## Supported formats
 
-### Native inputs · 15 formats
+### Native inputs · 17 formats
 
-| Document family | Formats |
+| Document family | File extensions |
 | --- | --- |
-| PDF with native text | PDF |
-| Word & rich text | DOC, DOCX, RTF |
-| Presentations | PPT, PPTX |
-| Spreadsheets | XLS, XLSX, CSV, TSV |
-| OpenDocument | ODT, ODS, ODP |
-| E-books & web documents | EPUB, HTML |
-| Open Fixed-layout Document | OFD |
+| PDF with native text | `.pdf` |
+| Word & rich text | `.doc`, `.docx`, `.rtf` |
+| Presentations | `.ppt`, `.pptx` |
+| Spreadsheets | `.xls`, `.xlsx`, `.csv`, `.tsv` |
+| OpenDocument | `.odt`, `.ods`, `.odp` |
+| E-books | `.epub` |
+| Web documents | `.html`, `.htm`, `.shtml` |
+| Web archives | `.mhtml`, `.mht` |
+| Open Fixed-layout Document | `.ofd` |
 
 ### Outputs · 7 formats
 

{docvortex-0.4.22 → docvortex-0.4.23}/README.md

@@ -73,17 +73,19 @@ use `overwrite=True` in Python or `--overwrite` in the CLI to replace them.
 
 ## Supported formats
 
-### Native inputs · 15 formats
+### Native inputs · 17 formats
 
-| Document family | Formats |
+| Document family | File extensions |
 | --- | --- |
-| PDF with native text | PDF |
-| Word & rich text | DOC, DOCX, RTF |
-| Presentations | PPT, PPTX |
-| Spreadsheets | XLS, XLSX, CSV, TSV |
-| OpenDocument | ODT, ODS, ODP |
-| E-books & web documents | EPUB, HTML |
-| Open Fixed-layout Document | OFD |
+| PDF with native text | `.pdf` |
+| Word & rich text | `.doc`, `.docx`, `.rtf` |
+| Presentations | `.ppt`, `.pptx` |
+| Spreadsheets | `.xls`, `.xlsx`, `.csv`, `.tsv` |
+| OpenDocument | `.odt`, `.ods`, `.odp` |
+| E-books | `.epub` |
+| Web documents | `.html`, `.htm`, `.shtml` |
+| Web archives | `.mhtml`, `.mht` |
+| Open Fixed-layout Document | `.ofd` |
 
 ### Outputs · 7 formats
 

{docvortex-0.4.22 → docvortex-0.4.23}/schemas/middle-2.0.json

@@ -765,6 +765,7 @@
             "tsv",
             "epub",
             "html",
+            "mhtml",
             "ofd",
             "odt",
             "ods",

{docvortex-0.4.22 → docvortex-0.4.23}/schemas/model-2.0.json

@@ -18,6 +18,7 @@
             "tsv",
             "epub",
             "html",
+            "mhtml",
             "ofd",
             "odt",
             "ods",

{docvortex-0.4.22 → docvortex-0.4.23}/src/docvortex/analyzers/native/html/converter.py

@@ -41,6 +41,10 @@ class HtmlConverter:
             raise HtmlResourceLimitError(f"HTML resource limit exceeded: max_html_bytes={MAX_HTML_BYTES}")
         document = parse_html_document(file_bytes, source_context)
         resources = HtmlResourceContext(document.source_context, base_href=document.base_href)
+        self._convert_document(document, resources)
+
+    def _convert_document(self, document: HtmlDocument, resources: HtmlResourceContext) -> None:
+        """复用已解析 DOM 和资源适配器，供 HTML 与网页归档共享正文投影。"""
         wire_result = decode_docvortex_html_wire(document.body, resources)
         if wire_result.blocks is not None:
             blocks = wire_result.blocks

{docvortex-0.4.22 → docvortex-0.4.23}/src/docvortex/analyzers/native/html/selector.py

@@ -331,6 +331,7 @@ def _is_containment_equivalent(first: _ScoredCandidate, second: _ScoredCandidate
 
 def _soft_prune(root: etree._Element) -> None:
     """在候选副本中删除确定的导航/表单和高噪声 token 子树。"""
+    _prune_article_comments(root)
     metrics_by_element = _collect_soft_prune_metrics(root)
     for element in list(root.iterdescendants()):
         if not isinstance(element.tag, str):
@@ -349,6 +350,36 @@ def _soft_prune(root: etree._Element) -> None:
             _drop_tree_preserve_tail(element)
 
 
+def _is_comment_region(element: etree._Element) -> bool:
+    """按完整词与驼峰边界识别评论容器，不匹配正文文本或任意子串。"""
+    value = f"{element.get('id') or ''} {element.get('class') or ''}"
+    value = re.sub(r"([a-z0-9])([A-Z])", r"\1 \2", value).casefold()
+    return bool(frozenset(_TOKEN_RE.split(value)) & {"comment", "comments"})
+
+
+def _prune_article_comments(root: etree._Element) -> None:
+    """仅在单篇独立文章明确存在时移除附属评论，保留论坛及讨论型正文。"""
+    articles = [
+        element
+        for element in root.iter()
+        if isinstance(element.tag, str)
+        and (local_name(element) == "article" or "articlebody" in (element.get("itemprop") or "").casefold().split())
+        and not any(_is_comment_region(parent) for parent in (element, *element.iterancestors()))
+        and len(_normalized_text(" ".join(element.itertext()))) >= _MIN_TEXT_CHARS
+    ]
+    # 嵌套 articleBody 与 article 属于同一正文；多个独立 article 按论坛处理。
+    outer_articles = [element for element in articles if not any(parent in articles for parent in element.iterancestors())]
+    if len(outer_articles) != 1:
+        return
+    article = outer_articles[0]
+    for element in list(root.iterdescendants()):
+        if not isinstance(element.tag, str) or not _is_comment_region(element):
+            continue
+        if element is article or element in article.iterancestors():
+            continue
+        _drop_tree_preserve_tail(element)
+
+
 def _collect_soft_prune_metrics(root: etree._Element) -> dict[etree._Element, _SoftPruneMetrics]:
     """单次后序遍历预计算每个元素的完整子树文本、链接和语义对象统计。"""
     elements = [element for element in root.iter() if isinstance(element.tag, str)]

docvortex-0.4.23/src/docvortex/analyzers/native/mhtml/__init__.py

@@ -0,0 +1,3 @@
+"""MHTML 网页归档的 MIME 解包与静态正文解析。"""
+
+__all__: list[str] = []

docvortex-0.4.23/src/docvortex/analyzers/native/mhtml/converter.py

@@ -0,0 +1,73 @@
+"""把 MHTML 主文档及归档资源投影为现有 HTML 语义块。"""
+
+from __future__ import annotations
+
+from typing import BinaryIO
+
+from docvortex.document.contracts import HtmlSourceContext
+from docvortex.document.mhtml import MAX_ARCHIVE_BYTES, MhtmlArchive
+from docvortex.result import Diagnostic
+from docvortex.schema import DocumentProperties
+
+from ..html.converter import HtmlConverter
+from ..html.document import HtmlDocument, parse_html_document
+from ..html.metadata import read_html_properties
+from .resources import MhtmlResourceContext
+
+
+def read_archive_properties(archive: MhtmlArchive) -> tuple[DocumentProperties, list[str]]:
+    """复用 HTML 声明元数据，仅用 MIME Subject 补充缺失标题。"""
+    properties, warnings = read_html_properties(archive.html, archive.source_context)
+    properties.title = properties.title or archive.subject
+    return properties, warnings
+
+
+class MhtmlConverter(HtmlConverter):
+    """通过一次解包生成正文、元数据和可选资源诊断。"""
+
+    def __init__(self) -> None:
+        """初始化与单次转换绑定的元数据及诊断。"""
+        super().__init__()
+        self.properties = DocumentProperties()
+        self.diagnostics: tuple[Diagnostic, ...] = ()
+
+    def convert(self, file_binary: BinaryIO, *, source_context: HtmlSourceContext | None = None) -> None:
+        """加载主 HTML 后调用共享投影，不合并 iframe 或其他 HTML 附件。"""
+        archive = MhtmlArchive(file_binary.read(MAX_ARCHIVE_BYTES + 1), source_context)
+        self.properties, warnings = read_archive_properties(archive)
+        document = parse_html_document(archive.html, archive.source_context)
+        if document.title is None and archive.subject:
+            from dataclasses import replace
+
+            document = replace(document, title=archive.subject)
+        resources = MhtmlResourceContext(archive, base_href=document.base_href)
+        _restore_picture_sources(document, archive)
+        self._convert_document(document, resources)
+        self.diagnostics = tuple(Diagnostic("read_metadata_failed", warning) for warning in warnings) + tuple(
+            archive.diagnostics
+        )
+
+
+def _restore_picture_sources(document: HtmlDocument, archive: MhtmlArchive) -> None:
+    """当回退图片未归档时采用已保存的 picture/srcset 候选，不发起外部请求。"""
+    for image in document.body.iter("img"):
+        if (image.get("src") or "").strip().casefold().startswith("data:"):
+            continue
+        if archive.find(image.get("src") or "", base_href=document.base_href) is not None:
+            continue
+        picture = next((parent for parent in image.iterancestors() if parent.tag == "picture"), None)
+        sources = list(picture.iter("source")) if picture is not None else []
+        for source in [*sources, image]:
+            # 浏览器归档中的网络候选按逗号分隔；data URI 无需做归档匹配。
+            candidates = (source.get("srcset") or "").split(",")
+            selected = next(
+                (
+                    fields[0]
+                    for candidate in candidates
+                    if (fields := candidate.split()) and archive.find(fields[0], base_href=document.base_href) is not None
+                ),
+                None,
+            )
+            if selected:
+                image.set("src", selected)
+                break

docvortex-0.4.23/src/docvortex/analyzers/native/mhtml/resources.py

@@ -0,0 +1,80 @@
+"""使用 MIME 内嵌图片和样式的 HTML 资源适配器。"""
+
+from __future__ import annotations
+
+from docvortex.content.markup import ResolvedMarkupImage
+from docvortex.document.mhtml import ArchivePart, MhtmlArchive, MhtmlParseError
+from docvortex.foundation._svg_raster import looks_like_svg_payload, serialize_svg_image
+
+from ..html.constants import MAX_HTML_IMAGE_BYTES
+from ..html.errors import HtmlResourceLimitError
+from ..html.resources import HtmlResourceContext, _image_data_uri
+
+
+class MhtmlResourceContext(HtmlResourceContext):
+    """仅从归档读取资源，复用 HTML 的链接、图片校验和字节预算。"""
+
+    def __init__(self, archive: MhtmlArchive, *, base_href: str | None = None) -> None:
+        """绑定归档并初始化以 MIME 部件为键的去重缓存。"""
+        super().__init__(archive.source_context, base_href=base_href)
+        self.archive = archive
+        self._archive_images: dict[ArchivePart, str | None] = {}
+        self._archive_stylesheets: dict[ArchivePart, str | None] = {}
+
+    def _payload(self, part: ArchivePart, reference: str) -> bytes | None:
+        """可选部件损坏时记录诊断；资源限额错误仍向调用方传播。"""
+        try:
+            return self.archive.decode(part)
+        except MhtmlParseError:
+            self.archive.report("mhtml_resource_invalid", reference)
+            return None
+
+    def resolve_image(self, source: str, *, alt: str = "") -> ResolvedMarkupImage | None:
+        """优先恢复归档图片，缺失时沿用安全外链或替代文本降级。"""
+        if not source.strip() or source.strip().casefold().startswith("data:"):
+            return super().resolve_image(source, alt=alt)
+        part = self.archive.find(source, base_href=self.base_href)
+        if part is None:
+            self.archive.report("mhtml_resource_missing", source)
+            return super().resolve_image(source, alt=alt)
+        if part not in self._archive_images:
+            payload = self._payload(part, source)
+            data_uri = None
+            if payload is not None:
+                if len(payload) > MAX_HTML_IMAGE_BYTES:
+                    raise HtmlResourceLimitError(f"MHTML image exceeds max_html_image_bytes={MAX_HTML_IMAGE_BYTES}")
+                if looks_like_svg_payload(payload):
+                    data_uri = serialize_svg_image(payload)
+                else:
+                    data_uri = _image_data_uri(payload)
+                if data_uri is not None:
+                    self._charge_image_bytes(len(payload))
+                else:
+                    self.archive.report("mhtml_resource_invalid", source)
+            self._archive_images[part] = data_uri
+        if data_uri := self._archive_images[part]:
+            return ResolvedMarkupImage(image_base64=data_uri, alt=alt)
+        return super().resolve_image(source, alt=alt)
+
+    def load_stylesheet(self, href: str) -> str | None:
+        """按 HTML 源顺序读取归档 CSS，仅解释现有静态样式子集。"""
+        part = self.archive.find(href, base_href=self.base_href)
+        if part is None:
+            self.archive.report("mhtml_resource_missing", href)
+            return None
+        if part in self._archive_stylesheets:
+            return self._archive_stylesheets[part]
+        payload = self._payload(part, href)
+        text = None
+        if payload is not None:
+            if part.message.get_content_type() != "text/css":
+                self.archive.report("mhtml_resource_invalid", href)
+            else:
+                self._charge_stylesheet_bytes(len(payload))
+                try:
+                    text = payload.decode(part.message.get_content_charset() or "utf-8-sig", errors="replace")
+                except LookupError:
+                    self.archive.report("mhtml_resource_invalid", href)
+                    text = payload.decode("utf-8-sig", errors="replace")
+        self._archive_stylesheets[part] = text
+        return text

{docvortex-0.4.22 → docvortex-0.4.23}/src/docvortex/api.py

@@ -50,7 +50,7 @@ def analyze(
     try:
         properties = prepared.source_properties
         metadata_diagnostics = tuple(Diagnostic("read_metadata_failed", message) for message in prepared.metadata_warnings)
-        if prepared.file_suffix != "pdf":
+        if prepared.file_suffix not in {"pdf", "mhtml"}:
             from .errors import DocumentError
 
             try:
@@ -81,6 +81,14 @@ def analyze(
             )
         elif prepared.file_suffix == "html":
             pages = models.HtmlModel().predict(BytesIO(prepared.data), source_context=prepared.source_context)
+        elif prepared.file_suffix == "mhtml":
+            from .analyzers.native.mhtml.converter import MhtmlConverter
+
+            converter = MhtmlConverter()
+            converter.convert(BytesIO(prepared.data), source_context=prepared.source_context)
+            pages = converter.pages
+            properties = converter.properties
+            native_diagnostics = converter.diagnostics
         else:
             model_types: dict[FileSuffix, type[NativeBinaryAnalyzer]] = {
                 "csv": models.CsvModel,

{docvortex-0.4.22 → docvortex-0.4.23}/src/docvortex/document/detection.py

@@ -14,7 +14,7 @@ from typing import TYPE_CHECKING
 if TYPE_CHECKING:
     from magika import Magika
 
-from .filetypes import CSV_EXTENSIONS, HTML_EXTENSIONS, IMAGE_EXTENSIONS, rtf_header_offset
+from .filetypes import CSV_EXTENSIONS, HTML_EXTENSIONS, IMAGE_EXTENSIONS, has_mhtml_header, rtf_header_offset
 
 PDF_SIG_BYTES = b"%PDF"
 OLE2_SIG_BYTES = b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1"
@@ -54,6 +54,7 @@ _STRONG_CONTENT_SUFFIXES = frozenset(
         "xlsx",
         "rtf",
         "epub",
+        "mhtml",
         "ofd",
         "odt",
         "ods",
@@ -307,10 +308,13 @@ def _reject_unverified_package_suffix(detected_suffix: str) -> str:
 
 
 def guess_suffix_by_bytes(file_bytes: bytes, file_path: str | None = None) -> str:
+    """优先依据强签名与容器头识别格式，再使用通用识别器。"""
     if file_bytes[: len(PDF_SIG_BYTES)] == PDF_SIG_BYTES:
         return "pdf"
     if rtf_header_offset(file_bytes[:128]) is not None:
         return "rtf"
+    if has_mhtml_header(file_bytes):
+        return "mhtml"
 
     ofd_suffix = _guess_ofd_suffix_by_bytes(file_bytes)
     if ofd_suffix:
@@ -345,6 +349,7 @@ def guess_suffix_by_bytes(file_bytes: bytes, file_path: str | None = None) -> st
 
 
 def guess_suffix_by_path(file_path: str | Path) -> str:
+    """读取文件签名和有界 MIME 头，保持路径与字节入口一致。"""
     if not isinstance(file_path, Path):
         file_path = Path(file_path)
 
@@ -374,6 +379,10 @@ def guess_suffix_by_path(file_path: str | Path) -> str:
     if _has_pdf_signature_by_path(file_path):
         return "pdf"
 
+    with file_path.open("rb") as source:
+        if has_mhtml_header(source.read(65536)):
+            return "mhtml"
+
     suffix = _magika().identify_path(file_path).prediction.output.label
     if suffix in ["ai", "html"] and file_path.suffix.lower() in [".pdf"]:
         try:

{docvortex-0.4.22 → docvortex-0.4.23}/src/docvortex/document/filetypes.py

@@ -1,6 +1,8 @@
 from __future__ import annotations
 
 import re
+from email import policy
+from email.parser import BytesHeaderParser
 from pathlib import Path
 from typing import Final
 
@@ -37,6 +39,7 @@ OFD_EXTENSIONS: frozenset[str] = frozenset({"ofd"})
 OFFICE_EXTENSIONS: frozenset[str] = frozenset({"doc", "docx", "ppt", "pptx", "xls", "xlsx", "rtf"}) | ODF_EXTENSIONS
 
 HTML_EXTENSIONS: frozenset[str] = frozenset({"html", "htm", "shtml"})
+MHTML_EXTENSIONS: frozenset[str] = frozenset({"mhtml", "mht"})
 
 CSV_EXTENSIONS: frozenset[str] = frozenset({"csv", "tsv"})
 
@@ -49,7 +52,7 @@ TIERED_PARSE_EXTENSIONS: frozenset[str] = PDF_EXTENSIONS | IMAGE_EXTENSIONS
 PAGE_RANGE_PARSE_EXTENSIONS: frozenset[str] = PDF_EXTENSIONS
 
 FLASH_ONLY_PARSE_EXTENSIONS: frozenset[str] = (
-    OFFICE_EXTENSIONS | HTML_EXTENSIONS | CSV_EXTENSIONS | EPUB_EXTENSIONS | OFD_EXTENSIONS
+    OFFICE_EXTENSIONS | HTML_EXTENSIONS | MHTML_EXTENSIONS | CSV_EXTENSIONS | EPUB_EXTENSIONS | OFD_EXTENSIONS
 )
 
 PARSEABLE_EXTENSIONS: frozenset[str] = TIERED_PARSE_EXTENSIONS | FLASH_ONLY_PARSE_EXTENSIONS
@@ -65,6 +68,7 @@ FILE_TYPE_BY_EXTENSION: dict[str, str] = {
     **dict.fromkeys(IMAGE_EXTENSIONS, "image"),
     **{ext: ext for ext in OFFICE_EXTENSIONS},
     **dict.fromkeys(HTML_EXTENSIONS, "html"),
+    **dict.fromkeys(MHTML_EXTENSIONS, "mhtml"),
     "csv": "csv",
     "tsv": "tsv",
     **dict.fromkeys(EPUB_EXTENSIONS, "epub"),
@@ -104,6 +108,8 @@ MIME_TYPE_BY_EXTENSION: dict[str, str] = {
     "html": "text/html",
     "htm": "text/html",
     "shtml": "text/html",
+    "mhtml": "multipart/related",
+    "mht": "multipart/related",
     "png": "image/png",
     "jpg": "image/jpeg",
     "jpeg": "image/jpeg",
@@ -131,6 +137,18 @@ _RTF_HEADER_RE: Final = re.compile(
 )
 
 
+def has_mhtml_header(file_bytes: bytes) -> bool:
+    """识别有界 MIME 头中的 related 容器，避免仅靠后缀误判普通邮件或 HTML。"""
+    prefix = file_bytes[:65536]
+    if b"multipart/related" not in prefix.lower():
+        return False
+    try:
+        message = BytesHeaderParser(policy=policy.default).parsebytes(prefix)
+        return message.get_content_type() == "multipart/related" and bool(message.get_boundary())
+    except ValueError:
+        return False
+
+
 def rtf_header_offset(file_bytes: bytes) -> int | None:
     """返回 RTF 根组左花括号偏移；不接受带任意前缀的伪装文本。"""
     match = _RTF_HEADER_RE.match(file_bytes)