docvortex 0.2.1__py3-none-any.whl

docvortex/codecs/html/resources.py

@@ -0,0 +1,34 @@
+"""HTML wire 物化依赖的资源协议，不依赖具体输入解析器。"""
+
+from __future__ import annotations
+
+from typing import Protocol
+from lxml import etree
+
+from ...content.markup import MarkupContext
+
+
+class WireAnchorResolver(Protocol):
+    """描述精确 HTML 文档提供的锚点解析能力。"""
+
+    def resolve_fragment(self, fragment: str) -> str | None:
+        """将源 fragment 解析为规范内部链接。"""
+
+    def heading_anchor(self, heading: etree._Element) -> str | None:
+        """返回标题的规范锚点。"""
+
+    def heading_label(self, anchor: str) -> str | None:
+        """返回标题锚点对应的文字。"""
+
+    def note_anchor(self, note: etree._Element) -> str | None:
+        """返回页面脚注的规范锚点。"""
+
+
+class WireResourceContext(MarkupContext, Protocol):
+    """在共享 markup 能力上增加绑定精确 wire 锚点的操作。"""
+
+    def bind_anchors(self, anchors: WireAnchorResolver) -> None:
+        """绑定经过整棵 wire 验证的锚点解析器。"""
+
+
+__all__ = ["WireAnchorResolver", "WireResourceContext"]

docvortex/codecs/json.py

@@ -0,0 +1,17 @@
+"""共享文档协议的显式读取入口。"""
+
+from typing import Any
+from ..schema import MiddleJson, ModelJson
+
+
+def load_model(value: dict[str, Any]) -> ModelJson:
+    """读取新版分析文档，不执行解析、补写来源或历史迁移。"""
+    return ModelJson.from_dict(value)
+
+
+def load_middle(value: dict[str, Any]) -> MiddleJson:
+    """读取新版语义文档，不访问源文件和外部素材。"""
+    return MiddleJson.from_dict(value)
+
+
+__all__ = ["load_model", "load_middle"]

docvortex/content/__init__.py

@@ -0,0 +1,5 @@
+"""共享内容操作及显式的 PDF 输出文字清洗。"""
+
+from .normalization import normalize_pdf_model_text
+
+__all__ = ["normalize_pdf_model_text"]

docvortex/content/inline.py

@@ -0,0 +1,248 @@
+"""Middle JSON 2.0 行内 Span 的规范化、可见文本与段落边界操作。"""
+
+from __future__ import annotations
+
+import re
+from copy import deepcopy
+from typing import Callable, Iterable
+
+from ..schema import CodeInlineSpan, EquationInlineSpan, HyperlinkSpan, InlineSpan, TextSpan, parse_inline_spans
+from ..foundation.language import detect_lang
+from ..foundation.text import CJK_LANGS, resolve_text_line_boundary
+
+_CJK_RE = re.compile(r"[\u3040-\u30ff\u3400-\u9fff\uac00-\ud7af]")
+
+
+def normalize_inline_spans(spans: Iterable[InlineSpan | dict[str, object]]) -> list[InlineSpan]:
+    """严格解析、递归规范化并合并相邻等样式文字 Span。"""
+    parsed = parse_inline_spans(list(spans))
+    normalized: list[InlineSpan] = []
+    for span in parsed:
+        current: InlineSpan
+        if isinstance(span, HyperlinkSpan):
+            children = normalize_inline_spans(span.content)
+            non_link_children = [child for child in children if not isinstance(child, HyperlinkSpan)]
+            if not non_link_children:
+                continue
+            current = span.model_copy(update={"content": non_link_children})
+        else:
+            current = span.model_copy(deep=True)
+        if (
+            normalized
+            and isinstance(normalized[-1], TextSpan)
+            and isinstance(current, TextSpan)
+            and normalized[-1].styles == current.styles
+        ):
+            normalized[-1].content += current.content
+            continue
+        if (
+            normalized
+            and isinstance(normalized[-1], HyperlinkSpan)
+            and isinstance(current, HyperlinkSpan)
+            and normalized[-1].url == current.url
+        ):
+            merged_children = normalize_inline_spans([*normalized[-1].content, *current.content])
+            normalized[-1].content = [child for child in merged_children if not isinstance(child, HyperlinkSpan)]
+            continue
+        normalized.append(current)
+    return normalized
+
+
+def inline_plain_text(spans: Iterable[InlineSpan]) -> str:
+    """提取 Span 列表的完整可见文字，供排序、合并和标题判断使用。"""
+    parts: list[str] = []
+    for span in spans:
+        if isinstance(span, (TextSpan, CodeInlineSpan, EquationInlineSpan)):
+            parts.append(span.content)
+        elif isinstance(span, HyperlinkSpan):
+            parts.append(inline_plain_text(span.content))
+    return "".join(parts)
+
+
+def join_inline_spans(contents: Iterable[Iterable[InlineSpan]]) -> list[InlineSpan]:
+    """按物理段落边界规则合并多组 Span，并保持结构化语义。"""
+    merged: list[InlineSpan] = []
+    for content in contents:
+        current = normalize_inline_spans(list(content))
+        if not current:
+            continue
+        # 边界裁剪最多清空最后一个根节点；保留其前驱以重新合并新相邻的 Span。
+        boundary_start = max(0, len(merged) - 2)
+        if merged:
+            _join_inline_span_sequences(merged, current)
+        merged.extend(current)
+        merged[boundary_start:] = normalize_inline_spans(_drop_empty_text_spans(merged[boundary_start:]))
+    return merged
+
+
+def strip_inline_spans(spans: Iterable[InlineSpan]) -> list[InlineSpan]:
+    """删除行内内容首尾空白，同时保留内部 Span 边界和样式。"""
+    normalized = normalize_inline_spans(deepcopy(list(spans)))
+    first = _first_text_span(normalized)
+    last = _last_text_span(normalized)
+    if first is not None:
+        object.__setattr__(first, "content", first.content.lstrip())
+    if last is not None:
+        object.__setattr__(last, "content", last.content.rstrip())
+    return _drop_empty_text_spans(normalized)
+
+
+def replace_inline_text(spans: Iterable[InlineSpan], content: str) -> list[InlineSpan]:
+    """把纯文本回填为单一 TextSpan，供确实丢弃原样式的规则使用。"""
+    if not content:
+        return []
+    return [TextSpan(type="text", content=content)]
+
+
+def slice_inline_spans(spans: Iterable[InlineSpan], start: int = 0, end: int | None = None) -> list[InlineSpan]:
+    """按可见字符偏移裁剪 Span，并保留覆盖范围内的样式和链接。"""
+    normalized = normalize_inline_spans(deepcopy(list(spans)))
+    visible_length = len(inline_plain_text(normalized))
+    resolved_start = min(max(start, 0), visible_length)
+    resolved_end = visible_length if end is None else min(max(end, resolved_start), visible_length)
+    output: list[InlineSpan] = []
+    cursor = 0
+    for span in normalized:
+        span_length = len(inline_plain_text([span]))
+        span_end = cursor + span_length
+        overlap_start = max(resolved_start, cursor)
+        overlap_end = min(resolved_end, span_end)
+        if overlap_start < overlap_end:
+            local_start = overlap_start - cursor
+            local_end = overlap_end - cursor
+            sliced = _slice_inline_span(span, local_start, local_end)
+            if sliced is not None:
+                output.append(sliced)
+        cursor = span_end
+        if cursor >= resolved_end:
+            break
+    return normalize_inline_spans(output)
+
+
+def map_text_span_content(spans: Iterable[InlineSpan], transform: Callable[[str], str]) -> list[InlineSpan]:
+    """递归转换 TextSpan 正文，同时保留其它 Span 语义。"""
+    output: list[InlineSpan] = []
+    for span in normalize_inline_spans(deepcopy(list(spans))):
+        if isinstance(span, TextSpan):
+            content = transform(span.content)
+            if content:
+                output.append(span.model_copy(update={"content": content}))
+        elif isinstance(span, HyperlinkSpan):
+            children = map_text_span_content(span.content, transform)
+            non_link_children = [child for child in children if not isinstance(child, HyperlinkSpan)]
+            if non_link_children:
+                output.append(span.model_copy(update={"content": non_link_children}))
+        else:
+            output.append(span)
+    return normalize_inline_spans(output)
+
+
+def _join_inline_span_sequences(previous: list[InlineSpan], current: list[InlineSpan]) -> None:
+    """在两组 Span 之间应用语言相关的换行拼接规则。"""
+    previous_visible = inline_plain_text(previous).rstrip()
+    current_visible = inline_plain_text(current).lstrip()
+    if not previous_visible or not current_visible:
+        return
+
+    last_text = _last_text_span(previous)
+    first_text = _first_text_span(current)
+    if last_text is not None:
+        object.__setattr__(last_text, "content", last_text.content.rstrip())
+    if first_text is not None:
+        object.__setattr__(first_text, "content", first_text.content.lstrip())
+
+    language = _detect_boundary_language(f"{previous_visible}{current_visible}")
+    if last_text is not None:
+        processed, separator = resolve_text_line_boundary(
+            last_text.content,
+            block_language=language,
+            next_content=current_visible,
+        )
+        object.__setattr__(last_text, "content", processed)
+    else:
+        separator = "" if language in CJK_LANGS else " "
+    if separator:
+        previous.append(TextSpan(type="text", content=separator))
+
+
+def _detect_boundary_language(content: str) -> str:
+    """检测段落边界语言，短 CJK 文本优先使用字符范围兜底。"""
+    if _CJK_RE.search(content):
+        return "zh"
+    try:
+        return detect_lang(content)
+    except Exception:
+        return ""
+
+
+def _first_text_span(spans: list[InlineSpan]) -> TextSpan | None:
+    """返回首个可见叶子为文字时对应的 TextSpan。"""
+    for span in spans:
+        if not inline_plain_text([span]):
+            continue
+        if isinstance(span, TextSpan):
+            return span
+        if isinstance(span, HyperlinkSpan):
+            return _first_text_span(list(span.content))
+        return None
+    return None
+
+
+def _last_text_span(spans: list[InlineSpan]) -> TextSpan | None:
+    """返回末个可见叶子为文字时对应的 TextSpan。"""
+    for span in reversed(spans):
+        if not inline_plain_text([span]):
+            continue
+        if isinstance(span, TextSpan):
+            return span
+        if isinstance(span, HyperlinkSpan):
+            return _last_text_span(list(span.content))
+        return None
+    return None
+
+
+def _drop_empty_text_spans(spans: list[InlineSpan]) -> list[InlineSpan]:
+    """删除裁剪后为空的 TextSpan，并递归清理空链接。"""
+    result: list[InlineSpan] = []
+    for span in spans:
+        if isinstance(span, TextSpan) and not span.content:
+            continue
+        if isinstance(span, HyperlinkSpan):
+            children = _drop_empty_text_spans(list(span.content))
+            non_link_children = [child for child in children if not isinstance(child, HyperlinkSpan)]
+            if not non_link_children:
+                continue
+            span.content = non_link_children
+        result.append(span)
+    return result
+
+
+def _slice_inline_span(span: InlineSpan, start: int, end: int) -> InlineSpan | None:
+    """裁剪单个 Span 的局部可见区间。"""
+    if start >= end:
+        return None
+    if isinstance(span, TextSpan):
+        content = span.content[start:end]
+        return span.model_copy(update={"content": content}) if content else None
+    if isinstance(span, EquationInlineSpan):
+        content = span.content[start:end]
+        return span.model_copy(update={"content": content}) if content.strip() else None
+    if isinstance(span, CodeInlineSpan):
+        content = span.content[start:end]
+        return span.model_copy(update={"content": content}) if content else None
+    if isinstance(span, HyperlinkSpan):
+        children = slice_inline_spans(span.content, start, end)
+        non_link_children = [child for child in children if not isinstance(child, HyperlinkSpan)]
+        return span.model_copy(update={"content": non_link_children}) if non_link_children else None
+    return None
+
+
+__all__ = [
+    "inline_plain_text",
+    "join_inline_spans",
+    "map_text_span_content",
+    "normalize_inline_spans",
+    "replace_inline_text",
+    "slice_inline_spans",
+    "strip_inline_spans",
+]

docvortex/content/markup/__init__.py

@@ -0,0 +1,44 @@
+"""Flash EPUB 与 HTML 共用的静态标记文档投影能力。"""
+
+from docvortex.content.markup.anchors import (
+    AnchorTextNormalization,
+    AnchorVisibilityScope,
+    MarkupAnchorDocument,
+    MarkupAnchorPolicy,
+    MarkupAnchorRegistry,
+    canonical_anchor,
+    element_id,
+    visible_element_text,
+)
+from docvortex.content.markup.formula import (
+    FormulaDisplay,
+    FormulaExtraction,
+    FormulaSourceKind,
+    extract_formula,
+    strip_formula_delimiters,
+)
+from docvortex.content.markup.projector import MarkupContext, MarkupProjector, ResolvedMarkupImage
+from docvortex.content.markup.styles import ElementStyle, MarkupStylesheet, TextStyle, TextStyleDelta
+
+__all__ = [
+    "AnchorTextNormalization",
+    "AnchorVisibilityScope",
+    "ElementStyle",
+    "FormulaDisplay",
+    "FormulaExtraction",
+    "FormulaSourceKind",
+    "MarkupAnchorDocument",
+    "MarkupAnchorPolicy",
+    "MarkupAnchorRegistry",
+    "MarkupContext",
+    "MarkupProjector",
+    "MarkupStylesheet",
+    "ResolvedMarkupImage",
+    "TextStyle",
+    "TextStyleDelta",
+    "canonical_anchor",
+    "element_id",
+    "extract_formula",
+    "strip_formula_delimiters",
+    "visible_element_text",
+]

docvortex/content/markup/anchors.py

@@ -0,0 +1,188 @@
+"""集中建立静态 HTML/XHTML 标题、脚注与 fragment anchor 索引。"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+import hashlib
+import html
+import re
+from typing import Literal, Protocol, TypeAlias
+
+from lxml import etree  # type: ignore[reportMissingImports]
+
+from docvortex.content.markup.projector import local_name, visible_raw_text_with_style
+from docvortex.content.markup.styles import MarkupStylesheet, TextStyle
+from docvortex.foundation.type_identity import preserve_type_module
+
+
+AnchorVisibilityScope: TypeAlias = Literal["all_ancestors", "nearest_body"]
+AnchorTextNormalization: TypeAlias = Literal["unicode_whitespace", "xhtml_whitespace"]
+
+_HEADING_TAGS = frozenset({"h1", "h2", "h3", "h4", "h5", "h6"})
+_XHTML_WHITESPACE_RE = re.compile(r"[\t\r\n\f ]+")
+_XML_ID = "{http://www.w3.org/XML/1998/namespace}id"
+
+
+@dataclass(frozen=True, slots=True)
+class MarkupAnchorDocument:
+    """描述一个待建立 anchor 索引的 DOM、样式表与兼容可见性规则。"""
+
+    key: str
+    root: etree._Element
+    stylesheet: MarkupStylesheet
+    visibility_scope: AnchorVisibilityScope = "all_ancestors"
+    text_normalization: AnchorTextNormalization = "unicode_whitespace"
+
+
+class MarkupAnchorPolicy(Protocol):
+    """定义格式适配器生成标题、脚注 anchor 所需的稳定策略。"""
+
+    anchor_prefix: str
+    register_document_start: bool
+
+    def heading_identity(self, element: etree._Element, ordinal: int) -> str:
+        """返回当前标题参与稳定摘要的格式专属 identity。"""
+
+    def is_materializable_note(self, element: etree._Element, document: MarkupAnchorDocument) -> bool:
+        """判断当前元素是否是能够兑现文本 anchor 的格式专属脚注。"""
+
+    def note_identity(self, element: etree._Element, ordinal: int) -> str:
+        """返回当前脚注参与稳定摘要的格式专属 identity。"""
+
+
+def element_id(element: etree._Element) -> str | None:
+    """返回元素去除首尾空白后的 HTML id 或 xml:id。"""
+    value = (element.get("id") or element.get(_XML_ID) or "").strip()
+    return value or None
+
+
+def visible_element_text(element: etree._Element, document: MarkupAnchorDocument) -> str:
+    """按文档兼容配置解析祖先样式链，并返回最终可输出的纯文本。"""
+    inherited = TextStyle()
+    visibility_hidden = False
+    chain = [ancestor for ancestor in reversed(list(element.iterancestors())) if isinstance(ancestor.tag, str)]
+    if document.visibility_scope == "nearest_body":
+        body_index = next((index for index, ancestor in enumerate(chain) if local_name(ancestor) == "body"), None)
+        if body_index is not None:
+            chain = chain[body_index:]
+    chain.append(element)
+    for current in chain:
+        resolved = document.stylesheet.resolve(current, inherited, visibility_hidden)
+        if resolved.subtree_hidden:
+            return ""
+        inherited = resolved.text
+        visibility_hidden = resolved.visibility_hidden
+    value = visible_raw_text_with_style(element, document.stylesheet, inherited, visibility_hidden)
+    if document.text_normalization == "xhtml_whitespace":
+        return _XHTML_WHITESPACE_RE.sub(" ", html.unescape(value)).strip()
+    return " ".join(value.split())
+
+
+def canonical_anchor(prefix: str, document_key: str, identity: str) -> str:
+    """按格式前缀、文档 key 与 identity 生成稳定的二十位摘要 anchor。"""
+    digest = hashlib.sha256(f"{document_key}#{identity}".encode()).hexdigest()[:20]
+    return f"{prefix}-{digest}"
+
+
+class MarkupAnchorRegistry:
+    """统一登记多文档标题、脚注及源 fragment 到实际输出 anchor 的映射。"""
+
+    def __init__(self, documents: list[MarkupAnchorDocument], policy: MarkupAnchorPolicy) -> None:
+        """按调用方文档顺序建立稳定索引，并保留格式专属 identity 规则。"""
+        self._policy = policy
+        self._heading_anchors: dict[etree._Element, str] = {}
+        self._note_anchors: dict[etree._Element, str] = {}
+        self._targets: dict[tuple[str, str | None], str] = {}
+        self._heading_labels: dict[str, str] = {}
+        for document in documents:
+            self._register_document(document)
+
+    def _register_document(self, document: MarkupAnchorDocument) -> None:
+        """登记单个 DOM 的标题、脚注及全部可解析 fragment 别名。"""
+        headings: list[tuple[etree._Element, str]] = []
+        for element in document.root.iter():
+            if not isinstance(element.tag, str) or local_name(element) not in _HEADING_TAGS:
+                continue
+            if label := visible_element_text(element, document):
+                headings.append((element, label))
+        for ordinal, (heading, label) in enumerate(headings):
+            identity = self._policy.heading_identity(heading, ordinal)
+            anchor = canonical_anchor(self._policy.anchor_prefix, document.key, identity)
+            self._heading_anchors[heading] = anchor
+            self._heading_labels[anchor] = label
+
+        notes = [
+            element
+            for element in document.root.iter()
+            if isinstance(element.tag, str) and self._policy.is_materializable_note(element, document)
+        ]
+        for ordinal, note in enumerate(notes):
+            identity = self._policy.note_identity(note, ordinal)
+            self._note_anchors[note] = canonical_anchor(self._policy.anchor_prefix, document.key, identity)
+
+        if self._policy.register_document_start and headings:
+            self._targets[(document.key, None)] = self._heading_anchors[headings[0][0]]
+        for element in document.root.iter():
+            if not isinstance(element.tag, str) or not (fragment := element_id(element)):
+                continue
+            target_key = (document.key, fragment)
+            if target_key in self._targets:
+                continue
+            if anchor := self._target_anchor(element):
+                self._targets[target_key] = anchor
+
+    def _target_anchor(self, element: etree._Element) -> str | None:
+        """把任意 fragment 元素映射到自身、最近祖先或首个后代输出目标。"""
+        direct = self._heading_anchors.get(element) or self._note_anchors.get(element)
+        if direct is not None:
+            return direct
+        ancestor = next(
+            (parent for parent in element.iterancestors() if parent in self._heading_anchors or parent in self._note_anchors),
+            None,
+        )
+        if ancestor is not None:
+            return self._heading_anchors.get(ancestor) or self._note_anchors.get(ancestor)
+        descendant = next(
+            (
+                child
+                for child in element.iterdescendants()
+                if isinstance(child.tag, str) and (child in self._heading_anchors or child in self._note_anchors)
+            ),
+            None,
+        )
+        if descendant is None:
+            return None
+        return self._heading_anchors.get(descendant) or self._note_anchors.get(descendant)
+
+    def heading_anchor(self, heading: etree._Element) -> str | None:
+        """返回一个已登记标题的规范 anchor。"""
+        return self._heading_anchors.get(heading)
+
+    def heading_label(self, anchor: str) -> str | None:
+        """返回规范标题 anchor 对应的可见标题文本。"""
+        return self._heading_labels.get(anchor)
+
+    def note_anchor(self, note: etree._Element) -> str | None:
+        """返回一个已登记脚注的规范 anchor。"""
+        return self._note_anchors.get(note)
+
+    def resolve_target(self, document_key: str, fragment: str | None) -> str | None:
+        """按文档 key 与可选源 fragment 返回不带井号的规范 anchor。"""
+        return self._targets.get((document_key, fragment))
+
+
+__all__ = [
+    "AnchorTextNormalization",
+    "AnchorVisibilityScope",
+    "MarkupAnchorDocument",
+    "MarkupAnchorPolicy",
+    "MarkupAnchorRegistry",
+    "canonical_anchor",
+    "element_id",
+    "visible_element_text",
+]
+
+# 保持既有公开类型的 pickle 路径，所有旧、新入口指向同一个类。
+preserve_type_module(MarkupAnchorDocument, "docvortex.analyzers.native._shared.markup.anchors")
+preserve_type_module(MarkupAnchorPolicy, "docvortex.analyzers.native._shared.markup.anchors")
+preserve_type_module(MarkupAnchorRegistry, "docvortex.analyzers.native._shared.markup.anchors")