docvortex 0.2.6__tar.gz → 0.3.0__tar.gz

{docvortex-0.2.6/src/docvortex.egg-info → docvortex-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: docvortex
-Version: 0.2.6
+Version: 0.3.0
 Summary: A fast, multi-format document parsing and conversion engine
 License-Expression: MIT
 Project-URL: Repository, https://github.com/myhloli/DocVortex
@@ -38,7 +38,6 @@ Requires-Dist: mathml2omml==0.0.2
 Requires-Dist: resvg_py<0.6,>=0.5.0
 Requires-Dist: reportlab
 Requires-Dist: ziamath<0.14,>=0.13
-Requires-Dist: fast-langdetect<0.3.0,>=0.2.3
 Requires-Dist: magika<1.1.0,>=0.6.2
 Requires-Dist: opencv-python>=4.11.0.86
 Provides-Extra: test
@@ -209,3 +208,10 @@ print(inspection.metadata.document.authors)
 All 15 native document formats support this API. Normal parsing also carries these properties in
 `metadata.document`, preserving the original PDF properties across page selections. See the
 [field definitions, format matrix, and compatibility notes](docs/METADATA.md).
+
+
+### Python 3.14 and text joining (0.3.0)
+
+DocVortex supports ordinary CPython 3.10–3.14. Physical text lines are joined by deterministic Unicode boundary rules, with URL and dehyphenation safeguards. Language detection models and the fast-langdetect / fasttext-predict dependencies are no longer required.
+
+The shared `resolve_text_line_boundary` and `merge_text_line_contents` helpers no longer accept `block_language`; `detect_lang` has been removed. Hosts should call the shared boundary helpers directly. OCR language selection and Magika code-language recognition are unchanged.

{docvortex-0.2.6 → docvortex-0.3.0}/README.md

@@ -155,3 +155,10 @@ print(inspection.metadata.document.authors)
 All 15 native document formats support this API. Normal parsing also carries these properties in
 `metadata.document`, preserving the original PDF properties across page selections. See the
 [field definitions, format matrix, and compatibility notes](docs/METADATA.md).
+
+
+### Python 3.14 and text joining (0.3.0)
+
+DocVortex supports ordinary CPython 3.10–3.14. Physical text lines are joined by deterministic Unicode boundary rules, with URL and dehyphenation safeguards. Language detection models and the fast-langdetect / fasttext-predict dependencies are no longer required.
+
+The shared `resolve_text_line_boundary` and `merge_text_line_contents` helpers no longer accept `block_language`; `detect_lang` has been removed. Hosts should call the shared boundary helpers directly. OCR language selection and Magika code-language recognition are unchanged.

{docvortex-0.2.6 → docvortex-0.3.0}/pyproject.toml

@@ -27,7 +27,7 @@ dependencies = [
     "mammoth>=1.11.0,<2", "openpyxl>=3.1.5,<4", "olefile>=0.47,<1",
     "pylatexenc>=2.10,<3", "latex2mathml>=3.81.0,<4", "mathml2omml==0.0.2",
     "resvg_py>=0.5.0,<0.6",
-    "reportlab", "ziamath>=0.13,<0.14", "fast-langdetect>=0.2.3,<0.3.0",
+    "reportlab", "ziamath>=0.13,<0.14",
     "magika>=0.6.2,<1.1.0", "opencv-python>=4.11.0.86",
 ]
 

{docvortex-0.2.6 → docvortex-0.3.0}/src/docvortex/analyzers/native/pdf/table_materialization.py

@@ -16,7 +16,6 @@ from .table_text_styles import render_native_table_html_with_scripts
 from ....foundation.text import merge_text_line_contents
 from .spatial_text import project_pdf_table_text
 from ....schema import BBox
-from ....foundation.language import detect_lang
 from .models import _LineItem, _PageSource, _TableAnnotation, _TableCandidate
 from .geometry import (
     _bbox_axis_overlap_ratio,
@@ -320,14 +319,7 @@ def _merge_table_annotation_content(line_texts: list[str]) -> str:
     normalized_lines = [normalized for text in line_texts if (normalized := _normalize_native_run_text(str(text or "")))]
     if not normalized_lines:
         return ""
-    try:
-        block_language = detect_lang("".join(normalized_lines))
-    except Exception:
-        block_language = ""
-    return merge_text_line_contents(
-        normalized_lines,
-        block_language=block_language,
-    )
+    return merge_text_line_contents(normalized_lines)
 
 
 def _table_body_materialization_bbox(

{docvortex-0.2.6 → docvortex-0.3.0}/src/docvortex/analyzers/native/pdf/table_recovery/text.py

@@ -234,7 +234,6 @@ def _cell_row_separator(
 
     _processed_previous, separator = resolve_text_line_boundary(
         accumulated_content,
-        block_language="en",
         next_content=next_line,
     )
     if separator != " ":

{docvortex-0.2.6 → docvortex-0.3.0}/src/docvortex/analyzers/native/pdf/tables.py

@@ -107,9 +107,6 @@ from ....schema import (
 from ....document.pdf.document import (
     PDFPathInfo as PDFPathInfo,
 )
-from ....foundation.language import (
-    detect_lang as detect_lang,
-)
 from .models import (
     _Fragment as _Fragment,
     _LineItem as _LineItem,

{docvortex-0.2.6 → docvortex-0.3.0}/src/docvortex/analyzers/native/pdf/text_assembly/common.py

@@ -5,7 +5,6 @@ from __future__ import annotations
 import re
 from typing import Any, Sequence
 
-from .....foundation.language import detect_lang
 from .....foundation.text import merge_text_line_contents
 from ..geometry import _bbox_axis_overlap_ratio, _bbox_center_y, _bbox_union_many
 from ..native_text import _normalize_native_run_text
@@ -437,14 +436,7 @@ def _merge_text_line_content(line_texts: Sequence[str]) -> str:
     normalized_lines = [text for text in normalized_lines if text]
     if not normalized_lines:
         return ""
-    try:
-        block_language = detect_lang("".join(normalized_lines))
-    except Exception:
-        block_language = ""
-    return merge_text_line_contents(
-        normalized_lines,
-        block_language=block_language,
-    )
+    return merge_text_line_contents(normalized_lines)
 
 
 __all__ = [

{docvortex-0.2.6 → docvortex-0.3.0}/src/docvortex/content/inline.py

@@ -2,15 +2,12 @@
 
 from __future__ import annotations
 
-import re
 from copy import deepcopy
 from typing import Callable, Iterable
 
 from ..schema import CodeInlineSpan, EquationInlineSpan, HyperlinkSpan, InlineSpan, TextSpan, parse_inline_spans
-from ..foundation.language import detect_lang
-from ..foundation.text import CJK_LANGS, resolve_text_line_boundary
-
-_CJK_RE = re.compile(r"[\u3040-\u30ff\u3400-\u9fff\uac00-\ud7af]")
+from ..foundation.text import resolve_text_line_boundary
+from ..foundation.language import remove_invalid_surrogates
 
 
 def normalize_inline_spans(spans: Iterable[InlineSpan | dict[str, object]]) -> list[InlineSpan]:
@@ -63,7 +60,7 @@ def join_inline_spans(contents: Iterable[Iterable[InlineSpan]]) -> list[InlineSp
     """按物理段落边界规则合并多组 Span，并保持结构化语义。"""
     merged: list[InlineSpan] = []
     for content in contents:
-        current = normalize_inline_spans(list(content))
+        current = map_text_span_content(list(content), remove_invalid_surrogates)
         if not current:
             continue
         # 边界裁剪最多清空最后一个根节点；保留其前驱以重新合并新相邻的 Span。
@@ -138,7 +135,7 @@ def map_text_span_content(spans: Iterable[InlineSpan], transform: Callable[[str]
 
 
 def _join_inline_span_sequences(previous: list[InlineSpan], current: list[InlineSpan]) -> None:
-    """在两组 Span 之间应用语言相关的换行拼接规则。"""
+    """按两组 Span 的可见边界拼接，只修改可写文本叶子，保留公式和代码。"""
     previous_visible = inline_plain_text(previous).rstrip()
     current_visible = inline_plain_text(current).lstrip()
     if not previous_visible or not current_visible:
@@ -151,30 +148,15 @@ def _join_inline_span_sequences(previous: list[InlineSpan], current: list[Inline
     if first_text is not None:
         object.__setattr__(first_text, "content", first_text.content.lstrip())
 
-    language = _detect_boundary_language(f"{previous_visible}{current_visible}")
+    processed, separator = resolve_text_line_boundary(previous_visible, next_content=current_visible)
     if last_text is not None:
-        processed, separator = resolve_text_line_boundary(
-            last_text.content,
-            block_language=language,
-            next_content=current_visible,
-        )
-        object.__setattr__(last_text, "content", processed)
-    else:
-        separator = "" if language in CJK_LANGS else " "
+        # 共享规则最多删除一个行末断词符；不能把整个可见投影写回单一叶子。
+        if processed == previous_visible[:-1] and last_text.content.endswith(previous_visible[-1]):
+            object.__setattr__(last_text, "content", last_text.content[:-1])
     if separator:
         previous.append(TextSpan(type="text", content=separator))
 
 
-def _detect_boundary_language(content: str) -> str:
-    """检测段落边界语言，短 CJK 文本优先使用字符范围兜底。"""
-    if _CJK_RE.search(content):
-        return "zh"
-    try:
-        return detect_lang(content)
-    except Exception:
-        return ""
-
-
 def _first_text_span(spans: list[InlineSpan]) -> TextSpan | None:
     """返回首个可见叶子为文字时对应的 TextSpan。"""
     for span in spans:

{docvortex-0.2.6 → docvortex-0.3.0}/src/docvortex/foundation/language.py

@@ -1,43 +1,14 @@
-import os
 import re
-import unicodedata
 
 DEFAULT_CODE_LANGUAGE = "txt"
 _INVALID_SURROGATES = re.compile("[\ud800-\udfff]")
 
 
-def _detect_language(text: str) -> object:
-    """首次检测时配置本地模型缓存并惰性加载语言识别器。"""
-    from fast_langdetect import detect_language
-
-    return detect_language(text, low_memory=True)
-
-
 def remove_invalid_surrogates(text: str) -> str:
     """等价移除代理码点；常见合法 Unicode 文本无需逐字符 Python 扫描。"""
     return _INVALID_SURROGATES.sub("", text)
 
 
-def detect_lang(text: str) -> str:
-    if len(text) == 0:
-        return ""
-
-    text = text.replace("\n", "")
-    text = remove_invalid_surrogates(text)
-
-    try:
-        lang_upper = _detect_language(text)
-    except Exception:
-        html_no_ctrl_chars = "".join([c for c in text if unicodedata.category(c)[0] not in ["C"]])
-        lang_upper = _detect_language(html_no_ctrl_chars)
-
-    try:
-        lang = lang_upper.lower()
-    except Exception:
-        lang = ""
-    return lang
-
-
 def _normalize_text_for_language_guess(code: str) -> str:
     """移除孤立代理字符并还原合法代理对，供代码语言识别使用。"""
     if not code:
@@ -80,13 +51,4 @@ def guess_code_language(code: str) -> str:
     return lang if lang != "unknown" else DEFAULT_CODE_LANGUAGE
 
 
-if __name__ == "__main__":
-    print(os.getenv("FTLANG_CACHE"))
-    print(detect_lang("This is a test."))
-    print(detect_lang("<html>This is a test</html>"))
-    print(detect_lang("这个是中文测试。"))
-    print(detect_lang("<html>这个是中文测试。</html>"))
-    print(detect_lang("〖\ud835\udc46\ud835〗这是个包含utf-16的中文测试"))
-
-
-__all__ = ["DEFAULT_CODE_LANGUAGE", "detect_lang", "guess_code_language", "remove_invalid_surrogates"]
+__all__ = ["DEFAULT_CODE_LANGUAGE", "guess_code_language", "remove_invalid_surrogates"]

{docvortex-0.2.6 → docvortex-0.3.0}/src/docvortex/foundation/text.py

@@ -1,10 +1,33 @@
 """跨模型共享的文本字符规范化与换行连接规则。"""
 
 import re
+import unicodedata
 from collections.abc import Sequence
 
-# 中日韩文本的物理换行通常不需要插入额外空格，集中定义以供各后端和渲染器共享。
-CJK_LANGS = frozenset({"zh", "ja", "ko"})
+from .language import remove_invalid_surrogates
+
+# 使用固定码点区间覆盖扩展汉字、假名和中日标点，避免不同 Python 的 Unicode 版本改变结果。
+_UNSPACED_RANGES = (
+    (0x3000, 0x303F),
+    (0x3040, 0x30FF),
+    (0x31F0, 0x31FF),
+    (0x3400, 0x4DBF),
+    (0x4E00, 0x9FFF),
+    (0xF900, 0xFAFF),
+    (0xFF61, 0xFF9F),
+    (0x1AFF0, 0x1AFFF),
+    (0x1B000, 0x1B16F),
+    (0x20000, 0x2EE5F),
+    (0x2F800, 0x2FA1F),
+    (0x30000, 0x323AF),
+)
+_CJK_PUNCTUATION = frozenset("，。！？；：、（）【】《》〈〉「」『』〔〕［］｛｝")
+_CLOSING_PUNCTUATION = frozenset(",.;:!?%")
+# 明确的复合词前缀与连接词只用于保守保留硬连字符，不承担语言识别或词典分词。
+_COMPOUND_PREFIXES = frozenset({"open", "non", "self", "cross", "anti", "pre", "post", "co", "semi", "multi", "quasi"})
+_COMPOUND_CONNECTORS = frozenset({"of", "the", "to", "and", "in", "for", "on", "by", "with"})
+# 只投影已有的行内样式标记，不把任意 HTML 或公式解析为可改写内容。
+_INLINE_STYLE_TAG_RE = re.compile(r"</?(?:sup|sub|b|strong|i|em|u|s|del|strike)\b[^>]*>", re.IGNORECASE)
 
 # PDF 文本抽取时，英文跨行断词可能被编码为多种 hyphen 字符。
 # 这里只用于判断“行末英文断词符”，不要扩展到 en/em dash 等普通破折号。
@@ -45,48 +68,82 @@ def _url_spans_line_boundary(previous_content: str, next_content: str) -> bool:
 def resolve_text_line_boundary(
     previous_content: str,
     *,
-    block_language: str,
     next_content: str,
 ) -> tuple[str, str]:
     """返回处理后的上一行内容和本次物理行边界分隔符。
 
     严格横跨边界的 URL 候选直接连接，但下一行自身为完整 URL 时保留空格。
-    其余 CJK 文本直接连接物理行，普通西文行插入一个空格。西文行末如果是
-    合法的 hyphen，则始终直接连接下一行，并仅在下一行以小写字母开头时删除
-    hyphen。
+    按当前可见边界判断空格，不检测整段语言。汉字和假名直接连接，韩文与
+    西文保留词间空格；既定 URL 和英文断词规则优先，且只改写真实的行末断词符。
     """
-    processed_content = previous_content.rstrip()
+    processed_content = remove_invalid_surrogates(previous_content).rstrip()
     if not processed_content:
         return "", ""
-    stripped_next = next_content.lstrip()
+    stripped_next = remove_invalid_surrogates(next_content).lstrip()
+    if not stripped_next:
+        return processed_content, ""
     if _url_spans_line_boundary(processed_content, stripped_next):
         if _URL_AT_LINE_START_RE.match(stripped_next):
             return processed_content, " "
         return processed_content, ""
-    if block_language in CJK_LANGS:
+    if is_hyphen_at_line_end(processed_content):
+        previous_word = re.search(r"([A-Za-z]+)[-\u00ad\u2010\u2011\u2043]$", processed_content)
+        # GLGE-difficult 等缩写复合词的连接符有语义，不能当作普通断词符删除。
+        acronym = previous_word is not None and len(previous_word[1]) > 1 and previous_word[1].isupper()
+        hard_compound = (
+            previous_word is not None
+            and processed_content.endswith("-")
+            and (
+                previous_word[1].lower() in _COMPOUND_PREFIXES
+                or (
+                    previous_word[1].lower() in _COMPOUND_CONNECTORS
+                    and previous_word.start() > 0
+                    and processed_content[previous_word.start() - 1] == "-"
+                )
+            )
+        )
+        if stripped_next[0].islower() and not acronym and not hard_compound:
+            return processed_content[:-1], ""
+        return processed_content, ""
+    if re.search(r"\d[-–]$", processed_content) and stripped_next[0].isdigit():
+        return processed_content, ""
+    previous_char = _boundary_character(processed_content, from_end=True)
+    next_char = _boundary_character(stripped_next, from_end=False)
+    if not previous_char or not next_char:
+        return processed_content, ""
+    if _is_unspaced_character(previous_char) or _is_unspaced_character(next_char):
         return processed_content, ""
-    if not is_hyphen_at_line_end(processed_content):
-        return processed_content, " "
-    if stripped_next and stripped_next[0].islower():
-        return processed_content[:-1], ""
-    return processed_content, ""
+    if unicodedata.category(previous_char) in {"Ps", "Pi"}:
+        return processed_content, ""
+    if unicodedata.category(next_char) in {"Pe", "Pf"} or next_char in _CLOSING_PUNCTUATION:
+        return processed_content, ""
+    return processed_content, " "
+
+
+def _boundary_character(content: str, *, from_end: bool) -> str:
+    """读取可见边界字符，忽略样式标记及附着的组合字符，不改写原始正文。"""
+    visible = _INLINE_STYLE_TAG_RE.sub("", content).strip()
+    characters = reversed(visible) if from_end else iter(visible)
+    return next((char for char in characters if unicodedata.category(char)[0] not in {"M", "C"}), "")
+
+
+def _is_unspaced_character(char: str) -> bool:
+    """判断汉字、假名及中日标点，韩文字母不属于无空格文字。"""
+    return char in _CJK_PUNCTUATION or any(start <= ord(char) <= end for start, end in _UNSPACED_RANGES)
 
 
 def merge_text_line_contents(
     line_contents: Sequence[str],
-    *,
-    block_language: str,
 ) -> str:
     """按累计文本上下文折叠物理行，支持跨越三行以上的 URL 连续拼接。"""
 
-    normalized_lines = [str(content) for content in line_contents if str(content)]
+    normalized_lines = [cleaned for content in line_contents if (cleaned := remove_invalid_surrogates(str(content)).strip())]
     if not normalized_lines:
         return ""
     merged_content = normalized_lines[0]
     for current_line in normalized_lines[1:]:
         merged_content, separator = resolve_text_line_boundary(
             merged_content,
-            block_language=block_language,
             next_content=current_line,
         )
         merged_content = f"{merged_content}{separator}{current_line}"

{docvortex-0.2.6 → docvortex-0.3.0}/src/docvortex/version.py

@@ -1,3 +1,3 @@
-__version__ = "0.2.6"
+__version__ = "0.3.0"
 
 __all__ = ["__version__"]

{docvortex-0.2.6 → docvortex-0.3.0/src/docvortex.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: docvortex
-Version: 0.2.6
+Version: 0.3.0
 Summary: A fast, multi-format document parsing and conversion engine
 License-Expression: MIT
 Project-URL: Repository, https://github.com/myhloli/DocVortex
@@ -38,7 +38,6 @@ Requires-Dist: mathml2omml==0.0.2
 Requires-Dist: resvg_py<0.6,>=0.5.0
 Requires-Dist: reportlab
 Requires-Dist: ziamath<0.14,>=0.13
-Requires-Dist: fast-langdetect<0.3.0,>=0.2.3
 Requires-Dist: magika<1.1.0,>=0.6.2
 Requires-Dist: opencv-python>=4.11.0.86
 Provides-Extra: test
@@ -209,3 +208,10 @@ print(inspection.metadata.document.authors)
 All 15 native document formats support this API. Normal parsing also carries these properties in
 `metadata.document`, preserving the original PDF properties across page selections. See the
 [field definitions, format matrix, and compatibility notes](docs/METADATA.md).
+
+
+### Python 3.14 and text joining (0.3.0)
+
+DocVortex supports ordinary CPython 3.10–3.14. Physical text lines are joined by deterministic Unicode boundary rules, with URL and dehyphenation safeguards. Language detection models and the fast-langdetect / fasttext-predict dependencies are no longer required.
+
+The shared `resolve_text_line_boundary` and `merge_text_line_contents` helpers no longer accept `block_language`; `detect_lang` has been removed. Hosts should call the shared boundary helpers directly. OCR language selection and Magika code-language recognition are unchanged.

{docvortex-0.2.6 → docvortex-0.3.0}/src/docvortex.egg-info/SOURCES.txt

@@ -560,6 +560,7 @@ tests/unittest/test_structured_content_render.py
 tests/unittest/test_table_merge.py
 tests/unittest/test_table_merge_cell_merge.py
 tests/unittest/test_text_block_anchor.py
+tests/unittest/test_text_boundary_rules.py
 tests/unittest/test_text_utils.py
 tests/unittest/pdfs/flash_table_annotations_synthetic.pdf
 tests/unittest/pdfs/metabolic_pathway_page_3.pdf

{docvortex-0.2.6 → docvortex-0.3.0}/src/docvortex.egg-info/requires.txt

@@ -22,7 +22,6 @@ mathml2omml==0.0.2
 resvg_py<0.6,>=0.5.0
 reportlab
 ziamath<0.14,>=0.13
-fast-langdetect<0.3.0,>=0.2.3
 magika<1.1.0,>=0.6.2
 opencv-python>=4.11.0.86