docvortex 0.2.1__py3-none-any.whl

docvortex/document/filetypes.py

@@ -0,0 +1,175 @@
+from __future__ import annotations
+
+import re
+from pathlib import Path
+from typing import Final
+
+
+# ── file types ─────────────────────────────────────────────────────
+
+# Legacy Office binary formats and RTF are natively parsed by model.flash.office converters.
+# Unsupported document/e-book/archive-like formats:
+# "key",
+# "mobi",
+# "numbers",
+# "pages",
+# Unsupported mail formats:
+# "eml",
+# "mbox",
+# Other doc formats:
+# "djvu/djv"
+# "epdf",
+# "xps",
+# "ps",
+# "eps/epsi/epi"
+# "ept/ept2/ept3"
+
+PDF_EXTENSIONS: frozenset[str] = frozenset({"pdf"})
+
+IMAGE_EXTENSIONS: frozenset[str] = frozenset({"png", "jpg", "jpeg", "webp", "gif", "bmp", "tiff", "jp2"})
+
+ODF_EXTENSIONS: frozenset[str] = frozenset({"odt", "ods", "odp"})
+
+EPUB_EXTENSIONS: frozenset[str] = frozenset({"epub"})
+
+OFD_EXTENSIONS: frozenset[str] = frozenset({"ofd"})
+
+OFFICE_EXTENSIONS: frozenset[str] = frozenset({"doc", "docx", "ppt", "pptx", "xls", "xlsx", "rtf"}) | ODF_EXTENSIONS
+
+HTML_EXTENSIONS: frozenset[str] = frozenset({"html", "htm", "shtml"})
+
+CSV_EXTENSIONS: frozenset[str] = frozenset({"csv", "tsv"})
+
+TEXT_EXTENSIONS: frozenset[str] = frozenset(
+    {"txt", "text", "ftxt", "md", "markdown", "mdx", "rst", "tex", "latex", "adoc", "asciidoc"}
+)
+
+TIERED_PARSE_EXTENSIONS: frozenset[str] = PDF_EXTENSIONS | IMAGE_EXTENSIONS
+
+PAGE_RANGE_PARSE_EXTENSIONS: frozenset[str] = PDF_EXTENSIONS
+
+FLASH_ONLY_PARSE_EXTENSIONS: frozenset[str] = (
+    OFFICE_EXTENSIONS | HTML_EXTENSIONS | CSV_EXTENSIONS | EPUB_EXTENSIONS | OFD_EXTENSIONS
+)
+
+PARSEABLE_EXTENSIONS: frozenset[str] = TIERED_PARSE_EXTENSIONS | FLASH_ONLY_PARSE_EXTENSIONS
+
+# Acceptable for doclib ingest
+INGESTIBLE_EXTENSIONS: frozenset[str] = PARSEABLE_EXTENSIONS | TEXT_EXTENSIONS
+
+# Used for doclib scan and watch
+DISCOVERABLE_EXTENSIONS: frozenset[str] = INGESTIBLE_EXTENSIONS - IMAGE_EXTENSIONS
+
+FILE_TYPE_BY_EXTENSION: dict[str, str] = {
+    **dict.fromkeys(PDF_EXTENSIONS, "pdf"),
+    **dict.fromkeys(IMAGE_EXTENSIONS, "image"),
+    **{ext: ext for ext in OFFICE_EXTENSIONS},
+    **dict.fromkeys(HTML_EXTENSIONS, "html"),
+    **dict.fromkeys(CSV_EXTENSIONS, "csv"),
+    **dict.fromkeys(EPUB_EXTENSIONS, "epub"),
+    **dict.fromkeys(OFD_EXTENSIONS, "ofd"),
+    "txt": "text",
+    "text": "text",
+    "ftxt": "text",
+    "md": "markdown",
+    "markdown": "markdown",
+    "mdx": "markdown",
+    "rst": "rst",
+    "tex": "tex",
+    "latex": "tex",
+    "adoc": "asciidoc",
+    "asciidoc": "asciidoc",
+}
+
+TEXT_FILE_TYPES: frozenset[str] = frozenset(FILE_TYPE_BY_EXTENSION[ext] for ext in TEXT_EXTENSIONS)
+
+MIME_TYPE_BY_EXTENSION: dict[str, str] = {
+    "pdf": "application/pdf",
+    "doc": "application/msword",
+    "docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+    "ppt": "application/vnd.ms-powerpoint",
+    "pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+    "xls": "application/vnd.ms-excel",
+    "xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+    "rtf": "application/rtf",
+    "odt": "application/vnd.oasis.opendocument.text",
+    "ods": "application/vnd.oasis.opendocument.spreadsheet",
+    "odp": "application/vnd.oasis.opendocument.presentation",
+    "csv": "text/csv",
+    "tsv": "text/tab-separated-values",
+    "epub": "application/epub+zip",
+    # OFD 尚无 IANA 注册 subtype；使用生态中通行的项目级 MIME 映射。
+    "ofd": "application/ofd",
+    "html": "text/html",
+    "htm": "text/html",
+    "shtml": "text/html",
+    "png": "image/png",
+    "jpg": "image/jpeg",
+    "jpeg": "image/jpeg",
+    "jp2": "image/jp2",
+    "webp": "image/webp",
+    "gif": "image/gif",
+    "bmp": "image/bmp",
+    "tiff": "image/tiff",
+    "txt": "text/plain",
+    "text": "text/plain",
+    "ftxt": "text/plain",
+    "md": "text/markdown",
+    "markdown": "text/markdown",
+    "mdx": "text/markdown",
+    "rst": "text/x-rst",
+    "tex": "text/x-tex",
+    "latex": "application/x-latex",
+    "adoc": "text/plain",
+    "asciidoc": "text/plain",
+}
+
+_RTF_HEADER_RE: Final = re.compile(
+    rb"\A(?:\xef\xbb\xbf)?[ \t\r\n]{0,64}\{\\rtf(?=[0-9])",
+    re.IGNORECASE,
+)
+
+
+def rtf_header_offset(file_bytes: bytes) -> int | None:
+    """返回 RTF 根组左花括号偏移；不接受带任意前缀的伪装文本。"""
+    match = _RTF_HEADER_RE.match(file_bytes)
+    if match is None:
+        return None
+    return file_bytes.find(b"{", 0, match.end())
+
+
+def normalize_parse_extension(path_or_ext: str | Path) -> str:
+    text = str(path_or_ext)
+    if text.startswith("."):
+        return text.lower().lstrip(".")
+    suffix = Path(text).suffix
+    if suffix:
+        return suffix.lower().lstrip(".")
+    return text.lower().lstrip(".")
+
+
+def file_type_for_extension(path_or_ext: str | Path) -> str:
+    ext = normalize_parse_extension(path_or_ext)
+    return FILE_TYPE_BY_EXTENSION.get(ext, ext or "unknown")
+
+
+def mime_type_for_extension(path_or_ext: str | Path, *, default: str = "application/octet-stream") -> str:
+    return MIME_TYPE_BY_EXTENSION.get(normalize_parse_extension(path_or_ext), default)
+
+
+def is_tiered_parse_extension(path_or_ext: str | Path) -> bool:
+    return normalize_parse_extension(path_or_ext) in TIERED_PARSE_EXTENSIONS
+
+
+def is_page_range_parse_extension(path_or_ext: str | Path) -> bool:
+    """仅允许 PDF 输入使用局部页范围解析。"""
+    return normalize_parse_extension(path_or_ext) in PAGE_RANGE_PARSE_EXTENSIONS
+
+
+def is_flash_only_parse_extension(path_or_ext: str | Path) -> bool:
+    return normalize_parse_extension(path_or_ext) in FLASH_ONLY_PARSE_EXTENSIONS
+
+
+def is_office_temp_lock_file(path: str | Path) -> bool:
+    file_path = Path(path)
+    return file_path.name.startswith("~$") and file_path.suffix.lower().lstrip(".") in OFFICE_EXTENSIONS

docvortex/document/page_range.py

@@ -0,0 +1,167 @@
+"""所有入口共享的页码范围语法、求值和格式化；默认选页策略由调用方决定。"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Iterable
+
+from loguru import logger
+
+from ..errors import InvalidRequestError
+
+PAGE_RANGE_DESCRIPTION = (
+    "PDF page selection: 1-based inclusive ranges, e.g. '1-5,8,r3-r1'; "
+    "'r1' is the last page and 'all' selects every page. Blank means unspecified. "
+    "Selections are sorted and deduplicated; out-of-bounds pages are omitted. "
+    "Reversed ranges, '~' and negative page numbers are invalid."
+)
+_SEGMENT_PATTERN = re.compile(r"(r?[1-9][0-9]*)(?:\s*-\s*(r?[1-9][0-9]*))?")
+
+
+def get_end_page_id(end_page_id: int | None, pdf_page_num: int) -> int:
+    """归一化旧 CLI 的 0-based 结束页，越界时钳制到最后一页。"""
+    normalized_end_page_id = end_page_id if end_page_id is not None and end_page_id >= 0 else pdf_page_num - 1
+    if normalized_end_page_id > pdf_page_num - 1:
+        logger.debug("end_page_id is out of range, use images length")
+        normalized_end_page_id = pdf_page_num - 1
+    return normalized_end_page_id
+
+
+def _invalid_range(raw: str | None, reason: str) -> InvalidRequestError:
+    """统一生成可跨 Python、CLI 和 HTTP 传递的页码错误。"""
+    return InvalidRequestError("page_range_invalid", f"Invalid page range {raw!r}: {reason}", "page_range")
+
+
+def _parse_endpoint(token: str) -> int:
+    """以负整数在内部标记倒数端点，外部输入只接受 rN。"""
+    return -int(token[1:]) if token.startswith("r") else int(token)
+
+
+def _parse_segments(raw: str | None) -> list[tuple[int, int]] | None:
+    """解析完整表达式；None 表示全部，倒数端点留待获得总页数后求值。"""
+    value = (raw or "").strip()
+    if not value or value == "all":
+        return None
+    segments: list[tuple[int, int]] = []
+    for part in value.split(","):
+        match = _SEGMENT_PATTERN.fullmatch(part.strip())
+        if match is None:
+            raise _invalid_range(raw, "expected a page, an inclusive range, or 'all'")
+        try:
+            start = _parse_endpoint(match[1])
+            end = _parse_endpoint(match[2]) if match[2] else start
+        except ValueError as exc:
+            raise _invalid_range(raw, "page number is too large") from exc
+        if (start > 0) == (end > 0) and start > end:
+            raise _invalid_range(raw, "reversed ranges are not supported")
+        segments.append((start, end))
+    return segments
+
+
+def _merge_intervals(intervals: Iterable[tuple[int, int]]) -> list[tuple[int, int]]:
+    """合并已求值区间，无需逐页展开即可排序、去重和连接相邻区间。"""
+    merged: list[tuple[int, int]] = []
+    for start, end in sorted(intervals):
+        if merged and start <= merged[-1][1] + 1:
+            merged[-1] = (merged[-1][0], max(end, merged[-1][1]))
+        else:
+            merged.append((start, end))
+    return merged
+
+
+def _format_endpoint(value: int) -> str:
+    """把内部端点转换为正整数或 rN 文本。"""
+    return str(value) if value > 0 else f"r{-value}"
+
+
+def _format_intervals(intervals: Iterable[tuple[int, int]]) -> str:
+    """生成无空白的范围文本，单页只输出一个端点。"""
+    return ",".join(
+        _format_endpoint(start) if start == end else f"{_format_endpoint(start)}-{_format_endpoint(end)}"
+        for start, end in intervals
+    )
+
+
+def normalize_page_range_input(raw: str | None) -> str:
+    """校验未绑定文档的输入，清理空白并保留 all/rN；空值统一为未指定。"""
+    segments = _parse_segments(raw)
+    if segments is None:
+        return "all" if (raw or "").strip() else ""
+    if all(start > 0 and end > 0 for start, end in segments):
+        segments = _merge_intervals(segments)
+    return _format_intervals(segments)
+
+
+def _resolved_intervals(raw: str | None, page_count: int) -> list[tuple[int, int]]:
+    """按文档总页数求值并裁剪，先验证所有区间再合并有效交集。"""
+    segments = _parse_segments(raw)
+    if page_count <= 0:
+        raise _invalid_range(raw, "document has no available pages")
+    if segments is None:
+        return [(1, page_count)]
+    intervals: list[tuple[int, int]] = []
+    for start, end in segments:
+        start = start if start > 0 else page_count + start + 1
+        end = end if end > 0 else page_count + end + 1
+        if start > end:
+            raise _invalid_range(raw, "reversed ranges are not supported")
+        lo, hi = max(1, start), min(page_count, end)
+        if lo <= hi:
+            intervals.append((lo, hi))
+    if not intervals:
+        raise _invalid_range(raw, "selection does not contain any available pages")
+    return _merge_intervals(intervals)
+
+
+def parse_page_range(raw: str, page_count: int) -> list[int]:
+    """把 1-based 表达式转换为去重升序的 0-based 页索引，空值选择全部。"""
+    return [page_idx for start, end in _resolved_intervals(raw, page_count) for page_idx in range(start - 1, end)]
+
+
+def expand_page_range(raw: str | None, page_count: int) -> str:
+    """将含 all/rN 的表达式展开为实际文档的正整数规范范围。"""
+    return _format_intervals(_resolved_intervals(raw, page_count))
+
+
+def _absolute_intervals(raw: str) -> list[tuple[int, int]]:
+    """读取已求值范围，兼容历史半角 ~；空文本为空集合，禁止 all/rN。"""
+    if not raw.strip():
+        return []
+    # 只在结果读取边界兼容旧分隔符，新请求及全角波浪号仍由严格语法拒绝。
+    segments = _parse_segments(raw.replace("~", "-"))
+    if segments is None or any(start < 0 or end < 0 for start, end in segments):
+        raise _invalid_range(raw, "page count is required to resolve all/rN")
+    return _merge_intervals(segments)
+
+
+def normalize_result_page_range(raw: str) -> str:
+    """将已求值的新旧结果范围规范化为连字符格式，不修改持久化记录或文件名。"""
+    return _format_intervals(_absolute_intervals(raw))
+
+
+def parse_page_range_set(raw: str) -> set[int]:
+    """把已求值的范围转换为 1-based 页码集合，供缓存覆盖与内容过滤使用。"""
+    return {page_no for start, end in _absolute_intervals(raw) for page_no in range(start, end + 1)}
+
+
+def count_pages_in_range(raw: str) -> int:
+    """统计已求值范围的唯一页数，不为统计分配逐页集合。"""
+    return sum(end - start + 1 for start, end in _absolute_intervals(raw))
+
+
+def format_page_range(page_numbers: Iterable[int]) -> str:
+    """将 1-based 页码集合格式化为去重升序的连续范围；空集合输出空字符串。"""
+    return _format_intervals(_merge_intervals((page_no, page_no) for page_no in page_numbers))
+
+
+__all__ = [
+    "PAGE_RANGE_DESCRIPTION",
+    "count_pages_in_range",
+    "expand_page_range",
+    "format_page_range",
+    "get_end_page_id",
+    "normalize_page_range_input",
+    "normalize_result_page_range",
+    "parse_page_range",
+    "parse_page_range_set",
+]

docvortex/document/pdf/__init__.py

@@ -0,0 +1,19 @@
+"""PDF 文档访问、显式分类及共享文本契约。"""
+
+from .document import PDFDocument, PDFPage, PDFPageTextGeometry, get_lines_from_chars
+from .text import Bbox, Char, Line, Span
+from .pdfium import PdfiumFontError, PdfiumRuntimeInfo, initialize_pdfium_runtime
+
+__all__ = [
+    "PDFDocument",
+    "PDFPage",
+    "PDFPageTextGeometry",
+    "get_lines_from_chars",
+    "Bbox",
+    "Char",
+    "Line",
+    "Span",
+    "initialize_pdfium_runtime",
+    "PdfiumRuntimeInfo",
+    "PdfiumFontError",
+]