ko-copycheck 0.1.1__py3-none-any.whl

ko_copycheck/__init__.py

@@ -0,0 +1,11 @@
+"""ko-copycheck: offline plagiarism checker for Korean prose.
+
+Public API: :func:`compare`, :class:`Config`, :class:`Report`. Everything else is
+internal and may change.
+"""
+from .engine import compare
+from .config import Config
+from .score import Report
+
+__version__ = "0.1.1"
+__all__ = ["compare", "Config", "Report", "__version__"]

ko_copycheck/cli.py

@@ -0,0 +1,225 @@
+"""Command line interface: scan, pair, extract."""
+from __future__ import annotations
+
+import argparse
+import sys
+import unicodedata
+from pathlib import Path
+
+from . import __version__
+from .config import Config
+from .engine import compare
+from .extract import SUPPORTED, Limits, SandboxedExtractor
+from .report import write_csv, write_json, write_report
+from .report.html import pair_page
+from .normalize import normalize_text
+from .score import Failure, Report
+
+EXIT_OK, EXIT_ERROR = 0, 2
+
+
+def _add_tuning(p: argparse.ArgumentParser) -> None:
+    d = Config()
+    p.add_argument("--prompt", action="append", default=[], metavar="FILE",
+                   help="assignment text every submission may quote (repeatable); matching parts are ignored")
+    p.add_argument("--k", type=int, default=d.k, help=f"k-gram length in characters (default {d.k})")
+    p.add_argument("--w", type=int, default=d.w, help=f"winnowing window (default {d.w})")
+    p.add_argument("--min-match", type=int, default=d.min_match, help=f"shortest verbatim match in characters (default {d.min_match})")
+    p.add_argument("--dice", type=float, default=d.dice, help=f"paraphrase Dice threshold (default {d.dice})")
+    p.add_argument("--min-sentence", type=int, default=d.min_sentence, help=f"shortest sentence for paraphrase layer (default {d.min_sentence})")
+    p.add_argument("--group-ratio", type=float, default=d.group_ratio, help=f"share of documents that makes a passage common (default {d.group_ratio})")
+    p.add_argument("--timeout", type=float, default=d.timeout, help=f"seconds allowed per file extraction (default {d.timeout:g})")
+
+
+def _config(a: argparse.Namespace) -> Config:
+    return Config(k=a.k, w=a.w, min_match=a.min_match, dice=a.dice, min_sentence=a.min_sentence,
+                  group_ratio=a.group_ratio, timeout=a.timeout)
+
+
+def build_parser() -> argparse.ArgumentParser:
+    ap = argparse.ArgumentParser(
+        prog="ko-copycheck",
+        description="Offline plagiarism checker for Korean prose submissions. Compares files with each other only; nothing is uploaded.",
+    )
+    ap.add_argument("--version", action="version", version=f"ko-copycheck {__version__}")
+    sub = ap.add_subparsers(dest="command", required=True)
+
+    scan = sub.add_parser("scan", help="compare every pair of files in a folder")
+    scan.add_argument("folder", type=Path)
+    scan.add_argument("--out", type=Path, default=Path("report"), help="output folder (default ./report)")
+    scan.add_argument("--recursive", action="store_true", help="also read sub-folders")
+    scan.add_argument("--top", type=int, default=300, help="write side-by-side pages for at most this many pairs (default 300)")
+    _add_tuning(scan)
+
+    pair = sub.add_parser("pair", help="compare two files and write one side-by-side HTML page")
+    pair.add_argument("left", type=Path)
+    pair.add_argument("right", type=Path)
+    pair.add_argument("--out", type=Path, default=Path("pair.html"), help="output HTML file (default ./pair.html)")
+    _add_tuning(pair)
+
+    ex = sub.add_parser("extract", help="print the text extracted from one file (check how it is read)")
+    ex.add_argument("file", type=Path)
+    ex.add_argument("--timeout", type=float, default=Config().timeout, help="seconds allowed (default 60)")
+    return ap
+
+
+def _say(msg: str) -> None:
+    print(msg, file=sys.stderr, flush=True)
+
+
+def _collect(folder: Path, recursive: bool, exclude: set[Path]) -> list[Path]:
+    it = folder.rglob("*") if recursive else folder.glob("*")
+    files = [p for p in it if p.is_file() and p.suffix.lower() in SUPPORTED and not p.name.startswith((".", "~$"))]
+    return sorted(p for p in files if p.resolve() not in exclude)
+
+
+def _extract_all(paths: list[Path], names: list[str], timeout: float, min_chars: int = 0) -> tuple[dict[str, str], list[Failure]]:
+    texts: dict[str, str] = {}
+    failures: list[Failure] = []
+    with SandboxedExtractor(timeout=timeout) as ex:
+        for i, (path, name) in enumerate(zip(paths, names), 1):
+            res = ex.extract(path)
+            if res.ok and res.text is not None and len(normalize_text(res.text)) < min_chars:
+                failures.append(Failure(name, f"no_text: fewer than {min_chars} readable characters (empty file?)"))
+            elif res.ok and res.text is not None:
+                texts[name] = res.text
+            else:
+                failures.append(Failure(name, res.error or "unknown"))
+            _say(f"extract {i}/{len(paths)}  failed {len(failures)}")
+    return texts, failures
+
+
+def _read_prompts(paths: list[str], timeout: float) -> list[str] | None:
+    files = [Path(p) for p in paths]
+    for f in files:
+        if not f.is_file():
+            _say(f"error: prompt file not found: {f}")
+            return None
+    if not files:
+        return []
+    texts, failures = _extract_all(files, [f.name for f in files], timeout)
+    if failures:
+        _say(f"error: cannot read prompt {failures[0].name}: {failures[0].reason}")
+        return None
+    return list(texts.values())
+
+
+def _width(s: str) -> int:
+    return sum(2 if unicodedata.east_asian_width(c) in "WF" else 1 for c in s)
+
+
+def _pad(s: str, n: int) -> str:
+    return s + " " * max(0, n - _width(s))
+
+
+def print_top(report: Report, n: int = 10) -> None:
+    """Terminal ranking of the most similar pairs."""
+    rows = [p for p in report.pairs if p.rank_key > 0][:n]
+    if not rows:
+        print("no overlapping pairs found")
+        return
+    label = [f"{p.left} ~ {p.right}" for p in rows]
+    wid = max(_width(x) for x in label)
+    print(f"{_pad('pair', wid)}  verbatim L<-R / R<-L   paraphrase  total L<-R / R<-L")
+    for text, p in zip(label, rows):
+        print(f"{_pad(text, wid)}  {p.verbatim_lr * 100:5.1f}% / {p.verbatim_rl * 100:5.1f}%      "
+              f"{p.para_sentences:6d}      {p.score_lr * 100:5.1f}% / {p.score_rl * 100:5.1f}%")
+
+
+def cmd_scan(a: argparse.Namespace) -> int:
+    cfg = _config(a)
+    cfg.validate()
+    if not a.folder.is_dir():
+        _say(f"error: not a folder: {a.folder}. Pass the folder that holds the submissions.")
+        return EXIT_ERROR
+    prompts = _read_prompts(a.prompt, cfg.timeout)
+    if prompts is None:
+        return EXIT_ERROR
+    exclude = {Path(p).resolve() for p in a.prompt}
+    paths = _collect(a.folder, a.recursive, exclude)
+    names = [p.relative_to(a.folder).as_posix() for p in paths]
+    if len(paths) < 2:
+        _say(f"error: need at least 2 files ({', '.join(SUPPORTED)}) in {a.folder}, found {len(paths)}. Try --recursive.")
+        return EXIT_ERROR
+    texts, failures = _extract_all(paths, names, cfg.timeout, cfg.k)
+    if len(texts) < 2:
+        _say("error: fewer than 2 files could be read; see the failure reasons above (run 'extract' on one file).")
+        for f in failures:
+            _say(f"  {f.name}: {f.reason}")
+        return EXIT_ERROR
+    report = compare(texts, prompts, cfg, failures)
+    a.out.mkdir(parents=True, exist_ok=True)
+    write_csv(report, a.out / "pairs.csv")
+    write_json(report, a.out / "pairs.json")
+    index = write_report(report, a.out, top=a.top)
+    print_top(report)
+    for f in failures:
+        print(f"FAILED {f.name}: {f.reason}")
+    s = report.stats
+    print(f"\n{s['documents']} documents, {s['candidate_pairs']} candidate pairs of {s['all_pairs']}, {s['elapsed_seconds']}s")
+    print(f"report: {index}")
+    return EXIT_OK
+
+
+def cmd_pair(a: argparse.Namespace) -> int:
+    cfg = _config(a)
+    cfg.validate()
+    for f in (a.left, a.right):
+        if not f.is_file():
+            _say(f"error: file not found: {f}")
+            return EXIT_ERROR
+    prompts = _read_prompts(a.prompt, cfg.timeout)
+    if prompts is None:
+        return EXIT_ERROR
+    left_name, right_name = a.left.name, a.right.name
+    if left_name == right_name:
+        left_name, right_name = str(a.left), str(a.right)
+    texts, failures = _extract_all([a.left, a.right], [left_name, right_name], cfg.timeout, cfg.k)
+    if failures:
+        for f in failures:
+            _say(f"error: cannot read {f.name}: {f.reason}")
+        return EXIT_ERROR
+    report = compare(texts, prompts, cfg)
+    result = report.pairs[0]
+    a.out.parent.mkdir(parents=True, exist_ok=True)
+    a.out.write_text(pair_page(report, result), encoding="utf-8")
+    print(f"{result.left} ~ {result.right}: verbatim {result.verbatim_lr * 100:.1f}% / {result.verbatim_rl * 100:.1f}%, "
+          f"paraphrase sentences {result.para_sentences}")
+    print(f"page: {a.out}")
+    return EXIT_OK
+
+
+def cmd_extract(a: argparse.Namespace) -> int:
+    if not a.file.is_file():
+        _say(f"error: file not found: {a.file}")
+        return EXIT_ERROR
+    with SandboxedExtractor(timeout=a.timeout, limits=Limits()) as ex:
+        res = ex.extract(a.file)
+    if not res.ok or res.text is None:
+        _say(f"error: {res.error}")
+        return EXIT_ERROR
+    sys.stdout.write(res.text if res.text.endswith("\n") else res.text + "\n")
+    return EXIT_OK
+
+
+def main(argv: list[str] | None = None) -> int:
+    for stream in (sys.stdout, sys.stderr):
+        if hasattr(stream, "reconfigure"):
+            stream.reconfigure(encoding="utf-8")
+    args = build_parser().parse_args(argv)
+    try:
+        if args.command == "scan":
+            return cmd_scan(args)
+        if args.command == "pair":
+            return cmd_pair(args)
+        return cmd_extract(args)
+    except ValueError as exc:
+        _say(f"error: {exc}")
+        return EXIT_ERROR
+    except KeyboardInterrupt:
+        _say("interrupted")
+        return 130
+
+
+if __name__ == "__main__":
+    sys.exit(main())

ko_copycheck/config.py

@@ -0,0 +1,44 @@
+"""Tunable settings. Defaults are documented in the README option table."""
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class Config:
+    """All knobs of a comparison run.
+
+    ``dice`` default comes from ``scripts/calibrate.py`` (hand-written calibration
+    corpus); it is a starting point, not a verdict. Tune it with ``--dice``.
+    """
+
+    k: int = 8  # characters per k-gram
+    w: int = 4  # winnowing window (guarantee threshold = w + k - 1 characters)
+    min_match: int = 20  # shortest verbatim match (normalized characters) that counts
+    dice: float = 0.30  # sentence bigram Dice threshold for the paraphrase layer
+    min_sentence: int = 10  # shortest sentence (normalized characters) for the paraphrase layer
+    group_ratio: float = 0.5  # k-grams shared by this share of documents are "common"...
+    group_min_docs: int = 3  # ...but only when at least this many documents share them
+    covered_skip: float = 0.8  # skip paraphrase pairs whose sentence is already this covered by verbatim
+    max_postings: int = 200  # ignore k-grams occurring more often than this in the index
+    timeout: float = 60.0  # seconds per file for extraction
+
+    def validate(self) -> None:
+        """Raise ``ValueError`` with a fix hint for settings that cannot work."""
+        if self.k < 3:
+            raise ValueError(f"k must be >= 3 (got {self.k}); try --k 8")
+        if self.w < 1:
+            raise ValueError(f"w must be >= 1 (got {self.w}); try --w 4")
+        if self.min_match < self.w + self.k - 1:
+            raise ValueError(
+                f"min_match ({self.min_match}) must be >= w + k - 1 ({self.w + self.k - 1}); "
+                "shorter matches are not guaranteed to be found. Try --min-match 20"
+            )
+        if not 0.0 < self.dice <= 1.0:
+            raise ValueError(f"dice must be in (0, 1] (got {self.dice}); try --dice {type(self).dice}")
+        if not 0.0 < self.group_ratio <= 1.0:
+            raise ValueError(f"group_ratio must be in (0, 1] (got {self.group_ratio})")
+        if self.min_sentence < 2:
+            raise ValueError(f"min_sentence must be >= 2 (got {self.min_sentence})")
+        if self.timeout <= 0:
+            raise ValueError(f"timeout must be > 0 seconds (got {self.timeout})")

ko_copycheck/engine.py

@@ -0,0 +1,131 @@
+"""Public entry point: compare a set of texts against each other."""
+from __future__ import annotations
+
+import time
+from collections import defaultdict
+from itertools import combinations
+from typing import Mapping, Sequence
+
+from .config import Config
+from .index import candidate_hits, document_frequency
+from .normalize import from_display_span, normalize
+from .paraphrase import Sentence, bigram_set, matches_prefix, split_sentences
+from .score import Doc, Failure, PairResult, ParaMatch, Report, finalize
+from .verbatim import Span, covered, merge_spans, pair_spans, prompt_mask
+from .winnow import kgram_hashes, winnow
+
+
+def _overlap_share(span: Span, others: list[Span]) -> float:
+    """Share of ``span`` covered by the (merged, sorted) ``others``."""
+    s, e = span
+    if e <= s:
+        return 0.0
+    hit = sum(max(0, min(e, oe) - max(s, os)) for os, oe in others)
+    return hit / (e - s)
+
+
+def compare(
+    texts: Mapping[str, str],
+    prompts: Sequence[str] = (),
+    config: Config | None = None,
+    failures: Sequence[Failure] = (),
+) -> Report:
+    """Compare every pair of ``texts`` (name -> text) and return a :class:`Report`.
+
+    ``prompts`` are the assignment texts every submission may quote; matching parts are
+    masked and never count. No network is used.
+    """
+    cfg = config or Config()
+    cfg.validate()
+    t0 = time.perf_counter()
+    names = sorted(texts)
+    norms = [normalize(texts[n]) for n in names]
+    prompt_norms = [normalize(p).text for p in prompts]
+    prompt_norms = [p for p in prompt_norms if len(p) >= cfg.k]
+
+    hash_lists = [kgram_hashes(n.text, cfg.k) for n in norms]
+    fps = [winnow(h, cfg.w) for h in hash_lists]
+    prompt_fps = [winnow(kgram_hashes(p, cfg.k), cfg.w) for p in prompt_norms]
+    df = document_frequency(hash_lists)
+    del hash_lists
+
+    docs: list[Doc] = []
+    for name, norm, doc_fps in zip(names, norms, fps):
+        mask = prompt_mask(norm.text, doc_fps, prompt_norms, prompt_fps, cfg.k)
+        sent_spans = [
+            from_display_span(norm.offsets, s, e) for s, e in split_sentences(norm.display, cfg.min_sentence)
+        ]
+        docs.append(Doc(name, norm, mask, sent_spans))
+
+    # Layer 1: verbatim matches only for pairs that share a (non-common) fingerprint.
+    hits = candidate_hits(fps, df, cfg)
+    verbatim: dict[tuple[int, int], tuple[list[Span], list[Span]]] = {}
+    for (i, j), pair_hits in hits.items():
+        sa, sb = pair_spans(norms[i].text, norms[j].text, pair_hits, docs[i].mask, docs[j].mask, cfg)
+        if sa:
+            verbatim[(i, j)] = (sa, sb)
+
+    # Layer 2: sentence Dice over all documents with the rare-bigram prefix filter.
+    sents: list[Sentence] = []
+    for di, doc in enumerate(docs):
+        for span in doc.sentences:
+            if _overlap_share(span, doc.mask) >= 0.5:  # mostly prompt text
+                continue
+            grams = bigram_set(doc.norm.text[span[0] : span[1]])
+            if grams:
+                sents.append(Sentence(di, span, grams))
+    sent_pairs, compared = matches_prefix(sents, cfg.dice)
+    para_by_pair: dict[tuple[int, int], list[tuple[float, Span, Span]]] = defaultdict(list)
+    for a, b in sent_pairs:
+        sa, sb = sents[a], sents[b]
+        if sa.doc > sb.doc:
+            sa, sb = sb, sa
+        d = 2 * len(sa.bigrams & sb.bigrams) / (len(sa.bigrams) + len(sb.bigrams))
+        para_by_pair[(sa.doc, sb.doc)].append((d, sa.span, sb.span))
+
+    results: dict[tuple[int, int], PairResult] = {}
+    for (i, j), cand in para_by_pair.items():
+        vl, vr = verbatim.get((i, j), ([], []))
+        matches = _one_to_one(cand, vl, vr, cfg.covered_skip)
+        results[(i, j)] = finalize(docs[i], docs[j], vl, vr, matches)
+    for (i, j), (vl, vr) in verbatim.items():
+        if (i, j) not in results:
+            results[(i, j)] = finalize(docs[i], docs[j], vl, vr, [])
+    pairs = [results.get(p) or PairResult(docs[p[0]].name, docs[p[1]].name) for p in combinations(range(len(docs)), 2)]
+    pairs.sort(key=lambda p: (-p.rank_key, -(p.score_lr + p.score_rl), p.left, p.right))
+
+    n = len(docs)
+    stats = {
+        "documents": n,
+        "all_pairs": n * (n - 1) // 2,
+        "candidate_pairs": len(results),  # pairs that reached either layer (= rows with a score)
+        "verbatim_pairs": len(verbatim),
+        "sentences": len(sents),
+        "sentence_candidates_compared": compared,
+        "elapsed_seconds": round(time.perf_counter() - t0, 3),
+    }
+    return Report({d.name: d for d in docs}, pairs, list(failures), cfg, stats, list(prompts))
+
+
+def _one_to_one(
+    cand: list[tuple[float, Span, Span]], vl: list[Span], vr: list[Span], skip: float
+) -> list[ParaMatch]:
+    """Greedy one-to-one sentence matching, best Dice first.
+
+    Pairs whose sentence is already mostly explained by a verbatim match are skipped:
+    that text is reported by layer 1.
+    """
+    ml, mr = merge_spans(vl), merge_spans(vr)
+    used_l: set[Span] = set()
+    used_r: set[Span] = set()
+    out: list[ParaMatch] = []
+    for d, sl, sr in sorted(cand, key=lambda c: (-c[0], c[1], c[2])):
+        if sl in used_l or sr in used_r:
+            continue
+        if _overlap_share(sl, ml) >= skip or _overlap_share(sr, mr) >= skip:
+            continue
+        used_l.add(sl)
+        used_r.add(sr)
+        out.append(ParaMatch(sl, sr, round(d, 4)))
+    out.sort(key=lambda m: m.left)
+    return out

ko_copycheck/extract/__init__.py

@@ -0,0 +1,5 @@
+"""Safe text extraction for txt, docx, hwpx and pdf submissions."""
+from .formats import SUPPORTED, ExtractError, Limits, extract_file
+from .sandbox import ExtractResult, SandboxedExtractor
+
+__all__ = ["SUPPORTED", "ExtractError", "Limits", "extract_file", "ExtractResult", "SandboxedExtractor"]

ko_copycheck/extract/formats.py

@@ -0,0 +1,175 @@
+"""Format readers: txt, docx, hwpx, pdf. Run these inside the sandboxed worker."""
+from __future__ import annotations
+
+import re
+import zipfile
+from dataclasses import dataclass
+from pathlib import Path
+from xml.etree import ElementTree as ET
+
+SUPPORTED = (".txt", ".docx", ".hwpx", ".pdf")
+
+
+class ExtractError(Exception):
+    """A file cannot be read. ``code`` is a short machine-readable reason."""
+
+    def __init__(self, code: str, message: str = "") -> None:
+        super().__init__(message or code)
+        self.code = code
+
+
+@dataclass(frozen=True)
+class Limits:
+    """Resource limits applied to every input file."""
+
+    max_file_bytes: int = 50 * 1024 * 1024
+    max_unzipped_bytes: int = 200 * 1024 * 1024
+    max_zip_entries: int = 5000
+
+
+def extract_file(path: str, limits: Limits | None = None) -> str:
+    """Return the plain text of ``path``; raise :class:`ExtractError` on failure."""
+    limits = limits or Limits()
+    p = Path(path)
+    ext = p.suffix.lower()
+    if ext not in SUPPORTED:
+        hint = " (binary .hwp is not supported; save as .hwpx or .pdf)" if ext == ".hwp" else ""
+        raise ExtractError("unsupported", f"unsupported file type {ext or '(none)'}{hint}")
+    size = p.stat().st_size
+    if size > limits.max_file_bytes:
+        raise ExtractError("too_large", f"{size} bytes exceeds the {limits.max_file_bytes} byte limit")
+    if ext == ".txt":
+        return read_txt(p)
+    if ext == ".pdf":
+        return read_pdf(p)
+    check_zip_budget(p, limits)
+    return read_docx(p) if ext == ".docx" else read_hwpx(p)
+
+
+def read_txt(path: Path) -> str:
+    """Decode a text file: UTF-16 with BOM, UTF-8 (with or without BOM), then CP949."""
+    raw = path.read_bytes()
+    if raw.startswith((b"\xff\xfe", b"\xfe\xff")):
+        return raw.decode("utf-16")
+    for enc in ("utf-8-sig", "cp949"):
+        try:
+            return raw.decode(enc)
+        except UnicodeDecodeError:
+            continue
+    return raw.decode("utf-8", errors="replace")
+
+
+def check_zip_budget(path: Path, limits: Limits) -> None:
+    """Refuse zip containers that expand beyond the limits.
+
+    Declared sizes in a zip header can lie, so every member is really decompressed in
+    small chunks and the *actual* byte count is checked.
+    """
+    try:
+        with zipfile.ZipFile(path) as zf:
+            infos = zf.infolist()
+            if len(infos) > limits.max_zip_entries:
+                raise ExtractError("too_large", f"{len(infos)} zip entries exceed {limits.max_zip_entries}")
+            total = 0
+            for info in infos:
+                with zf.open(info) as fh:
+                    while chunk := fh.read(1 << 20):
+                        total += len(chunk)
+                        if total > limits.max_unzipped_bytes:
+                            raise ExtractError(
+                                "too_large", f"unzipped size exceeds {limits.max_unzipped_bytes} bytes"
+                            )
+    except zipfile.BadZipFile as exc:
+        raise ExtractError("corrupt", f"not a valid zip container: {exc}") from exc
+
+
+def read_docx(path: Path) -> str:
+    """Paragraphs and table cells of a .docx, in document order."""
+    from docx import Document  # imported lazily: only the worker needs it
+
+    try:
+        doc = Document(str(path))
+    except Exception as exc:  # python-docx raises several error types
+        raise ExtractError("corrupt", f"cannot open docx: {exc}") from exc
+    lines = [p.text for p in doc.paragraphs]
+    for table in doc.tables:
+        for row in table.rows:
+            lines.extend(cell.text for cell in row.cells)
+    return "\n".join(lines)
+
+
+def _local(tag: str) -> str:
+    return tag.rsplit("}", 1)[-1]
+
+
+def _hwpx_section_order(zf: zipfile.ZipFile) -> list[str]:
+    """Section part names in reading order: content.hpf spine, else numeric order."""
+    members = set(zf.namelist())
+    names = [n for n in members if re.search(r"(^|/)section\d+\.xml$", n)]
+    try:
+        root = ET.fromstring(zf.read("Contents/content.hpf"))
+    except (KeyError, ET.ParseError):
+        root = None
+    if root is not None:
+        manifest = {el.get("id"): el.get("href") for el in root.iter() if _local(el.tag) == "item"}
+        spine = [el.get("idref") for el in root.iter() if _local(el.tag) == "itemref"]
+        ordered = []
+        for idref in spine:
+            href = manifest.get(idref)
+            if not href:
+                continue
+            found = next((c for c in (href, f"Contents/{href}") if c in members), None)
+            if found and re.search(r"section\d+\.xml$", found):
+                ordered.append(found)
+        if ordered:
+            return ordered
+    return sorted(names, key=lambda n: int(re.search(r"section(\d+)\.xml$", n).group(1)))  # type: ignore[union-attr]
+
+
+def _collect_text(el: ET.Element, out: list[str]) -> None:
+    """Depth-first text of an OWPML element: <t> text, line breaks, paragraph ends."""
+    name = _local(el.tag)
+    if name == "t":
+        out.append(el.text or "")
+        for child in el:
+            if _local(child.tag) == "lineBreak":
+                out.append("\n")
+            elif _local(child.tag) == "tab":
+                out.append("\t")
+            out.append(child.tail or "")
+        return
+    for child in el:
+        _collect_text(child, out)
+    if name == "p":
+        out.append("\n")
+
+
+def read_hwpx(path: Path) -> str:
+    """Body text of a .hwpx (OWPML zip), sections in spine order."""
+    try:
+        with zipfile.ZipFile(path) as zf:
+            sections = _hwpx_section_order(zf)
+            if not sections:
+                raise ExtractError("corrupt", "no Contents/section*.xml found; is this a real .hwpx?")
+            out: list[str] = []
+            for name in sections:
+                _collect_text(ET.fromstring(zf.read(name)), out)
+    except zipfile.BadZipFile as exc:
+        raise ExtractError("corrupt", f"not a valid hwpx (zip) file: {exc}") from exc
+    except ET.ParseError as exc:
+        raise ExtractError("corrupt", f"broken XML in hwpx: {exc}") from exc
+    return "".join(out)
+
+
+def read_pdf(path: Path) -> str:
+    """Text layer of a PDF. Scanned PDFs have none; that is reported as an error."""
+    from pypdf import PdfReader  # imported lazily: only the worker needs it
+
+    try:
+        reader = PdfReader(str(path))
+        text = "\n".join((page.extract_text() or "") for page in reader.pages)
+    except Exception as exc:
+        raise ExtractError("corrupt", f"cannot read pdf: {exc}") from exc
+    if len(text.strip()) < 5:
+        raise ExtractError("no_text", "no text layer found (scanned PDF? OCR is not supported)")
+    return text