pdf-redpen 0.2.0__py3-none-any.whl

pdf_redpen/__init__.py

@@ -0,0 +1,40 @@
+"""pdf-redpen: ground LLM review quotes to PDF coordinates and mark them up.
+
+Small public API (everything else is internal):
+
+    Finding, ground, annotate_pdf, InputError, __version__
+"""
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Iterable
+
+from .grounding import ground
+from .models import Finding, InputError, Match, Report
+from .writer import DEFAULT_AUTHOR
+
+__version__ = "0.2.0"
+__all__ = ["Finding", "Match", "Report", "InputError", "ground", "annotate_pdf", "__version__"]
+
+
+def annotate_pdf(
+    src: "str | Path",
+    findings: Iterable[Finding],
+    dst: "str | Path",
+    *,
+    max_per_page: int = 20,
+    author: str = DEFAULT_AUTHOR,
+    lang: "str | None" = None,
+) -> Report:
+    """Highlight each finding's quote in `src` and save the result to `dst`.
+
+    No model is called. Quotes that cannot be found become one memo per page. The memo tags
+    ("[not found]", "[over cap]", "(loose match)") follow `lang` ("en" or "ko"); by default the
+    language set with `pdf_redpen.i18n.set_lang` (English unless changed).
+    """
+    from .pipeline import JsonProvider, run_pipeline
+
+    return run_pipeline(
+        Path(src), Path(dst), JsonProvider(findings), mode="apply",
+        max_per_page=max_per_page, author=author, lang=lang,
+    )

pdf_redpen/__main__.py

@@ -0,0 +1,5 @@
+import sys
+
+from .cli import main
+
+sys.exit(main())

pdf_redpen/cli.py

@@ -0,0 +1,222 @@
+"""Command line: review, apply, clear, demo. Exit codes: 0 ok, 2 bad input, 1 unexpected."""
+from __future__ import annotations
+
+import argparse
+import sys
+from importlib import resources
+from pathlib import Path
+from typing import Optional, Sequence
+
+from . import __version__, prompts
+from .i18n import LANGS, CodedError, lang_from_argv, resolve_lang, set_lang, t
+from .llm import Client, MockClient, OpenAIChatClient, RecordingClient
+from .models import PAGE_SKIPPED, InputError, findings_from_json
+from .pipeline import JsonProvider, LLMProvider, run_pipeline
+from .report import mock_notice, page_detail, recall_line, stage_line, summary_line, write_reports
+from .writer import DEFAULT_AUTHOR, clear_annotations
+
+
+def parse_pages(spec: Optional[str]) -> Optional[list[int]]:
+    """'1-3,5' -> [1, 2, 3, 5]."""
+    if not spec:
+        return None
+    pages: list[int] = []
+    try:
+        for part in spec.split(","):
+            if "-" in part:
+                a, b = part.split("-", 1)
+                lo, hi = int(a), int(b)
+                if lo > hi:
+                    raise ValueError
+                pages += range(lo, hi + 1)
+            else:
+                pages.append(int(part))
+    except ValueError:
+        raise InputError("err.pages_format", spec=spec) from None
+    if any(p < 1 for p in pages):
+        raise InputError("err.pages_start")
+    return pages
+
+
+def _progress(result) -> None:
+    """One stderr line per page; failures (and a failed recall turn) show their cause."""
+    if result.detail and result.status != "skipped_no_text_layer":
+        line = t("progress.line_detail", page=result.page, status=result.status,
+                 detail=page_detail(result))
+    else:
+        line = t("progress.line", page=result.page, status=result.status)
+    print(line, file=sys.stderr)
+
+
+def _finish(report, dst: Path, notice_to=None) -> int:
+    """Print the summary; return 1 when every page that was sent to the model failed.
+
+    The mock notice is printed here (only once a run really replayed answers), to `notice_to`.
+    """
+    if "mock" in report.mode:
+        print(mock_notice(), file=notice_to or sys.stderr)
+    json_path, md_path = write_reports(report, dst)
+    print(summary_line(report))
+    print(stage_line(report))
+    if report.recall_pass:
+        print(recall_line(report))
+    for path in (dst, json_path, md_path):
+        print(t("out.wrote", path=path))
+    attempted = [p for p in report.pages if p.status != PAGE_SKIPPED]
+    if report.error_pages:
+        if attempted and report.error_pages == len(attempted):
+            print(t("err.all_failed"), file=sys.stderr)
+            return 1
+        print(t("warn.some_failed", n=report.error_pages), file=sys.stderr)
+    return 0
+
+
+def _default_out(src: Path, suffix: str) -> Path:
+    return src.with_name(f"{src.stem}.{suffix}.pdf")
+
+
+def _check_out(src: Path, dst: Path) -> None:
+    if src.resolve() == dst.resolve():
+        raise InputError("err.overwrite")
+
+
+def cmd_review(a: argparse.Namespace) -> int:
+    src = Path(a.input)
+    dst = Path(a.output) if a.output else _default_out(src, "reviewed")
+    _check_out(src, dst)
+    if a.mock and a.record:
+        raise InputError("err.mock_record")
+    instructions = prompts.DEFAULT_INSTRUCTIONS[a.lang]
+    if a.instructions:
+        ipath = Path(a.instructions)
+        if not ipath.is_file():
+            raise InputError("err.instructions_missing", path=ipath)
+        instructions = ipath.read_text(encoding="utf-8")
+    client: Client
+    if a.mock:
+        client, mode = MockClient(Path(a.mock)), "review (mock: practice replay)"
+    else:
+        client, mode = OpenAIChatClient.from_env(), "review"
+        if a.record:
+            client = RecordingClient(client, Path(a.record))
+    provider = LLMProvider(client, instructions, a.recall_pass, a.lang)
+    report = run_pipeline(src, dst, provider, mode=mode, max_per_page=a.max_highlights_per_page,
+                          pages=parse_pages(a.pages), author=a.author, recall=a.recall_pass,
+                          on_page_done=_progress)
+    return _finish(report, dst)
+
+
+def cmd_apply(a: argparse.Namespace) -> int:
+    src, fpath = Path(a.input), Path(a.findings)
+    dst = Path(a.output)
+    _check_out(src, dst)
+    if not fpath.is_file():
+        raise InputError("err.findings_missing", path=fpath)
+    findings = findings_from_json(fpath.read_text(encoding="utf-8"), str(fpath))
+    report = run_pipeline(src, dst, JsonProvider(findings), mode="apply",
+                          max_per_page=a.max_highlights_per_page, author=DEFAULT_AUTHOR)
+    return _finish(report, dst)
+
+
+def cmd_clear(a: argparse.Namespace) -> int:
+    src, dst = Path(a.input), Path(a.output)
+    _check_out(src, dst)
+    removed = clear_annotations(src, dst, a.author)
+    print(t("out.removed", n=removed, author=a.author, dst=dst))
+    return 0
+
+
+def cmd_demo(a: argparse.Namespace) -> int:
+    out = Path(a.out)
+    data = resources.files("pdf_redpen") / "data"
+    with resources.as_file(data / "sample_report.pdf") as sample, \
+            resources.as_file(data / "recorded") as recorded, \
+            resources.as_file(data / "review.md") as instr:
+        dst = out / "reviewed.pdf"
+        provider = LLMProvider(MockClient(recorded), Path(instr).read_text(encoding="utf-8"),
+                               recall=True)
+        report = run_pipeline(sample, dst, provider, mode="review (mock: practice replay)",
+                              max_per_page=a.max_highlights_per_page, recall=True,
+                              on_page_done=_progress)
+    return _finish(report, dst, notice_to=sys.stdout)
+
+
+def build_parser() -> argparse.ArgumentParser:
+    """Build the parser; help texts come from the current screen language (see i18n)."""
+    p = argparse.ArgumentParser(prog="pdf-redpen", description=t("help.description"))
+    p.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
+    # Screen language (messages, reports). Not the same as `review --lang` (prompt language).
+    p.add_argument("--lang", dest="ui_lang", choices=LANGS, default=None, help=t("help.lang_ui"))
+    sub = p.add_subparsers(dest="command", required=True)
+
+    cap = ["--max-highlights-per-page"]
+    r = sub.add_parser("review", help=t("help.review"))
+    r.add_argument("input")
+    r.add_argument("-o", "--output")
+    r.add_argument("--instructions", help=t("help.instructions"))
+    r.add_argument("--mock", metavar="DIR", help=t("help.mock"))
+    r.add_argument("--record", metavar="DIR", help=t("help.record"))
+    r.add_argument("--recall-pass", action="store_true", help=t("help.recall_pass"))
+    r.add_argument(*cap, type=int, default=20)
+    r.add_argument("--pages", help=t("help.pages"))
+    r.add_argument("--author", default=DEFAULT_AUTHOR)
+    r.add_argument("--lang", choices=("ko", "en"), default="ko", help=t("help.review_lang"))
+    r.set_defaults(func=cmd_review)
+
+    ap = sub.add_parser("apply", help=t("help.apply"))
+    ap.add_argument("input")
+    ap.add_argument("findings")
+    ap.add_argument("-o", "--output", required=True)
+    ap.add_argument(*cap, type=int, default=20)
+    ap.set_defaults(func=cmd_apply)
+
+    cl = sub.add_parser("clear", help=t("help.clear"))
+    cl.add_argument("input")
+    cl.add_argument("-o", "--output", required=True)
+    cl.add_argument("--author", default=DEFAULT_AUTHOR)
+    cl.set_defaults(func=cmd_clear)
+
+    d = sub.add_parser("demo", help=t("help.demo"))
+    d.add_argument("--out", default="out")
+    d.add_argument(*cap, type=int, default=20)
+    d.set_defaults(func=cmd_demo)
+    return p
+
+
+def main(argv: Optional[Sequence[str]] = None) -> int:
+    args_list = list(sys.argv[1:] if argv is None else argv)
+    # Pick the screen language before parsing so --help and parse errors use it too.
+    previous = set_lang(resolve_lang(lang_from_argv(args_list)))
+    try:
+        return _run(args_list)
+    finally:
+        set_lang(previous)  # do not leak the language into the caller (tests, library use)
+
+
+def _run(args_list: list[str]) -> int:
+    args = build_parser().parse_args(args_list)
+    try:
+        return args.func(args)
+    except CodedError as exc:  # InputError, LLMError: message id + params, shown in our language
+        print(t("err.prefix", message=exc.render()), file=sys.stderr)
+        return 2 if isinstance(exc, InputError) else 1
+    except OSError as exc:  # unreadable/unwritable paths, full disk, ...
+        reason = exc.strerror or str(exc)
+        if exc.filename:
+            path = str(exc.filename)
+            if path.endswith(".part"):  # internal temp file: show the path the user gave
+                path = path[: -len(".part")]
+            print(t("err.os", path=path, reason=reason), file=sys.stderr)
+        else:
+            print(t("err.os_nopath", reason=reason), file=sys.stderr)
+        return 2
+    except KeyboardInterrupt:
+        print(t("err.interrupted"), file=sys.stderr)
+        return 1
+    except Exception as exc:  # last resort: no traceback for users
+        print(t("err.unexpected", name=type(exc).__name__, detail=exc), file=sys.stderr)
+        return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

pdf_redpen/data/recorded/page1_turn1.md

@@ -0,0 +1,9 @@
+1쪽에서 문제가 되는 문장을 찾았습니다.
+
+| # | 원문 인용 | 문제점 | 심각도 |
+|---|-----------|--------|--------|
+| 1 | "근거 표가 제시되지 않았습니다" | 매출 증가 주장에 근거 표가 없음 | 높음 |
+| 2 | 신규  고객 수는 1,200명으로 | 다른 절(1,300명)과 숫자가 맞지 않음 | 높음 |
+| 이 줄은 | 열 수가 맞지 않는 깨진 행 |
+| 3 | `The quarterly revenue grew by fifteen-percent!` | Cite the table for the growth figure | 중간 |
+| 4 | intentionally wraps across several lines so that a quoted passage will NOT match exactly | Tail of the quote is paraphrased by the model | 낮음 |

pdf_redpen/data/recorded/page1_turn2.md

@@ -0,0 +1,4 @@
+| Quote (missed only) | Comment | Severity |
+|:--|:--|:--|
+| 근거 표가 제시되지 않았습니다. | 첫 지적과 같은 문장 (중복) | High |
+| 다른절에서는1,300명이라고 | 1,200명과 충돌하는 두 번째 수치 | High |

pdf_redpen/data/recorded/page2_turn1.md

@@ -0,0 +1,6 @@
+| 인용 | 지적 | 심각도 |
+|---|---|---|
+| 결론 문장은 다시 한번 결론 문장으로 반복됩니다 | 같은 문장이 반복됨 (첫 번째) | 낮음 |
+| 결론 문장은 다시 한번 결론 문장으로 반복됩니다 | 같은 문장이 반복됨 (두 번째 출현) | 낮음 |
+| 2025년 매출이 전년 대비 15% 증가했다고 | 모델이 쪽수를 틀림 (실제로는 1쪽 문장) | 중간 |
+| 모든 위험이 완전히 제거되었다고 보고합니다 | 본문에 없는 문장 (모델이 만들어 낸 인용) | 높음 |

pdf_redpen/data/recorded/page2_turn2.md

@@ -0,0 +1 @@
+추가로 놓친 지적은 없습니다.

pdf_redpen/data/review.md

@@ -0,0 +1,5 @@
+이 문서는 내부 검토용 보고서 초안입니다. 다음을 중점적으로 검토하세요.
+
+- 숫자나 통계가 다른 곳과 맞지 않는 부분
+- 근거 없이 단정하는 표현
+- 서로 모순되는 문장

pdf_redpen/grounding.py

@@ -0,0 +1,135 @@
+"""Five-stage quote locator.
+
+Models rarely copy a sentence exactly: they change whitespace, drop Korean word
+spacing, alter punctuation, paraphrase the tail, or name the wrong page. Each
+stage below is looser than the one before; the first stage that finds a span of
+characters that is not already used wins. Every stage searches a *projected*
+string and keeps a map back to the original Char indices, so the highlight is
+always drawn from real glyph boxes.
+"""
+from __future__ import annotations
+
+import re
+from typing import Callable, Optional
+
+from .models import Match
+from .textlayer import PageText, fold
+
+PREFIX_LEN = 25
+ALNUM_MIN = 4
+STAGE_ORDER = ("exact", "ws_collapse", "no_space", "alnum_only", "prefix25")
+
+
+def _find_unused(hay: str, needle: str, mapping: list, used: set[int]) -> Optional[list[int]]:
+    """First occurrence of needle whose chars are all still unused."""
+    start = 0
+    while needle:
+        pos = hay.find(needle, start)
+        if pos < 0:
+            return None
+        span = {mapping[k] for k in range(pos, pos + len(needle)) if mapping[k] is not None}
+        if span and not (span & used):
+            return sorted(span)
+        start = pos + 1
+    return None
+
+
+def _project(flat: str, idx: list, keep: Callable[[str], bool]) -> tuple[str, list]:
+    s, m = [], []
+    for ch, i in zip(flat, idx):
+        if keep(ch):
+            s.append(ch)
+            m.append(i)
+    return "".join(s), m
+
+
+def _collapse(flat: str, idx: list) -> tuple[str, list]:
+    """Replace every whitespace run by a single space."""
+    s, m = [], []
+    for ch, i in zip(flat, idx):
+        if ch.isspace():
+            if s and s[-1] == " ":
+                continue
+            s.append(" ")
+            m.append(i)
+        else:
+            s.append(ch)
+            m.append(i)
+    return "".join(s), m
+
+
+def ground(page: PageText, quote: str, used: set[int]) -> Optional[Match]:
+    """Find `quote` on one page, avoiding char indices in `used`. None if absent."""
+    quote = fold(quote).strip()
+    if not quote:
+        return None
+    flat, idx = page.flat, page.index_map
+
+    # 1. exact substring
+    hit = _find_unused(flat, quote, idx, used)
+    if hit:
+        return Match("exact", hit)
+
+    # 2. whitespace runs collapsed (double spaces, copied line breaks)
+    cflat, cidx = _collapse(flat, idx)
+    cq = re.sub(r"\s+", " ", quote)
+    hit = _find_unused(cflat, cq, cidx, used)
+    if hit:
+        return Match("ws_collapse", hit)
+
+    # 3. first 25 chars, extended to the quote's length (tail was paraphrased)
+    prefix_match: Optional[Match] = None
+    if len(cq) > PREFIX_LEN:
+        pre = cq[:PREFIX_LEN]
+        start = 0
+        while True:
+            pos = cflat.find(pre, start)
+            if pos < 0:
+                break
+            end = min(len(cflat), pos + len(cq))
+            span = {cidx[k] for k in range(pos, end) if cidx[k] is not None}
+            if span and not (span & used):
+                prefix_match = Match("prefix25", sorted(span))
+                break
+            start = pos + 1
+
+    # 4. all whitespace removed (Korean spacing errors)
+    sflat, sidx = _project(flat, idx, lambda ch: not ch.isspace())
+    sq = re.sub(r"\s+", "", quote)
+    hit = _find_unused(sflat, sq, sidx, used)
+    if hit:
+        return Match("no_space", hit)
+
+    # 5. letters and digits only (punctuation changed)
+    aflat, aidx = _project(flat, idx, lambda ch: ch.isalnum())
+    aq = "".join(ch for ch in quote if ch.isalnum())
+    if len(aq) >= ALNUM_MIN:
+        hit = _find_unused(aflat, aq, aidx, used)
+        if hit:
+            return Match("alnum_only", hit)
+
+    # The prefix match is the loosest guess (it may paint text the model never
+    # quoted), so it is only used when the stricter stages 4 and 5 found nothing.
+    return prefix_match
+
+
+def ground_finding(
+    get_page: Callable[[int], PageText],
+    n_pages: int,
+    page_no: int,
+    quote: str,
+    used: dict[int, set[int]],
+) -> Optional[tuple[int, Match]]:
+    """Search the claimed page first, then the previous and the next page.
+
+    Returns (actual 1-based page, Match) or None. `used` holds the already
+    painted char indices per 1-based page and is NOT modified here.
+    """
+    page_no = min(max(page_no, 1), n_pages)
+    for p in (page_no, page_no - 1, page_no + 1):
+        if not (1 <= p <= n_pages):
+            continue
+        m = ground(get_page(p), quote, used.setdefault(p, set()))
+        if m:
+            return p, m
+    return None