before-you-send 0.2.0__py3-none-any.whl

before_you_send/__init__.py

@@ -0,0 +1,22 @@
+"""Reads a PDF and reports what is still inside it that you may not mean to send."""
+
+# Read from the installed package rather than repeated here. A hand-written copy
+# drifts the moment a release is cut, and then the tool misreports itself. That
+# happened once already on a sibling project: 0.2.0 went out announcing itself as
+# 0.1.0, and no test caught it because the string agreed with itself everywhere it
+# appeared in the source tree. Only installing the built artefact and asking it
+# found the lie.
+try:
+    from importlib.metadata import PackageNotFoundError
+    from importlib.metadata import version as _installed_version
+
+    try:
+        __version__ = _installed_version("before-you-send")
+    except PackageNotFoundError:  # running from a source tree, not installed
+        __version__ = "unknown (not installed)"
+except ImportError:  # pragma: no cover - Python 3.7 and earlier
+    __version__ = "unknown"
+
+from before_you_send.findings import Blindspot, Finding, Level, Report, Unchecked
+
+__all__ = ["Blindspot", "Finding", "Level", "Report", "Unchecked", "__version__"]

before_you_send/annotations.py

@@ -0,0 +1,60 @@
+"""Reading a page's annotations without letting one bad entry hide the rest.
+
+An /Annots array can contain a reference to an object that is not there, or an entry
+that is not a dictionary at all. Wrapping the whole loop in one try means the first
+broken entry ends the loop, every annotation after it is never looked at, and the
+report says nothing was found. That is the failure mode this module exists to stop:
+the difference between "there were none" and "I could not read them" has to survive
+all the way to the output.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+
+@dataclass
+class AnnotationScan:
+    """The annotations that could be read, and a count of those that could not."""
+
+    items: list = field(default_factory=list)  # (page number, annotation dictionary)
+    unreadable: int = 0
+    pages_affected: set = field(default_factory=set)
+
+    def note_gap(self, report, topic: str) -> None:
+        """Record unreadable entries so they cannot be mistaken for absent ones."""
+        if not self.unreadable:
+            return
+        pages = ", ".join(str(p) for p in sorted(self.pages_affected))
+        report.note_unchecked(
+            topic,
+            f"{self.unreadable} annotation(s) on page(s) {pages} could not be read, so "
+            "anything they contain was not examined",
+        )
+
+
+def scan(pages) -> AnnotationScan:
+    """Every readable annotation across the document, with the failures counted."""
+    found = AnnotationScan()
+    for number, page in enumerate(pages, start=1):
+        try:
+            annots = page.get("/Annots")
+            if annots is None:
+                continue
+            annots = annots.get_object()
+            entries = list(annots)
+        except Exception:
+            found.unreadable += 1
+            found.pages_affected.add(number)
+            continue
+
+        for entry in entries:
+            try:
+                annot = entry.get_object()
+                if not hasattr(annot, "get"):
+                    raise TypeError("annotation is not a dictionary")
+                found.items.append((number, annot))
+            except Exception:
+                found.unreadable += 1
+                found.pages_affected.add(number)
+    return found

before_you_send/checks/__init__.py

@@ -0,0 +1,60 @@
+"""The checks, in the order a reader should meet them.
+
+Two kinds. Page checks look at what was painted on one page and need the drawing
+instructions. Document checks look at the file as a whole and need only the object
+graph. Keeping them apart means a page that cannot be interpreted costs the report
+its page checks and nothing else.
+"""
+
+from __future__ import annotations
+
+from before_you_send.checks.history import earlier_versions_retained
+from before_you_send.checks.metadata import (
+    annotation_authors,
+    build_path_in_metadata,
+    descriptive_metadata,
+    document_author,
+    xmp_metadata,
+)
+from before_you_send.checks.payload import active_content, embedded_files, form_field_values
+from before_you_send.checks.protection import (
+    encryption_without_a_password,
+    hidden_layers,
+    unapplied_redaction_marks,
+)
+from before_you_send.checks.visibility import (
+    covered_text,
+    image_over_text,
+    invisible_text,
+    text_clipped_away,
+    text_matching_background,
+    text_outside_page,
+    text_too_small_to_read,
+)
+
+PAGE_CHECKS = (
+    covered_text,
+    invisible_text,
+    text_matching_background,
+    text_clipped_away,
+    text_too_small_to_read,
+    text_outside_page,
+    image_over_text,
+)
+
+DOCUMENT_CHECKS = (
+    unapplied_redaction_marks,
+    embedded_files,
+    active_content,
+    form_field_values,
+    earlier_versions_retained,
+    hidden_layers,
+    annotation_authors,
+    document_author,
+    xmp_metadata,
+    build_path_in_metadata,
+    descriptive_metadata,
+    encryption_without_a_password,
+)
+
+__all__ = ["PAGE_CHECKS", "DOCUMENT_CHECKS"]

before_you_send/checks/history.py

@@ -0,0 +1,157 @@
+"""What the file remembers about its own past.
+
+A PDF can be edited without rewriting it. The original bytes stay exactly where they
+are and a new section is appended that points past them. Nothing is deleted. A page
+"removed" and saved this way is still in the file, in full, and comes back out with
+ordinary tools.
+
+This has to be read from the cross-reference chain rather than from the bytes.
+Counting how many times ``%%EOF`` appears in a file is not a count of its versions:
+an attached document contains its own, and ``/Prev`` is also a key on bookmarks and
+page trees. The question "does this file's cross-reference table point at an older
+one" has an exact answer, and it is the only one worth reporting.
+"""
+
+from __future__ import annotations
+
+import re
+
+from before_you_send.findings import Finding, Level
+
+# The /Prev entry of a cross-reference section, whether it sits in a classic trailer
+# dictionary or in the dictionary of a cross-reference stream.
+_PREV = re.compile(rb"/Prev\s+(\d+)")
+
+MAX_CHAIN = 64
+
+
+def _revision_count(doc) -> int:
+    """How many cross-reference sections this file chains together.
+
+    Each hop is one earlier version of the document. The walk starts from the offset
+    the file itself points at, so bytes that merely look like a trailer are ignored.
+    """
+    try:
+        last = doc.raw.rsplit(b"startxref", 1)[1]
+        offset = int(last.split()[0])
+    except Exception:
+        return 1
+
+    seen = set()
+    revisions = 1
+    while 0 < offset < len(doc.raw) and offset not in seen and revisions < MAX_CHAIN:
+        seen.add(offset)
+        # A cross-reference section ends at its own %%EOF. Reading past that would
+        # run into the next revision and match its /Prev instead of this one's.
+        window = doc.raw[offset : offset + 4096]
+        end = window.find(b"%%EOF")
+        if end != -1:
+            window = window[:end]
+        match = _PREV.search(window)
+        if not match:
+            break
+        revisions += 1
+        try:
+            offset = int(match.group(1))
+        except Exception:
+            break
+    return revisions
+
+
+def _is_linearized(doc) -> bool:
+    """Whether this file is laid out for fast web viewing.
+
+    A linearized file has a second cross-reference section by construction, so one
+    hop is expected and says nothing about the document's history.
+    """
+    return b"/Linearized" in doc.raw[:2048]
+
+
+def _has_signature(doc) -> bool:
+    """True when the document carries a signature, which requires an update to add."""
+    try:
+        root = doc.reader.trailer["/Root"].get_object()
+        form = root.get("/AcroForm")
+        if form is None:
+            return False
+        fields = form.get_object().get("/Fields")
+        if fields is None:
+            return False
+        for field in fields.get_object():
+            field = field.get_object()
+            if str(field.get("/FT", "")) == "/Sig" and field.get("/V") is not None:
+                return True
+    except Exception:
+        return False
+    return False
+
+
+def earlier_versions_retained(report, doc) -> None:
+    """Earlier revisions of the document still present in the file.
+
+    The count comes from walking the file's own bytes, never from the parsed
+    trailer. A parsed trailer only carries ``/Prev`` when the file uses a classic
+    cross-reference table; a file written with a cross-reference stream — which is
+    what Word, Acrobat, InDesign, Chrome and every linearized government PDF
+    produce — keeps ``/Prev`` inside the stream dictionary, where the parser does
+    not surface it. Gating on the parsed value meant this check returned on its
+    first line for most modern documents and never reached the byte walk written
+    for exactly this question. Measured on 887 real published PDFs, that gate hid
+    every retained revision in 256 of them.
+    """
+    revisions = _revision_count(doc)
+    if revisions < 2:
+        return
+
+    earlier = revisions - 1
+    signed = _has_signature(doc)
+    linearized = _is_linearized(doc)
+
+    # One extra section with an ordinary explanation is not a finding worth alarming
+    # anybody about, but it is still worth saying, because what was in that section
+    # is readable either way.
+    explained = (signed or linearized) and earlier == 1
+    reason = "it carries a signature" if signed else "it is laid out for fast web viewing"
+
+    if explained:
+        report.add(
+            Finding(
+                check="earlier_versions_retained",
+                level=Level.LOW,
+                page="document",
+                location="file structure",
+                summary=(
+                    "The file has one earlier cross-reference section, and "
+                    f"{reason}, which is the ordinary reason for that."
+                ),
+                detail=(
+                    "Signing a PDF, and laying one out for fast web viewing, both append "
+                    "to the file rather than rewriting it. One extra section is expected "
+                    "here. Whatever was in it is still readable, so if the document was "
+                    "edited before that step, the earlier content is in the file."
+                ),
+            )
+        )
+        return
+
+    report.add(
+        Finding(
+            check="earlier_versions_retained",
+            level=Level.MEDIUM if signed or linearized else Level.HIGH,
+            page="document",
+            location="file structure",
+            summary=(
+                f"The file contains {earlier} earlier version(s) of itself, kept in "
+                "full alongside the current one."
+            ),
+            detail=(
+                "PDFs are commonly edited by appending, which leaves every previous "
+                "version intact inside the same file. Text deleted from a page, a page "
+                "removed, or a value changed in an earlier draft can still be read out "
+                "of the older section. Saving the document again as a new file, rather "
+                "than saving over it, is what discards the history."
+                + (f" Note that {reason}, which accounts for one of these." if signed or
+                   linearized else "")
+            ),
+        )
+    )

before_you_send/checks/metadata.py

@@ -0,0 +1,237 @@
+"""Who made this, on what machine, and what they called it.
+
+None of this is hidden. All of it is two clicks away in any viewer, which is exactly
+why nobody looks at it before sending. A document's properties routinely carry the
+author's account name, the internal filename it was saved under, and the folder it
+was built in.
+"""
+
+from __future__ import annotations
+
+import re
+
+from before_you_send.annotations import scan
+from before_you_send.findings import Finding, Level
+
+DESCRIPTIVE_FIELDS = ("/Title", "/Subject", "/Keywords")
+
+# A value that looks like somewhere on somebody's disk rather than a description.
+PATH_LIKE = re.compile(
+    r"""(
+          [A-Za-z]:\\                 # C:\ ...
+        | \\\\[^\\]+\\                # \\server\share
+        | /(?:Users|home)/[^/\s]+     # /Users/name or /home/name
+        | /Volumes/[^/\s]+            # a mounted disk
+    )""",
+    re.VERBOSE,
+)
+
+# A web address with a host in it. Its path is published, not private: a link like
+# "https://www.gov.uk/home/guidance" is not a build path, and calling one "a
+# filesystem path naming a user account" is the kind of wrong that costs the reader's
+# trust in everything else the report says. Addresses are removed before the search.
+#
+# The host has to be non-empty for this to apply, which is exactly what keeps
+# "file:///Users/someone/draft.pdf" — a real leak, and a common one out of a word
+# processor — matching as the path it is.
+WEB_ADDRESS = re.compile(r"\b[a-z][a-z0-9+.\-]*://[^\s/]+\S*", re.IGNORECASE)
+
+
+def _without_web_addresses(value: str) -> str:
+    return WEB_ADDRESS.sub(" ", value)
+
+
+def _info(doc) -> dict:
+    try:
+        meta = doc.reader.metadata
+        if meta is None:
+            return {}
+        return {str(k): str(v) for k, v in meta.items() if v is not None}
+    except Exception:
+        return {}
+
+
+def document_author(report, doc) -> None:
+    """A named author recorded in the document properties."""
+    info = _info(doc)
+    author = (info.get("/Author") or "").strip()
+    if not author:
+        return
+    report.add(
+        Finding(
+            check="document_author",
+            level=Level.MEDIUM,
+            page="document",
+            location="document properties",
+            summary=f"The document names an author ({len(author)} characters).",
+            detail=(
+                "This is usually the account name of whoever created the file, and it "
+                "travels with the document. It is often not the person the recipient "
+                "is expecting, particularly when a document was drafted by one team "
+                "and sent by another."
+            ),
+            sample=author,
+        )
+    )
+
+
+def descriptive_metadata(report, doc) -> None:
+    """Title, subject or keywords left on the document."""
+    info = _info(doc)
+    present = [f for f in DESCRIPTIVE_FIELDS if (info.get(f) or "").strip()]
+    if not present:
+        return
+    labels = ", ".join(f.lstrip("/").lower() for f in present)
+    report.add(
+        Finding(
+            check="descriptive_metadata",
+            level=Level.LOW,
+            page="document",
+            location="document properties",
+            summary=f"The document carries {labels}.",
+            detail=(
+                "Exporting from a word processor usually copies the original filename "
+                "into the title. That filename is frequently more candid than the "
+                "document, because nobody expects it to be read."
+            ),
+            sample=" | ".join(f"{f.lstrip('/')}: {info[f]}" for f in present),
+        )
+    )
+
+
+def build_path_in_metadata(report, doc) -> None:
+    """A filesystem path left behind in the document properties."""
+    info = _info(doc)
+    hits = {
+        key: value
+        for key, value in info.items()
+        if PATH_LIKE.search(_without_web_addresses(value))
+    }
+    if not hits:
+        return
+    report.add(
+        Finding(
+            check="build_path_in_metadata",
+            level=Level.MEDIUM,
+            page="document",
+            location="document properties",
+            summary=(
+                f"{len(hits)} document property value(s) contain a filesystem path."
+            ),
+            detail=(
+                "A path names a user account, a machine, and often a project or client "
+                "folder. It is one of the few things in a document that describes the "
+                "sender's organisation rather than the document's subject."
+            ),
+            sample=" | ".join(f"{k.lstrip('/')}: {v}" for k, v in hits.items()),
+        )
+    )
+
+
+def xmp_metadata(report, doc) -> None:
+    """A second, separate metadata record that can disagree with the first."""
+    try:
+        xmp = doc.reader.xmp_metadata
+    except Exception:
+        xmp = None
+    if xmp is None:
+        return
+
+    info = _info(doc)
+    info_author = (info.get("/Author") or "").strip()
+
+    xmp_authors: list = []
+    try:
+        for value in xmp.dc_creator or []:
+            value = str(value).strip()
+            if value:
+                xmp_authors.append(value)
+    except Exception:
+        pass
+
+    disagrees = bool(xmp_authors) and bool(info_author) and info_author not in xmp_authors
+
+    if disagrees:
+        report.add(
+            Finding(
+                check="xmp_metadata",
+                level=Level.MEDIUM,
+                page="document",
+                location="XMP metadata",
+                summary=(
+                    "The document carries two separate author records and they do not "
+                    "match."
+                ),
+                detail=(
+                    "A PDF stores properties in two places. Editing tools often update "
+                    "one and leave the other, so the second record can name whoever "
+                    "held the document earlier. Clearing the visible author does not "
+                    "clear this one."
+                ),
+                sample=f"properties: {info_author} | XMP: {', '.join(xmp_authors)}",
+            )
+        )
+        return
+
+    if xmp_authors:
+        report.add(
+            Finding(
+                check="xmp_metadata",
+                level=Level.LOW,
+                page="document",
+                location="XMP metadata",
+                summary="A second metadata record also names an author.",
+                detail=(
+                    "Clearing the author in a viewer's properties dialog does not "
+                    "always clear this copy, so it is worth checking separately."
+                ),
+                sample=", ".join(xmp_authors),
+            )
+        )
+
+
+def annotation_authors(report, doc) -> None:
+    """Comments and markup, and the names attached to them."""
+    found = scan(doc.pages)
+    found.note_gap(report, "annotation_authors")
+
+    named: list = []
+    with_text = 0
+    for number, annot in found.items:
+        try:
+            subtype = str(annot.get("/Subtype", ""))
+            if subtype in ("/Link", "/Widget", "/Popup"):
+                continue
+            who = annot.get("/T")
+            what = annot.get("/Contents")
+            if who is not None and str(who).strip():
+                named.append((number, str(who).strip()))
+            if what is not None and str(what).strip():
+                with_text += 1
+        except Exception:
+            continue
+
+    if not named and not with_text:
+        return
+
+    people = sorted({who for _, who in named})
+    report.add(
+        Finding(
+            check="annotation_authors",
+            level=Level.MEDIUM,
+            page="document",
+            location="annotations",
+            summary=(
+                f"{len(named)} annotation(s) name an author"
+                + (f" and {with_text} carry comment text" if with_text else "")
+                + f", across {len({p for p, _ in named}) or doc.page_count} page(s)."
+            ),
+            detail=(
+                "Comments, highlights and sticky notes stay in the file and carry the "
+                "name of whoever made them. Many viewers hide them until the reader "
+                "turns comments on, so a document can look finished and still contain "
+                "the discussion that produced it."
+            ),
+            sample=" | ".join(people),
+        )
+    )