bantamkit 0.27.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- bantamkit/__init__.py +32 -0
- bantamkit/agent.py +458 -0
- bantamkit/assets/contracts/default.yaml +90 -0
- bantamkit/assets/evals/devteam/manifest.yaml +351 -0
- bantamkit/assets/evals/devteam/repo/HISTORY.md +18 -0
- bantamkit/assets/evals/devteam/repo/README.md +12 -0
- bantamkit/assets/evals/devteam/repo/docs/architecture.md +17 -0
- bantamkit/assets/evals/devteam/repo/docs/runbook.md +10 -0
- bantamkit/assets/evals/devteam/repo/issues/142-settlement-timeout.md +23 -0
- bantamkit/assets/evals/devteam/repo/patches/0009-retry-budget.patch +38 -0
- bantamkit/assets/evals/devteam/repo/src/ledger/__init__.py +3 -0
- bantamkit/assets/evals/devteam/repo/src/ledger/config.py +35 -0
- bantamkit/assets/evals/devteam/repo/src/ledger/errors.py +13 -0
- bantamkit/assets/evals/devteam/repo/src/ledger/posting.py +12 -0
- bantamkit/assets/evals/devteam/repo/src/ledger/registry.py +7 -0
- bantamkit/assets/evals/devteam/repo/src/ledger/report.py +9 -0
- bantamkit/assets/evals/devteam/repo/src/ledger/retry.py +17 -0
- bantamkit/assets/evals/devteam/repo/src/ledger/settle.py +16 -0
- bantamkit/assets/evals/devteam/repo/src/ledger/validate.py +14 -0
- bantamkit/assets/evals/devteam/repo/tests/test_posting.py +13 -0
- bantamkit/assets/evals/devteam/repo/tests/test_settle.py +9 -0
- bantamkit/assets/evals/devteam/tasks/dt-error-contract.yaml +186 -0
- bantamkit/assets/evals/devteam/tasks/dt-handler-map.yaml +183 -0
- bantamkit/assets/evals/devteam/tasks/dt-patch-before-after.yaml +182 -0
- bantamkit/assets/evals/devteam/tasks/dt-retry-attempts.yaml +181 -0
- bantamkit/assets/evals/devteam/tasks/dt-settlement-config.yaml +185 -0
- bantamkit/assets/evals/devteam/tasks/dt-symbol-home.yaml +181 -0
- bantamkit/assets/evals/devteam/tasks/dt-trace-blame.yaml +182 -0
- bantamkit/assets/evals/devteam/tasks/dt-unread-key.yaml +180 -0
- bantamkit/assets/evals/document/tasks/doc-large-in-137.yaml +38 -0
- bantamkit/assets/evals/document/tasks/doc-large-in-359.yaml +44 -0
- bantamkit/assets/evals/document/tasks/doc-large-in-372.yaml +38 -0
- bantamkit/assets/evals/document/tasks/doc-large-out-11764.yaml +37 -0
- bantamkit/assets/evals/document/tasks/doc-large-out-4137.yaml +37 -0
- bantamkit/assets/evals/document/tasks/doc-large-out-8022.yaml +37 -0
- bantamkit/assets/evals/document/tasks/doc-small-137.yaml +37 -0
- bantamkit/assets/evals/document/tasks/doc-small-261.yaml +37 -0
- bantamkit/assets/evals/document/tasks/doc-small-388.yaml +37 -0
- bantamkit/assets/evals/fixtures/.gitkeep +0 -0
- bantamkit/assets/evals/fixtures/catalog.json +6 -0
- bantamkit/assets/evals/perturbations/task-completion.yaml +576 -0
- bantamkit/assets/evals/tasks/.gitkeep +0 -0
- bantamkit/assets/evals/tasks/extract-contact.yaml +14 -0
- bantamkit/assets/evals/tasks/extract-invoice.yaml +14 -0
- bantamkit/assets/evals/tasks/extract-order.yaml +15 -0
- bantamkit/assets/evals/tasks/extract-schedule.yaml +14 -0
- bantamkit/assets/evals/tasks/extract-versions.yaml +17 -0
- bantamkit/assets/evals/tasks/nav-prod-port.yaml +84 -0
- bantamkit/assets/evals/tasks/nav-release-bundle.yaml +87 -0
- bantamkit/assets/evals/tasks/recall-audit-retention.yaml +17 -0
- bantamkit/assets/evals/tasks/recall-cache-ttl.yaml +13 -0
- bantamkit/assets/evals/tasks/recall-db-port.yaml +17 -0
- bantamkit/assets/evals/tasks/recall-deploy.yaml +13 -0
- bantamkit/assets/evals/tasks/recall-env-endpoint.yaml +18 -0
- bantamkit/assets/evals/tasks/recall-oncall-rotation.yaml +21 -0
- bantamkit/assets/evals/tasks/recall-oncall.yaml +13 -0
- bantamkit/assets/evals/tasks/recall-org-quota.yaml +18 -0
- bantamkit/assets/evals/tasks/recall-owner.yaml +13 -0
- bantamkit/assets/evals/tasks/shop-basket-total.yaml +10 -0
- bantamkit/assets/evals/tasks/shop-cheapest.yaml +9 -0
- bantamkit/assets/evals/tasks/shop-compare.yaml +9 -0
- bantamkit/assets/evals/tasks/shop-gadget-value.yaml +9 -0
- bantamkit/assets/evals/tasks/shop-stock-total.yaml +9 -0
- bantamkit/assets/evals/tasks/shop-total.yaml +9 -0
- bantamkit/assets/profiles/default.yaml +31 -0
- bantamkit/assets/profiles/patient.yaml +31 -0
- bantamkit/assets/rubrics/.gitkeep +0 -0
- bantamkit/assets/rubrics/code-quality.yaml +20 -0
- bantamkit/assets/rubrics/grounded-completion.yaml +37 -0
- bantamkit/assets/rubrics/task-completion.yaml +28 -0
- bantamkit/assets/schemas/shiftwork-checkpoint.json +188 -0
- bantamkit/assets/skills/.gitkeep +0 -0
- bantamkit/assets/skills/file-graph.md +7 -0
- bantamkit/assets/skills/memory.md +35 -0
- bantamkit/assets/tools/.gitkeep +0 -0
- bantamkit/assets/tools/bantamkit_read.json +48 -0
- bantamkit/assets/tools/bantamkit_status.json +25 -0
- bantamkit/assets/tools/build_identity.json +17 -0
- bantamkit/assets/tools/document_list.json +12 -0
- bantamkit/assets/tools/document_read.json +31 -0
- bantamkit/assets/tools/file_graph.json +12 -0
- bantamkit/assets/tools/memory_compact.json +31 -0
- bantamkit/assets/tools/memory_recall.json +38 -0
- bantamkit/assets/tools/memory_save.json +61 -0
- bantamkit/assets/tools/shiftwork_clock_in.json +25 -0
- bantamkit/assets/tools/shiftwork_clock_out.json +60 -0
- bantamkit/assets/tools/shiftwork_status.json +25 -0
- bantamkit/assets/tools/skill_audit.json +70 -0
- bantamkit/assets/tools/validate_json.json +31 -0
- bantamkit/assets.py +67 -0
- bantamkit/budget.py +114 -0
- bantamkit/client.py +329 -0
- bantamkit/contract.py +522 -0
- bantamkit/criticreplay.py +3241 -0
- bantamkit/critique.py +301 -0
- bantamkit/docread.py +1744 -0
- bantamkit/evalrun.py +2003 -0
- bantamkit/eventlog.py +282 -0
- bantamkit/filegraph.py +218 -0
- bantamkit/loopguard.py +101 -0
- bantamkit/mcpreport.py +763 -0
- bantamkit/mcpserver.py +1334 -0
- bantamkit/memory/__init__.py +28 -0
- bantamkit/memory/__main__.py +291 -0
- bantamkit/memory/component.py +569 -0
- bantamkit/memory/divergence.py +744 -0
- bantamkit/memory/layers.py +257 -0
- bantamkit/memory/store.py +940 -0
- bantamkit/pdfread.py +1402 -0
- bantamkit/profile.py +46 -0
- bantamkit/shiftwork.py +212 -0
- bantamkit/skillaudit.py +853 -0
- bantamkit/statusline.py +313 -0
- bantamkit/structured.py +125 -0
- bantamkit/textutil.py +30 -0
- bantamkit-0.27.0.dist-info/METADATA +207 -0
- bantamkit-0.27.0.dist-info/RECORD +119 -0
- bantamkit-0.27.0.dist-info/WHEEL +4 -0
- bantamkit-0.27.0.dist-info/entry_points.txt +2 -0
bantamkit/pdfread.py
ADDED
|
@@ -0,0 +1,1402 @@
|
|
|
1
|
+
"""PDF text extraction with nothing but the standard library — and a reason for every refusal.
|
|
2
|
+
|
|
3
|
+
**Why this file exists at all.** J25-PREP measured this machine, 2026-08-20: `pypdf`, `PyPDF2`,
|
|
4
|
+
`fitz`, `pdfminer`, `pdfplumber`, `pdftotext`, `mutool` and `qpdf` are all **absent**, and
|
|
5
|
+
`/usr/bin/textutil`, which is present, does not do PDF. The runtime dependency list is fixed at
|
|
6
|
+
three packages, so a PDF reader here is written or there is none. `zlib` is standard library,
|
|
7
|
+
which is the whole of the compression problem; everything else a PDF needs — a tokenizer, an
|
|
8
|
+
object graph, a page tree, a content-stream interpreter and a character map — is arithmetic.
|
|
9
|
+
|
|
10
|
+
**The one rule this module is built around: a PDF string's bytes are not characters.** They are
|
|
11
|
+
indices into a font's encoding. For a composite font with `Identity-H` encoding they are glyph
|
|
12
|
+
indices *in that font's own subset*, and without a `/ToUnicode` map nothing in the file says
|
|
13
|
+
what character a glyph is. Decoding those bytes anyway produces text that reads as plausible
|
|
14
|
+
noise and that no caller can distinguish from content — the same failure class as
|
|
15
|
+
`/usr/bin/textutil` echoing Mac Roman at exit 0 (measured, J25-D1). So every character this
|
|
16
|
+
module emits carries a provenance, and **a character it cannot vouch for is never emitted**:
|
|
17
|
+
|
|
18
|
+
- from a `/ToUnicode` CMap in the file — the file's own statement of what the code means;
|
|
19
|
+
- from a simple font under `WinAnsiEncoding`, `MacRomanEncoding` or `StandardEncoding`;
|
|
20
|
+
- from a `/Differences` glyph name that resolves to a codepoint (`uniXXXX`, `AGL`).
|
|
21
|
+
|
|
22
|
+
Anything else is **counted, not rendered**: `PdfPage.unmapped` is how many characters were
|
|
23
|
+
dropped and `PdfPage.unmapped_fonts` names the fonts that produced them, so the caller can
|
|
24
|
+
disclose the gap instead of quoting noise. A page whose every character is unmapped yields no
|
|
25
|
+
rows and says why; `docread` turns that into a refusal when it is true of the whole file.
|
|
26
|
+
|
|
27
|
+
**What a "row" is here, and what it is not.** A PDF has no rows. It has glyphs at coordinates.
|
|
28
|
+
This module groups glyphs into rows by baseline (`_rows_from_runs`) and orders them by x, so
|
|
29
|
+
that `docread.page()` has a slice unit. That is a *slicing* decision, not a claim about
|
|
30
|
+
structure: a two-column page will interleave and a table will not come back as columns.
|
|
31
|
+
|
|
32
|
+
**Structure, without the xref.** The cross-reference table is an index, and on a real corpus it
|
|
33
|
+
is the part most likely to be stale, rewritten by an incremental update, or replaced by an xref
|
|
34
|
+
*stream* that itself needs the object graph to read. So this module does not read it: it scans
|
|
35
|
+
the file for `N G obj` and parses every object it finds, skipping the byte ranges that belong to
|
|
36
|
+
stream data (which is where a false `obj` match would come from), then expands every
|
|
37
|
+
`/Type/ObjStm` object stream — the PDF 1.5+ container the naive J25-PREP pass could not see
|
|
38
|
+
through, 13 of the user's 28 files. Later definitions of an object number win, which is what an
|
|
39
|
+
incremental update means. This costs one linear scan and buys immunity to a broken index.
|
|
40
|
+
|
|
41
|
+
Encryption is refused by name and never guessed at: not one file on the J25 corpus is encrypted
|
|
42
|
+
(`/Encrypt` absent from all 28, measured `6994f96`), so any decryption code here would be
|
|
43
|
+
unmeasured code, which this program does not ship.
|
|
44
|
+
"""
|
|
45
|
+
|
|
46
|
+
from __future__ import annotations
|
|
47
|
+
|
|
48
|
+
import base64
|
|
49
|
+
import re
|
|
50
|
+
import unicodedata
|
|
51
|
+
import zlib
|
|
52
|
+
from dataclasses import dataclass, field
|
|
53
|
+
from pathlib import Path
|
|
54
|
+
|
|
55
|
+
__all__ = ["PdfError", "PdfPage", "PdfText", "read_pdf"]
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class PdfError(Exception):
|
|
59
|
+
"""The PDF could not be read. The message names what was seen, never a bare traceback."""
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
# --------------------------------------------------------------------------- object model
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
class Name(str):
|
|
66
|
+
"""A PDF name (`/Type`). A `str` subclass so comparisons read naturally, distinct from a
|
|
67
|
+
PDF *string*, which is `bytes` — the difference matters when a value may be either."""
|
|
68
|
+
|
|
69
|
+
__slots__ = ()
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
@dataclass(frozen=True)
|
|
73
|
+
class Ref:
|
|
74
|
+
num: int
|
|
75
|
+
gen: int
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
@dataclass
|
|
79
|
+
class Stream:
|
|
80
|
+
d: dict
|
|
81
|
+
raw: bytes
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
class _Keyword(str):
|
|
85
|
+
"""A bare content-stream token: an operator, or `obj`/`endobj` in the object graph."""
|
|
86
|
+
|
|
87
|
+
__slots__ = ()
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
# --------------------------------------------------------------------------------- tokenizer
|
|
91
|
+
|
|
92
|
+
_WS = b"\x00\t\n\x0c\r "
|
|
93
|
+
_DELIMS = b"()<>[]{}/%"
|
|
94
|
+
_ESCAPES = {0x6E: 10, 0x72: 13, 0x74: 9, 0x62: 8, 0x66: 12}
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _skip_ws(data: bytes, i: int) -> int:
|
|
98
|
+
n = len(data)
|
|
99
|
+
while i < n:
|
|
100
|
+
c = data[i]
|
|
101
|
+
if c == 0x25: # '%' comment runs to end of line
|
|
102
|
+
while i < n and data[i] not in (0x0A, 0x0D):
|
|
103
|
+
i += 1
|
|
104
|
+
elif c in _WS:
|
|
105
|
+
i += 1
|
|
106
|
+
else:
|
|
107
|
+
break
|
|
108
|
+
return i
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def _regular_run(data: bytes, i: int) -> tuple[bytes, int]:
|
|
112
|
+
n, j = len(data), i
|
|
113
|
+
while j < n and data[j] not in _WS and data[j] not in _DELIMS:
|
|
114
|
+
j += 1
|
|
115
|
+
return data[i:j], j
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
_HASH_ESCAPE = re.compile(rb"#([0-9A-Fa-f]{2})")
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def _parse_name(data: bytes, i: int) -> tuple[Name, int]:
|
|
122
|
+
token, j = _regular_run(data, i + 1)
|
|
123
|
+
token = _HASH_ESCAPE.sub(lambda m: bytes([int(m.group(1), 16)]), token)
|
|
124
|
+
return Name(token.decode("utf-8", errors="replace")), j
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _parse_literal_string(data: bytes, i: int) -> tuple[bytes, int]:
|
|
128
|
+
i += 1
|
|
129
|
+
depth, n = 1, len(data)
|
|
130
|
+
out = bytearray()
|
|
131
|
+
while i < n:
|
|
132
|
+
c = data[i]
|
|
133
|
+
if c == 0x5C: # backslash
|
|
134
|
+
i += 1
|
|
135
|
+
if i >= n:
|
|
136
|
+
break
|
|
137
|
+
e = data[i]
|
|
138
|
+
if e in _ESCAPES:
|
|
139
|
+
out.append(_ESCAPES[e])
|
|
140
|
+
i += 1
|
|
141
|
+
elif 0x30 <= e <= 0x37:
|
|
142
|
+
digits = bytearray()
|
|
143
|
+
while i < n and len(digits) < 3 and 0x30 <= data[i] <= 0x37:
|
|
144
|
+
digits.append(data[i])
|
|
145
|
+
i += 1
|
|
146
|
+
out.append(int(digits, 8) & 0xFF)
|
|
147
|
+
elif e == 0x0A:
|
|
148
|
+
i += 1
|
|
149
|
+
elif e == 0x0D:
|
|
150
|
+
i += 1
|
|
151
|
+
if i < n and data[i] == 0x0A:
|
|
152
|
+
i += 1
|
|
153
|
+
else:
|
|
154
|
+
out.append(e)
|
|
155
|
+
i += 1
|
|
156
|
+
elif c == 0x28:
|
|
157
|
+
depth += 1
|
|
158
|
+
out.append(c)
|
|
159
|
+
i += 1
|
|
160
|
+
elif c == 0x29:
|
|
161
|
+
depth -= 1
|
|
162
|
+
i += 1
|
|
163
|
+
if depth == 0:
|
|
164
|
+
return bytes(out), i
|
|
165
|
+
out.append(c)
|
|
166
|
+
else:
|
|
167
|
+
out.append(c)
|
|
168
|
+
i += 1
|
|
169
|
+
raise PdfError("a literal string runs off the end of the data (no closing parenthesis)")
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
_HEX_DIGITS = frozenset(b"0123456789abcdefABCDEF")
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def _parse_hex_string(data: bytes, i: int) -> tuple[bytes, int]:
|
|
176
|
+
i += 1
|
|
177
|
+
digits = bytearray()
|
|
178
|
+
n = len(data)
|
|
179
|
+
while i < n and data[i] != 0x3E: # '>'
|
|
180
|
+
if data[i] in _HEX_DIGITS:
|
|
181
|
+
digits.append(data[i])
|
|
182
|
+
i += 1
|
|
183
|
+
if len(digits) % 2:
|
|
184
|
+
digits.append(0x30)
|
|
185
|
+
return bytes.fromhex(digits.decode("ascii")), i + 1
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
_REF_TAIL = re.compile(rb"[\x00\t\r\n\f ]+(\d{1,5})[\x00\t\r\n\f ]+R(?![A-Za-z0-9])")
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def _parse(data: bytes, i: int, depth: int = 0) -> tuple[object, int]:
|
|
192
|
+
"""One PDF object starting at `i`. Returns the value and the offset just past it."""
|
|
193
|
+
if depth > 64:
|
|
194
|
+
raise PdfError("PDF object nesting is deeper than 64 levels")
|
|
195
|
+
i = _skip_ws(data, i)
|
|
196
|
+
if i >= len(data):
|
|
197
|
+
raise PdfError("the object data ends where an object was expected")
|
|
198
|
+
c = data[i]
|
|
199
|
+
if c == 0x2F: # '/'
|
|
200
|
+
return _parse_name(data, i)
|
|
201
|
+
if c == 0x28: # '('
|
|
202
|
+
return _parse_literal_string(data, i)
|
|
203
|
+
if c == 0x3C: # '<'
|
|
204
|
+
if data[i + 1 : i + 2] == b"<":
|
|
205
|
+
return _parse_dict(data, i, depth)
|
|
206
|
+
return _parse_hex_string(data, i)
|
|
207
|
+
if c == 0x5B: # '['
|
|
208
|
+
i += 1
|
|
209
|
+
out: list = []
|
|
210
|
+
while True:
|
|
211
|
+
i = _skip_ws(data, i)
|
|
212
|
+
if i >= len(data):
|
|
213
|
+
raise PdfError("an array runs off the end of the data")
|
|
214
|
+
if data[i] == 0x5D: # ']'
|
|
215
|
+
return out, i + 1
|
|
216
|
+
value, i = _parse(data, i, depth + 1)
|
|
217
|
+
out.append(value)
|
|
218
|
+
if c in (0x5D, 0x3E, 0x29, 0x7D):
|
|
219
|
+
raise PdfError(f"unbalanced {chr(c)!r} in the object data")
|
|
220
|
+
if c == 0x7B: # '{' PostScript calculator body; not data this reader uses
|
|
221
|
+
j = data.find(b"}", i)
|
|
222
|
+
return [], (j + 1 if j != -1 else len(data))
|
|
223
|
+
token, j = _regular_run(data, i)
|
|
224
|
+
if not token:
|
|
225
|
+
raise PdfError(f"byte {data[i]!r} starts no PDF object")
|
|
226
|
+
if token == b"true":
|
|
227
|
+
return True, j
|
|
228
|
+
if token == b"false":
|
|
229
|
+
return False, j
|
|
230
|
+
if token == b"null":
|
|
231
|
+
return None, j
|
|
232
|
+
try:
|
|
233
|
+
value: object = int(token)
|
|
234
|
+
except ValueError:
|
|
235
|
+
try:
|
|
236
|
+
value = float(token.replace(b"--", b"-"))
|
|
237
|
+
except ValueError:
|
|
238
|
+
return _Keyword(token.decode("latin-1")), j
|
|
239
|
+
else:
|
|
240
|
+
if value >= 0:
|
|
241
|
+
m = _REF_TAIL.match(data, j)
|
|
242
|
+
if m:
|
|
243
|
+
return Ref(int(token), int(m.group(1))), m.end()
|
|
244
|
+
return value, j
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
def _parse_dict(data: bytes, i: int, depth: int = 0) -> tuple[dict, int]:
|
|
248
|
+
i += 2
|
|
249
|
+
out: dict = {}
|
|
250
|
+
while True:
|
|
251
|
+
i = _skip_ws(data, i)
|
|
252
|
+
if i >= len(data):
|
|
253
|
+
raise PdfError("a dictionary runs off the end of the data")
|
|
254
|
+
if data[i : i + 2] == b">>":
|
|
255
|
+
return out, i + 2
|
|
256
|
+
if data[i] != 0x2F:
|
|
257
|
+
# A malformed key. Skip one object rather than losing the whole dictionary.
|
|
258
|
+
_, i = _parse(data, i, depth + 1)
|
|
259
|
+
continue
|
|
260
|
+
key, i = _parse_name(data, i)
|
|
261
|
+
value, i = _parse(data, i, depth + 1)
|
|
262
|
+
out[str(key)] = value
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
# ------------------------------------------------------------------------------ stream filters
|
|
266
|
+
|
|
267
|
+
# Filters whose output is an image, not text. Named so a refusal can say which one it met.
|
|
268
|
+
IMAGE_FILTERS = frozenset({"DCTDecode", "DCT", "JPXDecode", "JBIG2Decode", "CCITTFaxDecode", "CCF"})
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
class _ImageData(Exception):
|
|
272
|
+
"""The stream decodes to an image. Not an error — a fact the caller acts on."""
|
|
273
|
+
|
|
274
|
+
def __init__(self, filter_name: str):
|
|
275
|
+
super().__init__(filter_name)
|
|
276
|
+
self.filter_name = filter_name
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
def _flate(data: bytes) -> bytes:
|
|
280
|
+
"""`zlib`, then the tolerances a real corpus needs: truncated, raw, and leading junk."""
|
|
281
|
+
for attempt in (lambda: zlib.decompress(data), lambda: zlib.decompressobj().decompress(data)):
|
|
282
|
+
try:
|
|
283
|
+
return attempt()
|
|
284
|
+
except zlib.error:
|
|
285
|
+
pass
|
|
286
|
+
for skip in (1, 2, 3):
|
|
287
|
+
try:
|
|
288
|
+
return zlib.decompressobj().decompress(data[skip:])
|
|
289
|
+
except zlib.error:
|
|
290
|
+
pass
|
|
291
|
+
try:
|
|
292
|
+
return zlib.decompressobj(-15).decompress(data)
|
|
293
|
+
except zlib.error:
|
|
294
|
+
raise PdfError("a FlateDecode stream is not decompressible by zlib") from None
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
def _ascii85(data: bytes) -> bytes:
|
|
298
|
+
body = b"".join(data.split())
|
|
299
|
+
if body.startswith(b"<~"):
|
|
300
|
+
body = body[2:]
|
|
301
|
+
end = body.find(b"~>")
|
|
302
|
+
if end != -1:
|
|
303
|
+
body = body[:end]
|
|
304
|
+
try:
|
|
305
|
+
return base64.a85decode(body)
|
|
306
|
+
except ValueError:
|
|
307
|
+
raise PdfError("an ASCII85Decode stream is not valid ASCII85") from None
|
|
308
|
+
|
|
309
|
+
|
|
310
|
+
def _asciihex(data: bytes) -> bytes:
|
|
311
|
+
end = data.find(b">")
|
|
312
|
+
body = data[:end] if end != -1 else data
|
|
313
|
+
digits = bytes(ch for ch in body if ch in _HEX_DIGITS)
|
|
314
|
+
if len(digits) % 2:
|
|
315
|
+
digits += b"0"
|
|
316
|
+
return bytes.fromhex(digits.decode("ascii"))
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
def _runlength(data: bytes) -> bytes:
|
|
320
|
+
out = bytearray()
|
|
321
|
+
i, n = 0, len(data)
|
|
322
|
+
while i < n:
|
|
323
|
+
length = data[i]
|
|
324
|
+
i += 1
|
|
325
|
+
if length == 128:
|
|
326
|
+
break
|
|
327
|
+
if length < 128:
|
|
328
|
+
out += data[i : i + length + 1]
|
|
329
|
+
i += length + 1
|
|
330
|
+
else:
|
|
331
|
+
if i >= n:
|
|
332
|
+
break
|
|
333
|
+
out += bytes([data[i]]) * (257 - length)
|
|
334
|
+
i += 1
|
|
335
|
+
return bytes(out)
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
def _lzw(data: bytes, early: int = 1) -> bytes:
|
|
339
|
+
out = bytearray()
|
|
340
|
+
table = [bytes([k]) for k in range(256)] + [b"", b""]
|
|
341
|
+
bits, buf, nbits = 9, 0, 0
|
|
342
|
+
previous: bytes | None = None
|
|
343
|
+
for byte in data:
|
|
344
|
+
buf = (buf << 8) | byte
|
|
345
|
+
nbits += 8
|
|
346
|
+
while nbits >= bits:
|
|
347
|
+
code = (buf >> (nbits - bits)) & ((1 << bits) - 1)
|
|
348
|
+
nbits -= bits
|
|
349
|
+
if code == 256:
|
|
350
|
+
table = [bytes([k]) for k in range(256)] + [b"", b""]
|
|
351
|
+
bits, previous = 9, None
|
|
352
|
+
continue
|
|
353
|
+
if code == 257:
|
|
354
|
+
return bytes(out)
|
|
355
|
+
if previous is None:
|
|
356
|
+
if code >= len(table):
|
|
357
|
+
raise PdfError("an LZWDecode stream starts with an undefined code")
|
|
358
|
+
entry = table[code]
|
|
359
|
+
elif code < len(table):
|
|
360
|
+
entry = table[code]
|
|
361
|
+
table.append(previous + entry[:1])
|
|
362
|
+
else:
|
|
363
|
+
entry = previous + previous[:1]
|
|
364
|
+
table.append(entry)
|
|
365
|
+
out += entry
|
|
366
|
+
previous = entry
|
|
367
|
+
if len(table) + early >= (1 << bits) and bits < 12:
|
|
368
|
+
bits += 1
|
|
369
|
+
return bytes(out)
|
|
370
|
+
|
|
371
|
+
|
|
372
|
+
def _png_predictor(data: bytes, colors: int, bpc: int, columns: int) -> bytes:
|
|
373
|
+
bpp = max(1, (colors * bpc + 7) // 8)
|
|
374
|
+
rowlen = (columns * colors * bpc + 7) // 8
|
|
375
|
+
out = bytearray()
|
|
376
|
+
previous = bytearray(rowlen)
|
|
377
|
+
i, n = 0, len(data)
|
|
378
|
+
while i + 1 <= n - 1:
|
|
379
|
+
tag = data[i]
|
|
380
|
+
row = bytearray(data[i + 1 : i + 1 + rowlen])
|
|
381
|
+
if len(row) < rowlen:
|
|
382
|
+
row += bytes(rowlen - len(row))
|
|
383
|
+
i += 1 + rowlen
|
|
384
|
+
if tag == 1:
|
|
385
|
+
for k in range(bpp, rowlen):
|
|
386
|
+
row[k] = (row[k] + row[k - bpp]) & 0xFF
|
|
387
|
+
elif tag == 2:
|
|
388
|
+
for k in range(rowlen):
|
|
389
|
+
row[k] = (row[k] + previous[k]) & 0xFF
|
|
390
|
+
elif tag == 3:
|
|
391
|
+
for k in range(rowlen):
|
|
392
|
+
left = row[k - bpp] if k >= bpp else 0
|
|
393
|
+
row[k] = (row[k] + ((left + previous[k]) >> 1)) & 0xFF
|
|
394
|
+
elif tag == 4:
|
|
395
|
+
for k in range(rowlen):
|
|
396
|
+
a = row[k - bpp] if k >= bpp else 0
|
|
397
|
+
b = previous[k]
|
|
398
|
+
c = previous[k - bpp] if k >= bpp else 0
|
|
399
|
+
p = a + b - c
|
|
400
|
+
pa, pb, pc = abs(p - a), abs(p - b), abs(p - c)
|
|
401
|
+
best = a if (pa <= pb and pa <= pc) else (b if pb <= pc else c)
|
|
402
|
+
row[k] = (row[k] + best) & 0xFF
|
|
403
|
+
out += row
|
|
404
|
+
previous = row
|
|
405
|
+
return bytes(out)
|
|
406
|
+
|
|
407
|
+
|
|
408
|
+
def _tiff_predictor(data: bytes, colors: int, bpc: int, columns: int) -> bytes:
|
|
409
|
+
if bpc != 8:
|
|
410
|
+
return data
|
|
411
|
+
rowlen = columns * colors
|
|
412
|
+
out = bytearray(data)
|
|
413
|
+
for start in range(0, len(out) - rowlen + 1, rowlen):
|
|
414
|
+
for k in range(colors, rowlen):
|
|
415
|
+
out[start + k] = (out[start + k] + out[start + k - colors]) & 0xFF
|
|
416
|
+
return bytes(out)
|
|
417
|
+
|
|
418
|
+
|
|
419
|
+
# --------------------------------------------------------------------------- the object graph
|
|
420
|
+
|
|
421
|
+
_OBJ_RE = re.compile(rb"(?<![0-9])(\d{1,10})[\x00\t\r\n\f ]+(\d{1,5})[\x00\t\r\n\f ]+obj\b")
|
|
422
|
+
_TRAILER_RE = re.compile(rb"trailer[\x00\t\r\n\f ]*(?=<<)")
|
|
423
|
+
_HEADER_RE = re.compile(rb"%PDF-(\d+\.\d+)")
|
|
424
|
+
|
|
425
|
+
|
|
426
|
+
class PdfDocument:
|
|
427
|
+
"""Every object in the file, found by scanning rather than by trusting the xref index."""
|
|
428
|
+
|
|
429
|
+
def __init__(self, raw: bytes):
|
|
430
|
+
self.raw = raw
|
|
431
|
+
self.objects: dict[int, object] = {}
|
|
432
|
+
self._decoded: dict[int, bytes] = {}
|
|
433
|
+
self._scan()
|
|
434
|
+
self._expand_object_streams()
|
|
435
|
+
|
|
436
|
+
# -- loading ---------------------------------------------------------------------------
|
|
437
|
+
|
|
438
|
+
def _scan(self) -> None:
|
|
439
|
+
raw = self.raw
|
|
440
|
+
guard = 0
|
|
441
|
+
for m in _OBJ_RE.finditer(raw):
|
|
442
|
+
if m.start() < guard:
|
|
443
|
+
continue # inside a stream's data: a false `obj` match, not an object
|
|
444
|
+
num = int(m.group(1))
|
|
445
|
+
try:
|
|
446
|
+
value, j = _parse(raw, m.end())
|
|
447
|
+
except PdfError:
|
|
448
|
+
continue
|
|
449
|
+
head = _skip_ws(raw, j)
|
|
450
|
+
if isinstance(value, dict) and raw[head : head + 6] == b"stream":
|
|
451
|
+
k = head + 6
|
|
452
|
+
if raw[k : k + 2] == b"\r\n":
|
|
453
|
+
k += 2
|
|
454
|
+
elif raw[k : k + 1] in (b"\n", b"\r"):
|
|
455
|
+
k += 1
|
|
456
|
+
end = self._stream_end(value, k)
|
|
457
|
+
value = Stream(value, raw[k:end])
|
|
458
|
+
after = raw.find(b"endstream", end)
|
|
459
|
+
guard = (after + 9) if after != -1 else end
|
|
460
|
+
self.objects[num] = value
|
|
461
|
+
|
|
462
|
+
def _stream_end(self, d: dict, start: int) -> int:
|
|
463
|
+
"""Where the stream data stops. `/Length` when it is honest, `endstream` when it is not.
|
|
464
|
+
|
|
465
|
+
A `/Length` that is an indirect reference cannot be resolved yet — the object it points
|
|
466
|
+
at may not be scanned — so the search is the fallback, and it is also the fallback for a
|
|
467
|
+
`/Length` that simply lies, which real generators do.
|
|
468
|
+
"""
|
|
469
|
+
raw = self.raw
|
|
470
|
+
length = d.get("Length")
|
|
471
|
+
if isinstance(length, int) and 0 <= length <= len(raw) - start:
|
|
472
|
+
candidate = start + length
|
|
473
|
+
if re.match(rb"[\x00\t\r\n\f ]*endstream", raw[candidate : candidate + 20]):
|
|
474
|
+
return candidate
|
|
475
|
+
found = raw.find(b"endstream", start)
|
|
476
|
+
end = found if found != -1 else len(raw)
|
|
477
|
+
while end > start and raw[end - 1] in (0x0A, 0x0D):
|
|
478
|
+
end -= 1
|
|
479
|
+
return end
|
|
480
|
+
|
|
481
|
+
def _expand_object_streams(self) -> None:
|
|
482
|
+
for num, obj in list(self.objects.items()):
|
|
483
|
+
if not (isinstance(obj, Stream) and obj.d.get("Type") == "ObjStm"):
|
|
484
|
+
continue
|
|
485
|
+
try:
|
|
486
|
+
data = self.stream_data(num, obj)
|
|
487
|
+
except (PdfError, _ImageData):
|
|
488
|
+
continue
|
|
489
|
+
count = self.resolve(obj.d.get("N")) or 0
|
|
490
|
+
first = self.resolve(obj.d.get("First")) or 0
|
|
491
|
+
if not isinstance(count, int) or not isinstance(first, int):
|
|
492
|
+
continue
|
|
493
|
+
fields = data[:first].split()
|
|
494
|
+
for k in range(0, min(len(fields) - 1, 2 * count), 2):
|
|
495
|
+
try:
|
|
496
|
+
inner, offset = int(fields[k]), int(fields[k + 1])
|
|
497
|
+
except ValueError:
|
|
498
|
+
continue
|
|
499
|
+
if inner in self.objects:
|
|
500
|
+
continue # a direct definition is the newer one; it wins
|
|
501
|
+
try:
|
|
502
|
+
value, _ = _parse(data, first + offset)
|
|
503
|
+
except PdfError:
|
|
504
|
+
continue
|
|
505
|
+
self.objects[inner] = value
|
|
506
|
+
|
|
507
|
+
# -- access ----------------------------------------------------------------------------
|
|
508
|
+
|
|
509
|
+
def resolve(self, obj: object, depth: int = 0) -> object:
|
|
510
|
+
while isinstance(obj, Ref) and depth < 32:
|
|
511
|
+
obj = self.objects.get(obj.num)
|
|
512
|
+
depth += 1
|
|
513
|
+
return obj
|
|
514
|
+
|
|
515
|
+
def as_dict(self, obj: object) -> dict | None:
|
|
516
|
+
obj = self.resolve(obj)
|
|
517
|
+
if isinstance(obj, Stream):
|
|
518
|
+
return obj.d
|
|
519
|
+
return obj if isinstance(obj, dict) else None
|
|
520
|
+
|
|
521
|
+
def stream_data(self, key: object, stream: Stream) -> bytes:
|
|
522
|
+
"""Decoded bytes of a stream. Raises `_ImageData` when the filter chain is an image."""
|
|
523
|
+
cache_key = key if isinstance(key, int) else None
|
|
524
|
+
if cache_key is not None and cache_key in self._decoded:
|
|
525
|
+
return self._decoded[cache_key]
|
|
526
|
+
data = stream.raw
|
|
527
|
+
filters = self.resolve(stream.d.get("Filter", stream.d.get("F")))
|
|
528
|
+
if filters is None:
|
|
529
|
+
filters = []
|
|
530
|
+
elif not isinstance(filters, list):
|
|
531
|
+
filters = [filters]
|
|
532
|
+
parms = self.resolve(stream.d.get("DecodeParms", stream.d.get("DP")))
|
|
533
|
+
if not isinstance(parms, list):
|
|
534
|
+
parms = [parms] * len(filters)
|
|
535
|
+
for index, raw_filter in enumerate(filters):
|
|
536
|
+
name = str(self.resolve(raw_filter) or "")
|
|
537
|
+
if name in IMAGE_FILTERS:
|
|
538
|
+
raise _ImageData(name)
|
|
539
|
+
parm = self.as_dict(parms[index]) if index < len(parms) else None
|
|
540
|
+
if name in ("FlateDecode", "Fl"):
|
|
541
|
+
data = _flate(data)
|
|
542
|
+
elif name in ("LZWDecode", "LZW"):
|
|
543
|
+
early = self.resolve((parm or {}).get("EarlyChange", 1))
|
|
544
|
+
data = _lzw(data, 1 if early is None else int(early))
|
|
545
|
+
elif name in ("ASCII85Decode", "A85"):
|
|
546
|
+
data = _ascii85(data)
|
|
547
|
+
elif name in ("ASCIIHexDecode", "AHx"):
|
|
548
|
+
data = _asciihex(data)
|
|
549
|
+
elif name in ("RunLengthDecode", "RL"):
|
|
550
|
+
data = _runlength(data)
|
|
551
|
+
elif name in ("Crypt", ""):
|
|
552
|
+
continue
|
|
553
|
+
else:
|
|
554
|
+
raise PdfError(f"stream filter /{name} is one this reader does not implement")
|
|
555
|
+
if parm:
|
|
556
|
+
predictor = self.resolve(parm.get("Predictor", 1)) or 1
|
|
557
|
+
if isinstance(predictor, int) and predictor > 1:
|
|
558
|
+
colors = int(self.resolve(parm.get("Colors", 1)) or 1)
|
|
559
|
+
bpc = int(self.resolve(parm.get("BitsPerComponent", 8)) or 8)
|
|
560
|
+
columns = int(self.resolve(parm.get("Columns", 1)) or 1)
|
|
561
|
+
if predictor == 2:
|
|
562
|
+
data = _tiff_predictor(data, colors, bpc, columns)
|
|
563
|
+
else:
|
|
564
|
+
data = _png_predictor(data, colors, bpc, columns)
|
|
565
|
+
if cache_key is not None:
|
|
566
|
+
self._decoded[cache_key] = data
|
|
567
|
+
return data
|
|
568
|
+
|
|
569
|
+
def data_of(self, obj: object) -> bytes:
|
|
570
|
+
key = obj.num if isinstance(obj, Ref) else None
|
|
571
|
+
resolved = self.resolve(obj)
|
|
572
|
+
if not isinstance(resolved, Stream):
|
|
573
|
+
raise PdfError("expected a stream and found none")
|
|
574
|
+
return self.stream_data(key, resolved)
|
|
575
|
+
|
|
576
|
+
# -- structure -------------------------------------------------------------------------
|
|
577
|
+
|
|
578
|
+
def trailers(self) -> list[dict]:
|
|
579
|
+
out = []
|
|
580
|
+
for m in _TRAILER_RE.finditer(self.raw):
|
|
581
|
+
try:
|
|
582
|
+
d, _ = _parse_dict(self.raw, m.end())
|
|
583
|
+
except PdfError:
|
|
584
|
+
continue
|
|
585
|
+
out.append(d)
|
|
586
|
+
for obj in self.objects.values():
|
|
587
|
+
if isinstance(obj, Stream) and obj.d.get("Type") == "XRef":
|
|
588
|
+
out.append(obj.d)
|
|
589
|
+
return out
|
|
590
|
+
|
|
591
|
+
def encryption(self) -> dict | None:
|
|
592
|
+
for trailer in self.trailers():
|
|
593
|
+
if "Encrypt" in trailer:
|
|
594
|
+
return self.as_dict(trailer["Encrypt"]) or {}
|
|
595
|
+
return None
|
|
596
|
+
|
|
597
|
+
def catalog(self) -> dict | None:
|
|
598
|
+
for trailer in reversed(self.trailers()):
|
|
599
|
+
root = self.as_dict(trailer.get("Root"))
|
|
600
|
+
if root is not None and "Pages" in root:
|
|
601
|
+
return root
|
|
602
|
+
for num in sorted(self.objects, reverse=True):
|
|
603
|
+
d = self.as_dict(self.objects[num])
|
|
604
|
+
if isinstance(d, dict) and d.get("Type") == "Catalog":
|
|
605
|
+
return d
|
|
606
|
+
return None
|
|
607
|
+
|
|
608
|
+
_INHERITED = ("Resources", "MediaBox", "CropBox", "Rotate")
|
|
609
|
+
|
|
610
|
+
def pages(self) -> list[tuple[dict, dict]]:
|
|
611
|
+
"""Every page, in document order, paired with its inherited attributes."""
|
|
612
|
+
out: list[tuple[dict, dict]] = []
|
|
613
|
+
catalog = self.catalog()
|
|
614
|
+
if catalog is not None:
|
|
615
|
+
self._walk(catalog.get("Pages"), {}, out, set(), 0)
|
|
616
|
+
if out:
|
|
617
|
+
return out
|
|
618
|
+
# No usable page tree. Every `/Type/Page` object, in object-number order: not the
|
|
619
|
+
# document's order, and said so by the caller rather than passed off as it.
|
|
620
|
+
for num in sorted(self.objects):
|
|
621
|
+
d = self.as_dict(self.objects[num])
|
|
622
|
+
if isinstance(d, dict) and d.get("Type") == "Page":
|
|
623
|
+
out.append((d, dict(d)))
|
|
624
|
+
return out
|
|
625
|
+
|
|
626
|
+
def _walk(
|
|
627
|
+
self, node: object, inherited: dict, out: list, seen: set, depth: int
|
|
628
|
+
) -> None:
|
|
629
|
+
if depth > 64 or len(out) > 20000:
|
|
630
|
+
return
|
|
631
|
+
if isinstance(node, Ref):
|
|
632
|
+
if (node.num, node.gen) in seen:
|
|
633
|
+
return
|
|
634
|
+
seen.add((node.num, node.gen))
|
|
635
|
+
d = self.as_dict(node)
|
|
636
|
+
if d is None:
|
|
637
|
+
return
|
|
638
|
+
inherited = {**inherited, **{k: d[k] for k in self._INHERITED if k in d}}
|
|
639
|
+
kids = self.resolve(d.get("Kids"))
|
|
640
|
+
if d.get("Type") == "Page" or (not isinstance(kids, list) and "Contents" in d):
|
|
641
|
+
out.append((d, inherited))
|
|
642
|
+
return
|
|
643
|
+
if isinstance(kids, list):
|
|
644
|
+
for kid in kids:
|
|
645
|
+
self._walk(kid, inherited, out, seen, depth + 1)
|
|
646
|
+
|
|
647
|
+
|
|
648
|
+
# ------------------------------------------------------------------- characters, and vouching
|
|
649
|
+
|
|
650
|
+
# The Adobe Glyph List, restricted to what a Latin document actually uses. A glyph name this
|
|
651
|
+
# table does not hold and that is not a `uniXXXX` form is NOT guessed at — it is unmapped.
|
|
652
|
+
_AGL = {
|
|
653
|
+
"space": " ", "exclam": "!", "quotedbl": '"', "numbersign": "#", "dollar": "$",
|
|
654
|
+
"percent": "%", "ampersand": "&", "quotesingle": "'", "quoteright": "’",
|
|
655
|
+
"quoteleft": "‘", "parenleft": "(", "parenright": ")", "asterisk": "*",
|
|
656
|
+
"plus": "+", "comma": ",", "hyphen": "-", "period": ".", "slash": "/", "zero": "0",
|
|
657
|
+
"one": "1", "two": "2", "three": "3", "four": "4", "five": "5", "six": "6",
|
|
658
|
+
"seven": "7", "eight": "8", "nine": "9", "colon": ":", "semicolon": ";", "less": "<",
|
|
659
|
+
"equal": "=", "greater": ">", "question": "?", "at": "@", "bracketleft": "[",
|
|
660
|
+
"backslash": "\\", "bracketright": "]", "asciicircum": "^", "underscore": "_",
|
|
661
|
+
"grave": "`", "braceleft": "{", "bar": "|", "braceright": "}", "asciitilde": "~",
|
|
662
|
+
"quotedblleft": "“", "quotedblright": "”", "endash": "–",
|
|
663
|
+
"emdash": "—", "bullet": "•", "ellipsis": "…", "fi": "fi", "fl": "fl",
|
|
664
|
+
"ff": "ff", "ffi": "ffi", "ffl": "ffl", "dagger": "†", "daggerdbl": "‡",
|
|
665
|
+
"trademark": "™", "copyright": "©", "registered": "®",
|
|
666
|
+
"degree": "°", "plusminus": "±", "middot": "·", "sterling": "£",
|
|
667
|
+
"yen": "¥", "euro": "€", "cent": "¢", "section": "§",
|
|
668
|
+
"paragraph": "¶", "guillemotleft": "«", "guillemotright": "»",
|
|
669
|
+
"quotesinglbase": "‚", "quotedblbase": "„", "perthousand": "‰",
|
|
670
|
+
"nbspace": " ", "minus": "−", "divide": "÷", "multiply": "×",
|
|
671
|
+
}
|
|
672
|
+
for _letter in "abcdefghijklmnopqrstuvwxyz":
|
|
673
|
+
_AGL[_letter] = _letter
|
|
674
|
+
_AGL[_letter.upper()] = _letter.upper()
|
|
675
|
+
|
|
676
|
+
_UNI_NAME = re.compile(r"^uni([0-9A-Fa-f]{4,6})$")
|
|
677
|
+
_U_NAME = re.compile(r"^u([0-9A-Fa-f]{4,6})$")
|
|
678
|
+
|
|
679
|
+
|
|
680
|
+
def glyph_to_unicode(name: str) -> str | None:
|
|
681
|
+
"""A `/Differences` glyph name as a character, or None when nothing in the file says.
|
|
682
|
+
|
|
683
|
+
`g23`, `index7`, `cid100` and a subset font's private names all return None **on purpose**:
|
|
684
|
+
the name is an index into a font, and the file has not said what character it draws.
|
|
685
|
+
"""
|
|
686
|
+
if name in _AGL:
|
|
687
|
+
return _AGL[name]
|
|
688
|
+
m = _UNI_NAME.match(name) or _U_NAME.match(name)
|
|
689
|
+
if m:
|
|
690
|
+
try:
|
|
691
|
+
return chr(int(m.group(1), 16))
|
|
692
|
+
except (ValueError, OverflowError):
|
|
693
|
+
return None
|
|
694
|
+
base = name.split(".")[0]
|
|
695
|
+
if base != name and base in _AGL:
|
|
696
|
+
return _AGL[base]
|
|
697
|
+
return None
|
|
698
|
+
|
|
699
|
+
|
|
700
|
+
def _vouchable(text: str) -> bool:
|
|
701
|
+
"""Is this character one a caller can be handed? Control, private-use and unassigned are
|
|
702
|
+
exactly the shapes glyph-index noise takes, so they are dropped even when a map produced
|
|
703
|
+
them."""
|
|
704
|
+
for ch in text:
|
|
705
|
+
if ch in "\t\n\r ":
|
|
706
|
+
continue
|
|
707
|
+
if ch == "�":
|
|
708
|
+
return False
|
|
709
|
+
if unicodedata.category(ch) in ("Cc", "Cf", "Cn", "Co", "Cs"):
|
|
710
|
+
return False
|
|
711
|
+
return True
|
|
712
|
+
|
|
713
|
+
|
|
714
|
+
_STANDARD_CODECS = {
|
|
715
|
+
"WinAnsiEncoding": "cp1252",
|
|
716
|
+
"MacRomanEncoding": "mac_roman",
|
|
717
|
+
"PDFDocEncoding": "cp1252",
|
|
718
|
+
}
|
|
719
|
+
|
|
720
|
+
|
|
721
|
+
def _base_encoding_map(name: str) -> dict[int, str]:
|
|
722
|
+
"""Codes 0-255 under a named base encoding, ASCII-only when the file names none.
|
|
723
|
+
|
|
724
|
+
A font with no `/Encoding` uses its own built-in one, which the file does not state. ASCII
|
|
725
|
+
is where every Latin text encoding agrees, so ASCII is trusted and the high half is not.
|
|
726
|
+
"""
|
|
727
|
+
out: dict[int, str] = {}
|
|
728
|
+
codec = _STANDARD_CODECS.get(name)
|
|
729
|
+
if codec:
|
|
730
|
+
for code in range(32, 256):
|
|
731
|
+
try:
|
|
732
|
+
ch = bytes([code]).decode(codec)
|
|
733
|
+
except UnicodeDecodeError:
|
|
734
|
+
continue
|
|
735
|
+
if _vouchable(ch):
|
|
736
|
+
out[code] = ch
|
|
737
|
+
return out
|
|
738
|
+
for code in range(32, 127):
|
|
739
|
+
out[code] = chr(code)
|
|
740
|
+
return out
|
|
741
|
+
|
|
742
|
+
|
|
743
|
+
def parse_cmap(data: bytes) -> tuple[dict[int, str], set[int]]:
|
|
744
|
+
"""A `/ToUnicode` CMap as code -> text, plus the code byte-lengths it declares."""
|
|
745
|
+
mapping: dict[int, str] = {}
|
|
746
|
+
lengths: set[int] = set()
|
|
747
|
+
|
|
748
|
+
def utf16(hexbytes: bytes) -> str:
|
|
749
|
+
raw = hexbytes.decode("ascii")
|
|
750
|
+
if len(raw) % 2:
|
|
751
|
+
raw += "0"
|
|
752
|
+
try:
|
|
753
|
+
return bytes.fromhex(raw).decode("utf-16-be", errors="replace")
|
|
754
|
+
except ValueError:
|
|
755
|
+
return ""
|
|
756
|
+
|
|
757
|
+
for block in re.findall(rb"begincodespacerange(.*?)endcodespacerange", data, re.S):
|
|
758
|
+
for src, _dst in re.findall(rb"<([0-9A-Fa-f]*)>\s*<([0-9A-Fa-f]*)>", block):
|
|
759
|
+
if src:
|
|
760
|
+
lengths.add(max(1, len(src) // 2))
|
|
761
|
+
for block in re.findall(rb"beginbfchar(.*?)endbfchar", data, re.S):
|
|
762
|
+
for src, dst in re.findall(rb"<([0-9A-Fa-f]+)>\s*<([0-9A-Fa-f]*)>", block):
|
|
763
|
+
lengths.add(max(1, len(src) // 2))
|
|
764
|
+
mapping[int(src, 16)] = utf16(dst)
|
|
765
|
+
for block in re.findall(rb"beginbfrange(.*?)endbfrange", data, re.S):
|
|
766
|
+
pattern = rb"<([0-9A-Fa-f]+)>\s*<([0-9A-Fa-f]+)>\s*(?:<([0-9A-Fa-f]*)>|\[(.*?)\])"
|
|
767
|
+
for m in re.finditer(pattern, block, re.S):
|
|
768
|
+
lo, hi = int(m.group(1), 16), int(m.group(2), 16)
|
|
769
|
+
lengths.add(max(1, len(m.group(1)) // 2))
|
|
770
|
+
if hi < lo or hi - lo > 65535:
|
|
771
|
+
continue
|
|
772
|
+
if m.group(3) is not None:
|
|
773
|
+
base = m.group(3)
|
|
774
|
+
prefix, tail = base[:-4], base[-4:] if len(base) >= 4 else base
|
|
775
|
+
try:
|
|
776
|
+
start = int(tail, 16) if tail else 0
|
|
777
|
+
except ValueError:
|
|
778
|
+
continue
|
|
779
|
+
for k in range(hi - lo + 1):
|
|
780
|
+
mapping[lo + k] = utf16(prefix + f"{start + k:04X}".encode())
|
|
781
|
+
else:
|
|
782
|
+
for k, item in enumerate(re.findall(rb"<([0-9A-Fa-f]*)>", m.group(4) or b"")):
|
|
783
|
+
if lo + k <= hi:
|
|
784
|
+
mapping[lo + k] = utf16(item)
|
|
785
|
+
return mapping, lengths
|
|
786
|
+
|
|
787
|
+
|
|
788
|
+
@dataclass
|
|
789
|
+
class _Font:
|
|
790
|
+
"""One font resource, and everything needed to turn its codes into vouched characters."""
|
|
791
|
+
|
|
792
|
+
name: str
|
|
793
|
+
basefont: str
|
|
794
|
+
two_byte: bool
|
|
795
|
+
to_unicode: dict[int, str]
|
|
796
|
+
encoding: dict[int, str]
|
|
797
|
+
widths: dict[int, float]
|
|
798
|
+
default_width: float
|
|
799
|
+
|
|
800
|
+
def codes(self, raw: bytes) -> list[int]:
|
|
801
|
+
if self.two_byte:
|
|
802
|
+
if len(raw) % 2:
|
|
803
|
+
raw += b"\x00"
|
|
804
|
+
return [(raw[i] << 8) | raw[i + 1] for i in range(0, len(raw), 2)]
|
|
805
|
+
return list(raw)
|
|
806
|
+
|
|
807
|
+
def char(self, code: int, permissive: bool = False) -> str | None:
|
|
808
|
+
"""The character this code means, or None when nothing in the file says.
|
|
809
|
+
|
|
810
|
+
`permissive=True` is the audit path and only the audit path: it decodes an unmapped
|
|
811
|
+
code as `chr(code)`, which is precisely the assumption that turns a subset font's
|
|
812
|
+
glyph indices into mojibake. `read_pdf(vouch=False)` is its only caller.
|
|
813
|
+
"""
|
|
814
|
+
text = self.to_unicode.get(code)
|
|
815
|
+
if text is None:
|
|
816
|
+
text = self.encoding.get(code)
|
|
817
|
+
if text is None or not _vouchable(text):
|
|
818
|
+
if not permissive:
|
|
819
|
+
return None
|
|
820
|
+
try:
|
|
821
|
+
return text if text is not None else chr(code)
|
|
822
|
+
except ValueError:
|
|
823
|
+
return "\ufffd"
|
|
824
|
+
return text
|
|
825
|
+
|
|
826
|
+
def width(self, code: int) -> float:
|
|
827
|
+
return self.widths.get(code, self.default_width)
|
|
828
|
+
|
|
829
|
+
|
|
830
|
+
def _build_font(doc: PdfDocument, name: str, node: object) -> _Font:
|
|
831
|
+
d = doc.as_dict(node) or {}
|
|
832
|
+
subtype = str(doc.resolve(d.get("Subtype")) or "")
|
|
833
|
+
basefont = str(doc.resolve(d.get("BaseFont")) or "?")
|
|
834
|
+
to_unicode: dict[int, str] = {}
|
|
835
|
+
lengths: set[int] = set()
|
|
836
|
+
tu = doc.resolve(d.get("ToUnicode"))
|
|
837
|
+
if isinstance(tu, Stream):
|
|
838
|
+
try:
|
|
839
|
+
to_unicode, lengths = parse_cmap(doc.data_of(d.get("ToUnicode")))
|
|
840
|
+
except (PdfError, _ImageData):
|
|
841
|
+
to_unicode, lengths = {}, set()
|
|
842
|
+
if subtype == "Type0":
|
|
843
|
+
encoding_name = str(doc.resolve(d.get("Encoding")) or "")
|
|
844
|
+
# Composite-font codes are two bytes unless the ToUnicode CMap declares otherwise:
|
|
845
|
+
# Identity-H, and every predefined CJK CMap this corpus carries, are two-byte.
|
|
846
|
+
two_byte = lengths != {1}
|
|
847
|
+
del encoding_name
|
|
848
|
+
widths, default_width = _cid_widths(doc, d)
|
|
849
|
+
return _Font(
|
|
850
|
+
name=name,
|
|
851
|
+
basefont=basefont,
|
|
852
|
+
two_byte=two_byte,
|
|
853
|
+
to_unicode=to_unicode,
|
|
854
|
+
encoding={},
|
|
855
|
+
widths=widths,
|
|
856
|
+
# A composite font says nothing about characters without a ToUnicode map: its
|
|
857
|
+
# codes are CIDs, and for Identity-H they are glyph indices in a subset font.
|
|
858
|
+
# `encoding` is therefore left EMPTY, which is what makes `char()` return None
|
|
859
|
+
# for every code — the trust decision is that emptiness and nothing else.
|
|
860
|
+
default_width=default_width,
|
|
861
|
+
)
|
|
862
|
+
encoding_obj = doc.resolve(d.get("Encoding"))
|
|
863
|
+
base_name, differences = "", []
|
|
864
|
+
if isinstance(encoding_obj, Name):
|
|
865
|
+
base_name = str(encoding_obj)
|
|
866
|
+
elif isinstance(encoding_obj, dict):
|
|
867
|
+
base_name = str(doc.resolve(encoding_obj.get("BaseEncoding")) or "")
|
|
868
|
+
differences = doc.resolve(encoding_obj.get("Differences")) or []
|
|
869
|
+
encoding = _base_encoding_map(base_name)
|
|
870
|
+
code = 0
|
|
871
|
+
for item in differences if isinstance(differences, list) else []:
|
|
872
|
+
item = doc.resolve(item)
|
|
873
|
+
if isinstance(item, (int, float)):
|
|
874
|
+
code = int(item)
|
|
875
|
+
elif isinstance(item, Name):
|
|
876
|
+
ch = glyph_to_unicode(str(item))
|
|
877
|
+
if ch is None:
|
|
878
|
+
encoding.pop(code, None) # the file named a glyph, not a character
|
|
879
|
+
else:
|
|
880
|
+
encoding[code] = ch
|
|
881
|
+
code += 1
|
|
882
|
+
widths, default_width = _simple_widths(doc, d)
|
|
883
|
+
return _Font(
|
|
884
|
+
name=name,
|
|
885
|
+
basefont=basefont,
|
|
886
|
+
two_byte=False,
|
|
887
|
+
to_unicode=to_unicode,
|
|
888
|
+
encoding=encoding,
|
|
889
|
+
widths=widths,
|
|
890
|
+
default_width=default_width,
|
|
891
|
+
)
|
|
892
|
+
|
|
893
|
+
|
|
894
|
+
def _simple_widths(doc: PdfDocument, d: dict) -> tuple[dict[int, float], float]:
|
|
895
|
+
widths: dict[int, float] = {}
|
|
896
|
+
first = doc.resolve(d.get("FirstChar"))
|
|
897
|
+
table = doc.resolve(d.get("Widths"))
|
|
898
|
+
if isinstance(first, int) and isinstance(table, list):
|
|
899
|
+
for k, w in enumerate(table):
|
|
900
|
+
w = doc.resolve(w)
|
|
901
|
+
if isinstance(w, (int, float)):
|
|
902
|
+
widths[first + k] = float(w)
|
|
903
|
+
descriptor = doc.as_dict(d.get("FontDescriptor")) or {}
|
|
904
|
+
missing = doc.resolve(descriptor.get("MissingWidth", 500))
|
|
905
|
+
return widths, float(missing) if isinstance(missing, (int, float)) else 500.0
|
|
906
|
+
|
|
907
|
+
|
|
908
|
+
def _cid_widths(doc: PdfDocument, d: dict) -> tuple[dict[int, float], float]:
|
|
909
|
+
descendants = doc.resolve(d.get("DescendantFonts")) or []
|
|
910
|
+
child = doc.as_dict(descendants[0]) if isinstance(descendants, list) and descendants else {}
|
|
911
|
+
child = child or {}
|
|
912
|
+
default = doc.resolve(child.get("DW", 1000))
|
|
913
|
+
widths: dict[int, float] = {}
|
|
914
|
+
table = doc.resolve(child.get("W")) or []
|
|
915
|
+
index = 0
|
|
916
|
+
while isinstance(table, list) and index < len(table):
|
|
917
|
+
start = doc.resolve(table[index])
|
|
918
|
+
following = doc.resolve(table[index + 1]) if index + 1 < len(table) else None
|
|
919
|
+
if isinstance(following, list):
|
|
920
|
+
for k, w in enumerate(following):
|
|
921
|
+
w = doc.resolve(w)
|
|
922
|
+
if isinstance(start, int) and isinstance(w, (int, float)):
|
|
923
|
+
widths[start + k] = float(w)
|
|
924
|
+
index += 2
|
|
925
|
+
elif isinstance(following, int) and index + 2 < len(table):
|
|
926
|
+
w = doc.resolve(table[index + 2])
|
|
927
|
+
if isinstance(start, int) and isinstance(w, (int, float)) and following - start < 65536:
|
|
928
|
+
for cid in range(start, following + 1):
|
|
929
|
+
widths[cid] = float(w)
|
|
930
|
+
index += 3
|
|
931
|
+
else:
|
|
932
|
+
break
|
|
933
|
+
return widths, float(default) if isinstance(default, (int, float)) else 1000.0
|
|
934
|
+
|
|
935
|
+
|
|
936
|
+
# ------------------------------------------------------------- the content-stream interpreter
|
|
937
|
+
|
|
938
|
+
|
|
939
|
+
def _mul(m: tuple, n: tuple) -> tuple:
|
|
940
|
+
a1, b1, c1, d1, e1, f1 = m
|
|
941
|
+
a2, b2, c2, d2, e2, f2 = n
|
|
942
|
+
return (
|
|
943
|
+
a1 * a2 + b1 * c2,
|
|
944
|
+
a1 * b2 + b1 * d2,
|
|
945
|
+
c1 * a2 + d1 * c2,
|
|
946
|
+
c1 * b2 + d1 * d2,
|
|
947
|
+
e1 * a2 + f1 * c2 + e2,
|
|
948
|
+
e1 * b2 + f1 * d2 + f2,
|
|
949
|
+
)
|
|
950
|
+
|
|
951
|
+
|
|
952
|
+
_IDENTITY = (1.0, 0.0, 0.0, 1.0, 0.0, 0.0)
|
|
953
|
+
|
|
954
|
+
|
|
955
|
+
@dataclass
|
|
956
|
+
class _Run:
|
|
957
|
+
y: float
|
|
958
|
+
x: float
|
|
959
|
+
end_x: float
|
|
960
|
+
size: float
|
|
961
|
+
text: str
|
|
962
|
+
|
|
963
|
+
|
|
964
|
+
@dataclass
|
|
965
|
+
class _PageState:
|
|
966
|
+
runs: list[_Run] = field(default_factory=list)
|
|
967
|
+
show_ops: int = 0
|
|
968
|
+
vouched: int = 0
|
|
969
|
+
unmapped: int = 0
|
|
970
|
+
unmapped_fonts: set = field(default_factory=set)
|
|
971
|
+
images: int = 0
|
|
972
|
+
image_bytes: int = 0
|
|
973
|
+
image_kinds: set = field(default_factory=set)
|
|
974
|
+
permissive: bool = False
|
|
975
|
+
|
|
976
|
+
|
|
977
|
+
_INLINE_IMAGE = re.compile(rb"\bID[\x00\t\r\n\f ]")
|
|
978
|
+
|
|
979
|
+
|
|
980
|
+
def _tokenize_content(data: bytes) -> list:
|
|
981
|
+
"""Content-stream tokens. Inline images (`BI … ID … EI`) are skipped as the binary they are."""
|
|
982
|
+
out: list = []
|
|
983
|
+
i, n = 0, len(data)
|
|
984
|
+
while True:
|
|
985
|
+
i = _skip_ws(data, i)
|
|
986
|
+
if i >= n:
|
|
987
|
+
return out
|
|
988
|
+
try:
|
|
989
|
+
value, j = _parse(data, i)
|
|
990
|
+
except PdfError:
|
|
991
|
+
# A byte that starts no object. Step over it — and STEPPING is the point: a
|
|
992
|
+
# `continue` that left `i` where it was hung this reader forever on a real file
|
|
993
|
+
# (`Defect_Apr 2026 RELEASE_SIT_AP1827-38933.pdf`, measured 2026-08-20).
|
|
994
|
+
i += 1
|
|
995
|
+
continue
|
|
996
|
+
if j <= i:
|
|
997
|
+
j = i + 1
|
|
998
|
+
if isinstance(value, _Keyword) and value == "BI":
|
|
999
|
+
m = _INLINE_IMAGE.search(data, j)
|
|
1000
|
+
end = data.find(b"EI", m.end() if m else j)
|
|
1001
|
+
out.append(_Keyword("INLINE_IMAGE"))
|
|
1002
|
+
i = (end + 2) if end != -1 else n
|
|
1003
|
+
continue
|
|
1004
|
+
out.append(value)
|
|
1005
|
+
i = j
|
|
1006
|
+
|
|
1007
|
+
|
|
1008
|
+
def _run_content(
|
|
1009
|
+
doc: PdfDocument,
|
|
1010
|
+
data: bytes,
|
|
1011
|
+
resources: dict,
|
|
1012
|
+
state: _PageState,
|
|
1013
|
+
ctm: tuple,
|
|
1014
|
+
depth: int,
|
|
1015
|
+
seen: set,
|
|
1016
|
+
) -> None:
|
|
1017
|
+
fonts: dict[str, _Font] = {}
|
|
1018
|
+
font_dict = doc.as_dict(resources.get("Font")) or {}
|
|
1019
|
+
xobjects = doc.as_dict(resources.get("XObject")) or {}
|
|
1020
|
+
|
|
1021
|
+
def font_for(key: str) -> _Font:
|
|
1022
|
+
if key not in fonts:
|
|
1023
|
+
if key in font_dict:
|
|
1024
|
+
fonts[key] = _build_font(doc, key, font_dict[key])
|
|
1025
|
+
else:
|
|
1026
|
+
# A font the page never declared. Its codes mean nothing this file states.
|
|
1027
|
+
fonts[key] = _Font(key, "?", False, {}, {}, {}, 500.0)
|
|
1028
|
+
return fonts[key]
|
|
1029
|
+
|
|
1030
|
+
stack: list[tuple] = []
|
|
1031
|
+
tm = tlm = _IDENTITY
|
|
1032
|
+
font: _Font | None = None
|
|
1033
|
+
fsize = 0.0
|
|
1034
|
+
tc = tw = 0.0
|
|
1035
|
+
th = 1.0
|
|
1036
|
+
tl = 0.0
|
|
1037
|
+
rise = 0.0
|
|
1038
|
+
operands: list = []
|
|
1039
|
+
|
|
1040
|
+
def show(raw: bytes) -> None:
|
|
1041
|
+
nonlocal tm
|
|
1042
|
+
if font is None:
|
|
1043
|
+
return
|
|
1044
|
+
state.show_ops += 1
|
|
1045
|
+
placement = _mul(tm, ctm)
|
|
1046
|
+
trm = _mul((fsize * th, 0.0, 0.0, fsize, 0.0, rise), placement)
|
|
1047
|
+
scale = abs(trm[3]) or abs(fsize) or 1.0
|
|
1048
|
+
# The displacement below is computed in TEXT space; the run's extent on the page is
|
|
1049
|
+
# that displacement through the text and current transformation matrices. Using the
|
|
1050
|
+
# CTM alone under-measures every run a `Tm` scales, which shows up as spurious spaces
|
|
1051
|
+
# inside words ("Cus t omer") when the gap rule below compares extents.
|
|
1052
|
+
stretch = abs(placement[0]) or 1.0
|
|
1053
|
+
pieces: list[str] = []
|
|
1054
|
+
advance = 0.0
|
|
1055
|
+
for code in font.codes(raw):
|
|
1056
|
+
ch = font.char(code, state.permissive)
|
|
1057
|
+
if ch is None:
|
|
1058
|
+
state.unmapped += 1
|
|
1059
|
+
state.unmapped_fonts.add(font.basefont or font.name)
|
|
1060
|
+
else:
|
|
1061
|
+
state.vouched += 1
|
|
1062
|
+
pieces.append(ch)
|
|
1063
|
+
step = font.width(code) / 1000.0 * fsize + tc
|
|
1064
|
+
if code == 32 and not font.two_byte:
|
|
1065
|
+
step += tw
|
|
1066
|
+
advance += step * th
|
|
1067
|
+
text = "".join(pieces)
|
|
1068
|
+
if text.strip():
|
|
1069
|
+
x0, y0 = trm[4], trm[5]
|
|
1070
|
+
state.runs.append(
|
|
1071
|
+
_Run(y=y0, x=x0, end_x=x0 + advance * stretch, size=scale, text=text)
|
|
1072
|
+
)
|
|
1073
|
+
tm = _mul((1.0, 0.0, 0.0, 1.0, advance, 0.0), tm)
|
|
1074
|
+
|
|
1075
|
+
def adjust(amount: float) -> None:
|
|
1076
|
+
nonlocal tm
|
|
1077
|
+
tm = _mul((1.0, 0.0, 0.0, 1.0, -amount / 1000.0 * fsize * th, 0.0), tm)
|
|
1078
|
+
|
|
1079
|
+
def numbers(count: int) -> list[float]:
|
|
1080
|
+
got = [v for v in operands[-count:] if isinstance(v, (int, float))]
|
|
1081
|
+
return [float(v) for v in got] if len(got) == count else []
|
|
1082
|
+
|
|
1083
|
+
for token in _tokenize_content(data):
|
|
1084
|
+
if not isinstance(token, _Keyword):
|
|
1085
|
+
operands.append(token)
|
|
1086
|
+
if len(operands) > 64:
|
|
1087
|
+
del operands[:-32]
|
|
1088
|
+
continue
|
|
1089
|
+
op = str(token)
|
|
1090
|
+
if op == "q":
|
|
1091
|
+
stack.append(ctm)
|
|
1092
|
+
elif op == "Q":
|
|
1093
|
+
if stack:
|
|
1094
|
+
ctm = stack.pop()
|
|
1095
|
+
elif op == "cm":
|
|
1096
|
+
got = numbers(6)
|
|
1097
|
+
if got:
|
|
1098
|
+
ctm = _mul(tuple(got), ctm)
|
|
1099
|
+
elif op == "BT":
|
|
1100
|
+
tm = tlm = _IDENTITY
|
|
1101
|
+
elif op == "ET":
|
|
1102
|
+
tm = tlm = _IDENTITY
|
|
1103
|
+
elif op == "Tf":
|
|
1104
|
+
got = [v for v in operands if isinstance(v, Name)]
|
|
1105
|
+
size = numbers(1)
|
|
1106
|
+
if got:
|
|
1107
|
+
font = font_for(str(got[-1]))
|
|
1108
|
+
if size:
|
|
1109
|
+
fsize = size[0]
|
|
1110
|
+
elif op == "Td":
|
|
1111
|
+
got = numbers(2)
|
|
1112
|
+
if got:
|
|
1113
|
+
tlm = _mul((1.0, 0.0, 0.0, 1.0, got[0], got[1]), tlm)
|
|
1114
|
+
tm = tlm
|
|
1115
|
+
elif op == "TD":
|
|
1116
|
+
got = numbers(2)
|
|
1117
|
+
if got:
|
|
1118
|
+
tl = -got[1]
|
|
1119
|
+
tlm = _mul((1.0, 0.0, 0.0, 1.0, got[0], got[1]), tlm)
|
|
1120
|
+
tm = tlm
|
|
1121
|
+
elif op == "Tm":
|
|
1122
|
+
got = numbers(6)
|
|
1123
|
+
if got:
|
|
1124
|
+
tm = tlm = tuple(got)
|
|
1125
|
+
elif op == "T*":
|
|
1126
|
+
tlm = _mul((1.0, 0.0, 0.0, 1.0, 0.0, -tl), tlm)
|
|
1127
|
+
tm = tlm
|
|
1128
|
+
elif op == "TL":
|
|
1129
|
+
got = numbers(1)
|
|
1130
|
+
if got:
|
|
1131
|
+
tl = got[0]
|
|
1132
|
+
elif op == "Tc":
|
|
1133
|
+
got = numbers(1)
|
|
1134
|
+
if got:
|
|
1135
|
+
tc = got[0]
|
|
1136
|
+
elif op == "Tw":
|
|
1137
|
+
got = numbers(1)
|
|
1138
|
+
if got:
|
|
1139
|
+
tw = got[0]
|
|
1140
|
+
elif op == "Tz":
|
|
1141
|
+
got = numbers(1)
|
|
1142
|
+
if got:
|
|
1143
|
+
th = got[0] / 100.0
|
|
1144
|
+
elif op == "Ts":
|
|
1145
|
+
got = numbers(1)
|
|
1146
|
+
if got:
|
|
1147
|
+
rise = got[0]
|
|
1148
|
+
elif op == "Tj":
|
|
1149
|
+
if operands and isinstance(operands[-1], bytes):
|
|
1150
|
+
show(operands[-1])
|
|
1151
|
+
elif op == "'":
|
|
1152
|
+
tlm = _mul((1.0, 0.0, 0.0, 1.0, 0.0, -tl), tlm)
|
|
1153
|
+
tm = tlm
|
|
1154
|
+
if operands and isinstance(operands[-1], bytes):
|
|
1155
|
+
show(operands[-1])
|
|
1156
|
+
elif op == '"':
|
|
1157
|
+
got = [v for v in operands[-3:-1] if isinstance(v, (int, float))]
|
|
1158
|
+
if len(got) == 2:
|
|
1159
|
+
tw, tc = float(got[0]), float(got[1])
|
|
1160
|
+
tlm = _mul((1.0, 0.0, 0.0, 1.0, 0.0, -tl), tlm)
|
|
1161
|
+
tm = tlm
|
|
1162
|
+
if operands and isinstance(operands[-1], bytes):
|
|
1163
|
+
show(operands[-1])
|
|
1164
|
+
elif op == "TJ":
|
|
1165
|
+
array = operands[-1] if operands and isinstance(operands[-1], list) else []
|
|
1166
|
+
for item in array:
|
|
1167
|
+
if isinstance(item, bytes):
|
|
1168
|
+
show(item)
|
|
1169
|
+
elif isinstance(item, (int, float)):
|
|
1170
|
+
adjust(float(item))
|
|
1171
|
+
elif op == "INLINE_IMAGE":
|
|
1172
|
+
state.images += 1
|
|
1173
|
+
state.image_kinds.add("inline")
|
|
1174
|
+
elif op == "Do":
|
|
1175
|
+
got = [v for v in operands if isinstance(v, Name)]
|
|
1176
|
+
if got:
|
|
1177
|
+
_do_xobject(doc, str(got[-1]), xobjects, state, ctm, depth, seen)
|
|
1178
|
+
operands = []
|
|
1179
|
+
|
|
1180
|
+
|
|
1181
|
+
def _do_xobject(
|
|
1182
|
+
doc: PdfDocument, key: str, xobjects: dict, state: _PageState, ctm: tuple, depth: int, seen: set
|
|
1183
|
+
) -> None:
|
|
1184
|
+
node = xobjects.get(key)
|
|
1185
|
+
resolved = doc.resolve(node)
|
|
1186
|
+
if not isinstance(resolved, Stream):
|
|
1187
|
+
return
|
|
1188
|
+
subtype = str(doc.resolve(resolved.d.get("Subtype")) or "")
|
|
1189
|
+
if subtype == "Image":
|
|
1190
|
+
state.images += 1
|
|
1191
|
+
size = doc.resolve(resolved.d.get("Length"))
|
|
1192
|
+
state.image_bytes += int(size) if isinstance(size, int) else len(resolved.raw)
|
|
1193
|
+
filters = doc.resolve(resolved.d.get("Filter"))
|
|
1194
|
+
for name in filters if isinstance(filters, list) else [filters]:
|
|
1195
|
+
state.image_kinds.add(str(doc.resolve(name) or "raw"))
|
|
1196
|
+
return
|
|
1197
|
+
if subtype != "Form" or depth >= 8:
|
|
1198
|
+
return
|
|
1199
|
+
marker = (node.num, node.gen) if isinstance(node, Ref) else id(resolved)
|
|
1200
|
+
if marker in seen:
|
|
1201
|
+
return
|
|
1202
|
+
seen.add(marker)
|
|
1203
|
+
try:
|
|
1204
|
+
data = doc.data_of(node)
|
|
1205
|
+
except _ImageData as image:
|
|
1206
|
+
state.images += 1
|
|
1207
|
+
state.image_kinds.add(image.filter_name)
|
|
1208
|
+
return
|
|
1209
|
+
except PdfError:
|
|
1210
|
+
return
|
|
1211
|
+
inner = doc.as_dict(resolved.d.get("Resources")) or {}
|
|
1212
|
+
matrix = doc.resolve(resolved.d.get("Matrix"))
|
|
1213
|
+
local = ctm
|
|
1214
|
+
if isinstance(matrix, list) and len(matrix) == 6:
|
|
1215
|
+
try:
|
|
1216
|
+
local = _mul(tuple(float(doc.resolve(v)) for v in matrix), ctm)
|
|
1217
|
+
except (TypeError, ValueError):
|
|
1218
|
+
local = ctm
|
|
1219
|
+
_run_content(doc, data, inner, state, local, depth + 1, seen)
|
|
1220
|
+
seen.discard(marker)
|
|
1221
|
+
|
|
1222
|
+
|
|
1223
|
+
def _rows_from_runs(runs: list[_Run], rotate: int) -> tuple[str, ...]:
|
|
1224
|
+
"""Glyph runs, grouped into rows by baseline and ordered by x. A slicing unit, not layout."""
|
|
1225
|
+
if not runs:
|
|
1226
|
+
return ()
|
|
1227
|
+
if rotate % 360 in (90, 270):
|
|
1228
|
+
runs = [_Run(y=r.x, x=-r.y, end_x=-r.y + (r.end_x - r.x), size=r.size, text=r.text)
|
|
1229
|
+
for r in runs]
|
|
1230
|
+
runs.sort(key=lambda r: (-r.y, r.x))
|
|
1231
|
+
rows: list[list[_Run]] = []
|
|
1232
|
+
current: list[_Run] = []
|
|
1233
|
+
baseline = 0.0
|
|
1234
|
+
for run in runs:
|
|
1235
|
+
tolerance = max(1.5, 0.35 * run.size)
|
|
1236
|
+
if current and abs(run.y - baseline) <= tolerance:
|
|
1237
|
+
current.append(run)
|
|
1238
|
+
else:
|
|
1239
|
+
if current:
|
|
1240
|
+
rows.append(current)
|
|
1241
|
+
current = [run]
|
|
1242
|
+
baseline = run.y
|
|
1243
|
+
if current:
|
|
1244
|
+
rows.append(current)
|
|
1245
|
+
out: list[str] = []
|
|
1246
|
+
for row in rows:
|
|
1247
|
+
row.sort(key=lambda r: r.x)
|
|
1248
|
+
pieces: list[str] = []
|
|
1249
|
+
previous_end: float | None = None
|
|
1250
|
+
for run in row:
|
|
1251
|
+
if previous_end is not None and run.x - previous_end > 0.22 * max(run.size, 1.0):
|
|
1252
|
+
pieces.append(" ")
|
|
1253
|
+
pieces.append(run.text)
|
|
1254
|
+
previous_end = max(run.end_x, run.x)
|
|
1255
|
+
line = "".join(pieces).replace("\t", " ").replace("\r", " ").replace("\n", " ").strip()
|
|
1256
|
+
if line:
|
|
1257
|
+
out.append(line)
|
|
1258
|
+
return tuple(out)
|
|
1259
|
+
|
|
1260
|
+
|
|
1261
|
+
# ------------------------------------------------------------------------------- the API
|
|
1262
|
+
|
|
1263
|
+
|
|
1264
|
+
@dataclass(frozen=True)
|
|
1265
|
+
class PdfPage:
|
|
1266
|
+
"""One page: its rows, and the counts behind everything the rows do not carry."""
|
|
1267
|
+
|
|
1268
|
+
number: int
|
|
1269
|
+
rows: tuple[str, ...]
|
|
1270
|
+
show_ops: int
|
|
1271
|
+
vouched: int
|
|
1272
|
+
unmapped: int
|
|
1273
|
+
unmapped_fonts: tuple[str, ...]
|
|
1274
|
+
images: int
|
|
1275
|
+
image_bytes: int
|
|
1276
|
+
image_kinds: tuple[str, ...]
|
|
1277
|
+
|
|
1278
|
+
@property
|
|
1279
|
+
def characters(self) -> int:
|
|
1280
|
+
return sum(len(row.strip()) for row in self.rows)
|
|
1281
|
+
|
|
1282
|
+
|
|
1283
|
+
@dataclass(frozen=True)
|
|
1284
|
+
class PdfText:
|
|
1285
|
+
version: str
|
|
1286
|
+
pages: tuple[PdfPage, ...]
|
|
1287
|
+
|
|
1288
|
+
@property
|
|
1289
|
+
def vouched(self) -> int:
|
|
1290
|
+
return sum(page.vouched for page in self.pages)
|
|
1291
|
+
|
|
1292
|
+
@property
|
|
1293
|
+
def unmapped(self) -> int:
|
|
1294
|
+
return sum(page.unmapped for page in self.pages)
|
|
1295
|
+
|
|
1296
|
+
@property
|
|
1297
|
+
def show_ops(self) -> int:
|
|
1298
|
+
return sum(page.show_ops for page in self.pages)
|
|
1299
|
+
|
|
1300
|
+
@property
|
|
1301
|
+
def images(self) -> int:
|
|
1302
|
+
return sum(page.images for page in self.pages)
|
|
1303
|
+
|
|
1304
|
+
@property
|
|
1305
|
+
def unmapped_fonts(self) -> tuple[str, ...]:
|
|
1306
|
+
out: set[str] = set()
|
|
1307
|
+
for page in self.pages:
|
|
1308
|
+
out.update(page.unmapped_fonts)
|
|
1309
|
+
return tuple(sorted(out))
|
|
1310
|
+
|
|
1311
|
+
|
|
1312
|
+
def read_pdf(path: str | Path, vouch: bool = True) -> PdfText:
|
|
1313
|
+
"""Every page of `path`, as rows this reader can vouch for.
|
|
1314
|
+
|
|
1315
|
+
`vouch=False` renders the characters it CANNOT vouch for as well, mapping an unmapped code
|
|
1316
|
+
to the character its raw value would suggest. That output is mojibake by construction and
|
|
1317
|
+
is never what `docread` asks for — it exists so the audit in
|
|
1318
|
+
`.shiftwork/probes/J25-D3-pdf-coverage.py` can *demonstrate* what the guard is suppressing
|
|
1319
|
+
rather than assert it. Nothing in the runtime calls it with `vouch=False`.
|
|
1320
|
+
"""
|
|
1321
|
+
path = Path(path)
|
|
1322
|
+
raw = path.read_bytes()
|
|
1323
|
+
header = _HEADER_RE.match(raw[:1024]) or _HEADER_RE.search(raw[:1024])
|
|
1324
|
+
if header is None:
|
|
1325
|
+
raise PdfError("it does not begin with a %PDF- header, so it is not a PDF")
|
|
1326
|
+
version = header.group(1).decode("ascii")
|
|
1327
|
+
doc = PdfDocument(raw)
|
|
1328
|
+
encryption = doc.encryption()
|
|
1329
|
+
if encryption is not None:
|
|
1330
|
+
handler = encryption.get("Filter", "?")
|
|
1331
|
+
raise PdfError(
|
|
1332
|
+
f"it is an ENCRYPTED PDF (/Encrypt, security handler /{handler}) and this reader "
|
|
1333
|
+
"decrypts nothing — no file on the corpus it was measured against was encrypted, "
|
|
1334
|
+
"so decryption here would be unmeasured code"
|
|
1335
|
+
)
|
|
1336
|
+
page_nodes = doc.pages()
|
|
1337
|
+
if not page_nodes:
|
|
1338
|
+
raise PdfError(
|
|
1339
|
+
f"its structure yielded no page at all: {len(doc.objects)} objects were parsed and "
|
|
1340
|
+
"none of them is a page tree or a /Type/Page"
|
|
1341
|
+
)
|
|
1342
|
+
pages: list[PdfPage] = []
|
|
1343
|
+
for index, (node, inherited) in enumerate(page_nodes):
|
|
1344
|
+
state = _PageState()
|
|
1345
|
+
resources = doc.as_dict(inherited.get("Resources", node.get("Resources"))) or {}
|
|
1346
|
+
for chunk in _content_chunks(doc, node):
|
|
1347
|
+
try:
|
|
1348
|
+
_run_content(doc, chunk, resources, state, _IDENTITY, 0, set())
|
|
1349
|
+
except PdfError:
|
|
1350
|
+
continue
|
|
1351
|
+
except RecursionError:
|
|
1352
|
+
continue
|
|
1353
|
+
rotate = doc.resolve(inherited.get("Rotate", node.get("Rotate", 0)))
|
|
1354
|
+
rows = _rows_from_runs(state.runs, int(rotate) if isinstance(rotate, int) else 0)
|
|
1355
|
+
if not vouch:
|
|
1356
|
+
rows = _unvouched_rows(doc, node, resources)
|
|
1357
|
+
pages.append(
|
|
1358
|
+
PdfPage(
|
|
1359
|
+
number=index + 1,
|
|
1360
|
+
rows=rows,
|
|
1361
|
+
show_ops=state.show_ops,
|
|
1362
|
+
vouched=state.vouched,
|
|
1363
|
+
unmapped=state.unmapped,
|
|
1364
|
+
unmapped_fonts=tuple(sorted(state.unmapped_fonts)),
|
|
1365
|
+
images=state.images,
|
|
1366
|
+
image_bytes=state.image_bytes,
|
|
1367
|
+
image_kinds=tuple(sorted(state.image_kinds)),
|
|
1368
|
+
)
|
|
1369
|
+
)
|
|
1370
|
+
return PdfText(version=version, pages=tuple(pages))
|
|
1371
|
+
|
|
1372
|
+
|
|
1373
|
+
def _content_chunks(doc: PdfDocument, node: dict) -> list[bytes]:
|
|
1374
|
+
contents = node.get("Contents")
|
|
1375
|
+
targets = contents if isinstance(doc.resolve(contents), list) else [contents]
|
|
1376
|
+
if isinstance(doc.resolve(contents), list):
|
|
1377
|
+
targets = doc.resolve(contents)
|
|
1378
|
+
out: list[bytes] = []
|
|
1379
|
+
for target in targets:
|
|
1380
|
+
try:
|
|
1381
|
+
out.append(doc.data_of(target))
|
|
1382
|
+
except (PdfError, _ImageData):
|
|
1383
|
+
continue
|
|
1384
|
+
return out
|
|
1385
|
+
|
|
1386
|
+
|
|
1387
|
+
def _unvouched_rows(doc: PdfDocument, node: dict, resources: dict) -> tuple[str, ...]:
|
|
1388
|
+
"""What the page WOULD render if every code were decoded regardless of provenance.
|
|
1389
|
+
|
|
1390
|
+
Audit-only, and deliberately crude: an unmapped code becomes `chr(code)`, which is exactly
|
|
1391
|
+
the assumption a reader makes when it decodes a subset font's glyph indices as if they were
|
|
1392
|
+
characters. `read_pdf(vouch=False)` is the only caller, and nothing in the runtime calls
|
|
1393
|
+
that. It exists so the mojibake guard can be DEMONSTRATED rather than asserted.
|
|
1394
|
+
"""
|
|
1395
|
+
audit = _PageState(permissive=True)
|
|
1396
|
+
for chunk in _content_chunks(doc, node):
|
|
1397
|
+
try:
|
|
1398
|
+
_run_content(doc, chunk, resources, audit, _IDENTITY, 0, set())
|
|
1399
|
+
except (PdfError, RecursionError):
|
|
1400
|
+
continue
|
|
1401
|
+
rotate = doc.resolve(node.get("Rotate", 0))
|
|
1402
|
+
return _rows_from_runs(audit.runs, int(rotate) if isinstance(rotate, int) else 0)
|