bantamkit 0.27.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (119) hide show
  1. bantamkit/__init__.py +32 -0
  2. bantamkit/agent.py +458 -0
  3. bantamkit/assets/contracts/default.yaml +90 -0
  4. bantamkit/assets/evals/devteam/manifest.yaml +351 -0
  5. bantamkit/assets/evals/devteam/repo/HISTORY.md +18 -0
  6. bantamkit/assets/evals/devteam/repo/README.md +12 -0
  7. bantamkit/assets/evals/devteam/repo/docs/architecture.md +17 -0
  8. bantamkit/assets/evals/devteam/repo/docs/runbook.md +10 -0
  9. bantamkit/assets/evals/devteam/repo/issues/142-settlement-timeout.md +23 -0
  10. bantamkit/assets/evals/devteam/repo/patches/0009-retry-budget.patch +38 -0
  11. bantamkit/assets/evals/devteam/repo/src/ledger/__init__.py +3 -0
  12. bantamkit/assets/evals/devteam/repo/src/ledger/config.py +35 -0
  13. bantamkit/assets/evals/devteam/repo/src/ledger/errors.py +13 -0
  14. bantamkit/assets/evals/devteam/repo/src/ledger/posting.py +12 -0
  15. bantamkit/assets/evals/devteam/repo/src/ledger/registry.py +7 -0
  16. bantamkit/assets/evals/devteam/repo/src/ledger/report.py +9 -0
  17. bantamkit/assets/evals/devteam/repo/src/ledger/retry.py +17 -0
  18. bantamkit/assets/evals/devteam/repo/src/ledger/settle.py +16 -0
  19. bantamkit/assets/evals/devteam/repo/src/ledger/validate.py +14 -0
  20. bantamkit/assets/evals/devteam/repo/tests/test_posting.py +13 -0
  21. bantamkit/assets/evals/devteam/repo/tests/test_settle.py +9 -0
  22. bantamkit/assets/evals/devteam/tasks/dt-error-contract.yaml +186 -0
  23. bantamkit/assets/evals/devteam/tasks/dt-handler-map.yaml +183 -0
  24. bantamkit/assets/evals/devteam/tasks/dt-patch-before-after.yaml +182 -0
  25. bantamkit/assets/evals/devteam/tasks/dt-retry-attempts.yaml +181 -0
  26. bantamkit/assets/evals/devteam/tasks/dt-settlement-config.yaml +185 -0
  27. bantamkit/assets/evals/devteam/tasks/dt-symbol-home.yaml +181 -0
  28. bantamkit/assets/evals/devteam/tasks/dt-trace-blame.yaml +182 -0
  29. bantamkit/assets/evals/devteam/tasks/dt-unread-key.yaml +180 -0
  30. bantamkit/assets/evals/document/tasks/doc-large-in-137.yaml +38 -0
  31. bantamkit/assets/evals/document/tasks/doc-large-in-359.yaml +44 -0
  32. bantamkit/assets/evals/document/tasks/doc-large-in-372.yaml +38 -0
  33. bantamkit/assets/evals/document/tasks/doc-large-out-11764.yaml +37 -0
  34. bantamkit/assets/evals/document/tasks/doc-large-out-4137.yaml +37 -0
  35. bantamkit/assets/evals/document/tasks/doc-large-out-8022.yaml +37 -0
  36. bantamkit/assets/evals/document/tasks/doc-small-137.yaml +37 -0
  37. bantamkit/assets/evals/document/tasks/doc-small-261.yaml +37 -0
  38. bantamkit/assets/evals/document/tasks/doc-small-388.yaml +37 -0
  39. bantamkit/assets/evals/fixtures/.gitkeep +0 -0
  40. bantamkit/assets/evals/fixtures/catalog.json +6 -0
  41. bantamkit/assets/evals/perturbations/task-completion.yaml +576 -0
  42. bantamkit/assets/evals/tasks/.gitkeep +0 -0
  43. bantamkit/assets/evals/tasks/extract-contact.yaml +14 -0
  44. bantamkit/assets/evals/tasks/extract-invoice.yaml +14 -0
  45. bantamkit/assets/evals/tasks/extract-order.yaml +15 -0
  46. bantamkit/assets/evals/tasks/extract-schedule.yaml +14 -0
  47. bantamkit/assets/evals/tasks/extract-versions.yaml +17 -0
  48. bantamkit/assets/evals/tasks/nav-prod-port.yaml +84 -0
  49. bantamkit/assets/evals/tasks/nav-release-bundle.yaml +87 -0
  50. bantamkit/assets/evals/tasks/recall-audit-retention.yaml +17 -0
  51. bantamkit/assets/evals/tasks/recall-cache-ttl.yaml +13 -0
  52. bantamkit/assets/evals/tasks/recall-db-port.yaml +17 -0
  53. bantamkit/assets/evals/tasks/recall-deploy.yaml +13 -0
  54. bantamkit/assets/evals/tasks/recall-env-endpoint.yaml +18 -0
  55. bantamkit/assets/evals/tasks/recall-oncall-rotation.yaml +21 -0
  56. bantamkit/assets/evals/tasks/recall-oncall.yaml +13 -0
  57. bantamkit/assets/evals/tasks/recall-org-quota.yaml +18 -0
  58. bantamkit/assets/evals/tasks/recall-owner.yaml +13 -0
  59. bantamkit/assets/evals/tasks/shop-basket-total.yaml +10 -0
  60. bantamkit/assets/evals/tasks/shop-cheapest.yaml +9 -0
  61. bantamkit/assets/evals/tasks/shop-compare.yaml +9 -0
  62. bantamkit/assets/evals/tasks/shop-gadget-value.yaml +9 -0
  63. bantamkit/assets/evals/tasks/shop-stock-total.yaml +9 -0
  64. bantamkit/assets/evals/tasks/shop-total.yaml +9 -0
  65. bantamkit/assets/profiles/default.yaml +31 -0
  66. bantamkit/assets/profiles/patient.yaml +31 -0
  67. bantamkit/assets/rubrics/.gitkeep +0 -0
  68. bantamkit/assets/rubrics/code-quality.yaml +20 -0
  69. bantamkit/assets/rubrics/grounded-completion.yaml +37 -0
  70. bantamkit/assets/rubrics/task-completion.yaml +28 -0
  71. bantamkit/assets/schemas/shiftwork-checkpoint.json +188 -0
  72. bantamkit/assets/skills/.gitkeep +0 -0
  73. bantamkit/assets/skills/file-graph.md +7 -0
  74. bantamkit/assets/skills/memory.md +35 -0
  75. bantamkit/assets/tools/.gitkeep +0 -0
  76. bantamkit/assets/tools/bantamkit_read.json +48 -0
  77. bantamkit/assets/tools/bantamkit_status.json +25 -0
  78. bantamkit/assets/tools/build_identity.json +17 -0
  79. bantamkit/assets/tools/document_list.json +12 -0
  80. bantamkit/assets/tools/document_read.json +31 -0
  81. bantamkit/assets/tools/file_graph.json +12 -0
  82. bantamkit/assets/tools/memory_compact.json +31 -0
  83. bantamkit/assets/tools/memory_recall.json +38 -0
  84. bantamkit/assets/tools/memory_save.json +61 -0
  85. bantamkit/assets/tools/shiftwork_clock_in.json +25 -0
  86. bantamkit/assets/tools/shiftwork_clock_out.json +60 -0
  87. bantamkit/assets/tools/shiftwork_status.json +25 -0
  88. bantamkit/assets/tools/skill_audit.json +70 -0
  89. bantamkit/assets/tools/validate_json.json +31 -0
  90. bantamkit/assets.py +67 -0
  91. bantamkit/budget.py +114 -0
  92. bantamkit/client.py +329 -0
  93. bantamkit/contract.py +522 -0
  94. bantamkit/criticreplay.py +3241 -0
  95. bantamkit/critique.py +301 -0
  96. bantamkit/docread.py +1744 -0
  97. bantamkit/evalrun.py +2003 -0
  98. bantamkit/eventlog.py +282 -0
  99. bantamkit/filegraph.py +218 -0
  100. bantamkit/loopguard.py +101 -0
  101. bantamkit/mcpreport.py +763 -0
  102. bantamkit/mcpserver.py +1334 -0
  103. bantamkit/memory/__init__.py +28 -0
  104. bantamkit/memory/__main__.py +291 -0
  105. bantamkit/memory/component.py +569 -0
  106. bantamkit/memory/divergence.py +744 -0
  107. bantamkit/memory/layers.py +257 -0
  108. bantamkit/memory/store.py +940 -0
  109. bantamkit/pdfread.py +1402 -0
  110. bantamkit/profile.py +46 -0
  111. bantamkit/shiftwork.py +212 -0
  112. bantamkit/skillaudit.py +853 -0
  113. bantamkit/statusline.py +313 -0
  114. bantamkit/structured.py +125 -0
  115. bantamkit/textutil.py +30 -0
  116. bantamkit-0.27.0.dist-info/METADATA +207 -0
  117. bantamkit-0.27.0.dist-info/RECORD +119 -0
  118. bantamkit-0.27.0.dist-info/WHEEL +4 -0
  119. bantamkit-0.27.0.dist-info/entry_points.txt +2 -0
bantamkit/pdfread.py ADDED
@@ -0,0 +1,1402 @@
1
+ """PDF text extraction with nothing but the standard library — and a reason for every refusal.
2
+
3
+ **Why this file exists at all.** J25-PREP measured this machine, 2026-08-20: `pypdf`, `PyPDF2`,
4
+ `fitz`, `pdfminer`, `pdfplumber`, `pdftotext`, `mutool` and `qpdf` are all **absent**, and
5
+ `/usr/bin/textutil`, which is present, does not do PDF. The runtime dependency list is fixed at
6
+ three packages, so a PDF reader here is written or there is none. `zlib` is standard library,
7
+ which is the whole of the compression problem; everything else a PDF needs — a tokenizer, an
8
+ object graph, a page tree, a content-stream interpreter and a character map — is arithmetic.
9
+
10
+ **The one rule this module is built around: a PDF string's bytes are not characters.** They are
11
+ indices into a font's encoding. For a composite font with `Identity-H` encoding they are glyph
12
+ indices *in that font's own subset*, and without a `/ToUnicode` map nothing in the file says
13
+ what character a glyph is. Decoding those bytes anyway produces text that reads as plausible
14
+ noise and that no caller can distinguish from content — the same failure class as
15
+ `/usr/bin/textutil` echoing Mac Roman at exit 0 (measured, J25-D1). So every character this
16
+ module emits carries a provenance, and **a character it cannot vouch for is never emitted**:
17
+
18
+ - from a `/ToUnicode` CMap in the file — the file's own statement of what the code means;
19
+ - from a simple font under `WinAnsiEncoding`, `MacRomanEncoding` or `StandardEncoding`;
20
+ - from a `/Differences` glyph name that resolves to a codepoint (`uniXXXX`, `AGL`).
21
+
22
+ Anything else is **counted, not rendered**: `PdfPage.unmapped` is how many characters were
23
+ dropped and `PdfPage.unmapped_fonts` names the fonts that produced them, so the caller can
24
+ disclose the gap instead of quoting noise. A page whose every character is unmapped yields no
25
+ rows and says why; `docread` turns that into a refusal when it is true of the whole file.
26
+
27
+ **What a "row" is here, and what it is not.** A PDF has no rows. It has glyphs at coordinates.
28
+ This module groups glyphs into rows by baseline (`_rows_from_runs`) and orders them by x, so
29
+ that `docread.page()` has a slice unit. That is a *slicing* decision, not a claim about
30
+ structure: a two-column page will interleave and a table will not come back as columns.
31
+
32
+ **Structure, without the xref.** The cross-reference table is an index, and on a real corpus it
33
+ is the part most likely to be stale, rewritten by an incremental update, or replaced by an xref
34
+ *stream* that itself needs the object graph to read. So this module does not read it: it scans
35
+ the file for `N G obj` and parses every object it finds, skipping the byte ranges that belong to
36
+ stream data (which is where a false `obj` match would come from), then expands every
37
+ `/Type/ObjStm` object stream — the PDF 1.5+ container the naive J25-PREP pass could not see
38
+ through, 13 of the user's 28 files. Later definitions of an object number win, which is what an
39
+ incremental update means. This costs one linear scan and buys immunity to a broken index.
40
+
41
+ Encryption is refused by name and never guessed at: not one file on the J25 corpus is encrypted
42
+ (`/Encrypt` absent from all 28, measured `6994f96`), so any decryption code here would be
43
+ unmeasured code, which this program does not ship.
44
+ """
45
+
46
+ from __future__ import annotations
47
+
48
+ import base64
49
+ import re
50
+ import unicodedata
51
+ import zlib
52
+ from dataclasses import dataclass, field
53
+ from pathlib import Path
54
+
55
+ __all__ = ["PdfError", "PdfPage", "PdfText", "read_pdf"]
56
+
57
+
58
+ class PdfError(Exception):
59
+ """The PDF could not be read. The message names what was seen, never a bare traceback."""
60
+
61
+
62
+ # --------------------------------------------------------------------------- object model
63
+
64
+
65
+ class Name(str):
66
+ """A PDF name (`/Type`). A `str` subclass so comparisons read naturally, distinct from a
67
+ PDF *string*, which is `bytes` — the difference matters when a value may be either."""
68
+
69
+ __slots__ = ()
70
+
71
+
72
+ @dataclass(frozen=True)
73
+ class Ref:
74
+ num: int
75
+ gen: int
76
+
77
+
78
+ @dataclass
79
+ class Stream:
80
+ d: dict
81
+ raw: bytes
82
+
83
+
84
+ class _Keyword(str):
85
+ """A bare content-stream token: an operator, or `obj`/`endobj` in the object graph."""
86
+
87
+ __slots__ = ()
88
+
89
+
90
+ # --------------------------------------------------------------------------------- tokenizer
91
+
92
+ _WS = b"\x00\t\n\x0c\r "
93
+ _DELIMS = b"()<>[]{}/%"
94
+ _ESCAPES = {0x6E: 10, 0x72: 13, 0x74: 9, 0x62: 8, 0x66: 12}
95
+
96
+
97
+ def _skip_ws(data: bytes, i: int) -> int:
98
+ n = len(data)
99
+ while i < n:
100
+ c = data[i]
101
+ if c == 0x25: # '%' comment runs to end of line
102
+ while i < n and data[i] not in (0x0A, 0x0D):
103
+ i += 1
104
+ elif c in _WS:
105
+ i += 1
106
+ else:
107
+ break
108
+ return i
109
+
110
+
111
+ def _regular_run(data: bytes, i: int) -> tuple[bytes, int]:
112
+ n, j = len(data), i
113
+ while j < n and data[j] not in _WS and data[j] not in _DELIMS:
114
+ j += 1
115
+ return data[i:j], j
116
+
117
+
118
+ _HASH_ESCAPE = re.compile(rb"#([0-9A-Fa-f]{2})")
119
+
120
+
121
+ def _parse_name(data: bytes, i: int) -> tuple[Name, int]:
122
+ token, j = _regular_run(data, i + 1)
123
+ token = _HASH_ESCAPE.sub(lambda m: bytes([int(m.group(1), 16)]), token)
124
+ return Name(token.decode("utf-8", errors="replace")), j
125
+
126
+
127
+ def _parse_literal_string(data: bytes, i: int) -> tuple[bytes, int]:
128
+ i += 1
129
+ depth, n = 1, len(data)
130
+ out = bytearray()
131
+ while i < n:
132
+ c = data[i]
133
+ if c == 0x5C: # backslash
134
+ i += 1
135
+ if i >= n:
136
+ break
137
+ e = data[i]
138
+ if e in _ESCAPES:
139
+ out.append(_ESCAPES[e])
140
+ i += 1
141
+ elif 0x30 <= e <= 0x37:
142
+ digits = bytearray()
143
+ while i < n and len(digits) < 3 and 0x30 <= data[i] <= 0x37:
144
+ digits.append(data[i])
145
+ i += 1
146
+ out.append(int(digits, 8) & 0xFF)
147
+ elif e == 0x0A:
148
+ i += 1
149
+ elif e == 0x0D:
150
+ i += 1
151
+ if i < n and data[i] == 0x0A:
152
+ i += 1
153
+ else:
154
+ out.append(e)
155
+ i += 1
156
+ elif c == 0x28:
157
+ depth += 1
158
+ out.append(c)
159
+ i += 1
160
+ elif c == 0x29:
161
+ depth -= 1
162
+ i += 1
163
+ if depth == 0:
164
+ return bytes(out), i
165
+ out.append(c)
166
+ else:
167
+ out.append(c)
168
+ i += 1
169
+ raise PdfError("a literal string runs off the end of the data (no closing parenthesis)")
170
+
171
+
172
+ _HEX_DIGITS = frozenset(b"0123456789abcdefABCDEF")
173
+
174
+
175
+ def _parse_hex_string(data: bytes, i: int) -> tuple[bytes, int]:
176
+ i += 1
177
+ digits = bytearray()
178
+ n = len(data)
179
+ while i < n and data[i] != 0x3E: # '>'
180
+ if data[i] in _HEX_DIGITS:
181
+ digits.append(data[i])
182
+ i += 1
183
+ if len(digits) % 2:
184
+ digits.append(0x30)
185
+ return bytes.fromhex(digits.decode("ascii")), i + 1
186
+
187
+
188
+ _REF_TAIL = re.compile(rb"[\x00\t\r\n\f ]+(\d{1,5})[\x00\t\r\n\f ]+R(?![A-Za-z0-9])")
189
+
190
+
191
+ def _parse(data: bytes, i: int, depth: int = 0) -> tuple[object, int]:
192
+ """One PDF object starting at `i`. Returns the value and the offset just past it."""
193
+ if depth > 64:
194
+ raise PdfError("PDF object nesting is deeper than 64 levels")
195
+ i = _skip_ws(data, i)
196
+ if i >= len(data):
197
+ raise PdfError("the object data ends where an object was expected")
198
+ c = data[i]
199
+ if c == 0x2F: # '/'
200
+ return _parse_name(data, i)
201
+ if c == 0x28: # '('
202
+ return _parse_literal_string(data, i)
203
+ if c == 0x3C: # '<'
204
+ if data[i + 1 : i + 2] == b"<":
205
+ return _parse_dict(data, i, depth)
206
+ return _parse_hex_string(data, i)
207
+ if c == 0x5B: # '['
208
+ i += 1
209
+ out: list = []
210
+ while True:
211
+ i = _skip_ws(data, i)
212
+ if i >= len(data):
213
+ raise PdfError("an array runs off the end of the data")
214
+ if data[i] == 0x5D: # ']'
215
+ return out, i + 1
216
+ value, i = _parse(data, i, depth + 1)
217
+ out.append(value)
218
+ if c in (0x5D, 0x3E, 0x29, 0x7D):
219
+ raise PdfError(f"unbalanced {chr(c)!r} in the object data")
220
+ if c == 0x7B: # '{' PostScript calculator body; not data this reader uses
221
+ j = data.find(b"}", i)
222
+ return [], (j + 1 if j != -1 else len(data))
223
+ token, j = _regular_run(data, i)
224
+ if not token:
225
+ raise PdfError(f"byte {data[i]!r} starts no PDF object")
226
+ if token == b"true":
227
+ return True, j
228
+ if token == b"false":
229
+ return False, j
230
+ if token == b"null":
231
+ return None, j
232
+ try:
233
+ value: object = int(token)
234
+ except ValueError:
235
+ try:
236
+ value = float(token.replace(b"--", b"-"))
237
+ except ValueError:
238
+ return _Keyword(token.decode("latin-1")), j
239
+ else:
240
+ if value >= 0:
241
+ m = _REF_TAIL.match(data, j)
242
+ if m:
243
+ return Ref(int(token), int(m.group(1))), m.end()
244
+ return value, j
245
+
246
+
247
+ def _parse_dict(data: bytes, i: int, depth: int = 0) -> tuple[dict, int]:
248
+ i += 2
249
+ out: dict = {}
250
+ while True:
251
+ i = _skip_ws(data, i)
252
+ if i >= len(data):
253
+ raise PdfError("a dictionary runs off the end of the data")
254
+ if data[i : i + 2] == b">>":
255
+ return out, i + 2
256
+ if data[i] != 0x2F:
257
+ # A malformed key. Skip one object rather than losing the whole dictionary.
258
+ _, i = _parse(data, i, depth + 1)
259
+ continue
260
+ key, i = _parse_name(data, i)
261
+ value, i = _parse(data, i, depth + 1)
262
+ out[str(key)] = value
263
+
264
+
265
+ # ------------------------------------------------------------------------------ stream filters
266
+
267
+ # Filters whose output is an image, not text. Named so a refusal can say which one it met.
268
+ IMAGE_FILTERS = frozenset({"DCTDecode", "DCT", "JPXDecode", "JBIG2Decode", "CCITTFaxDecode", "CCF"})
269
+
270
+
271
+ class _ImageData(Exception):
272
+ """The stream decodes to an image. Not an error — a fact the caller acts on."""
273
+
274
+ def __init__(self, filter_name: str):
275
+ super().__init__(filter_name)
276
+ self.filter_name = filter_name
277
+
278
+
279
+ def _flate(data: bytes) -> bytes:
280
+ """`zlib`, then the tolerances a real corpus needs: truncated, raw, and leading junk."""
281
+ for attempt in (lambda: zlib.decompress(data), lambda: zlib.decompressobj().decompress(data)):
282
+ try:
283
+ return attempt()
284
+ except zlib.error:
285
+ pass
286
+ for skip in (1, 2, 3):
287
+ try:
288
+ return zlib.decompressobj().decompress(data[skip:])
289
+ except zlib.error:
290
+ pass
291
+ try:
292
+ return zlib.decompressobj(-15).decompress(data)
293
+ except zlib.error:
294
+ raise PdfError("a FlateDecode stream is not decompressible by zlib") from None
295
+
296
+
297
+ def _ascii85(data: bytes) -> bytes:
298
+ body = b"".join(data.split())
299
+ if body.startswith(b"<~"):
300
+ body = body[2:]
301
+ end = body.find(b"~>")
302
+ if end != -1:
303
+ body = body[:end]
304
+ try:
305
+ return base64.a85decode(body)
306
+ except ValueError:
307
+ raise PdfError("an ASCII85Decode stream is not valid ASCII85") from None
308
+
309
+
310
+ def _asciihex(data: bytes) -> bytes:
311
+ end = data.find(b">")
312
+ body = data[:end] if end != -1 else data
313
+ digits = bytes(ch for ch in body if ch in _HEX_DIGITS)
314
+ if len(digits) % 2:
315
+ digits += b"0"
316
+ return bytes.fromhex(digits.decode("ascii"))
317
+
318
+
319
+ def _runlength(data: bytes) -> bytes:
320
+ out = bytearray()
321
+ i, n = 0, len(data)
322
+ while i < n:
323
+ length = data[i]
324
+ i += 1
325
+ if length == 128:
326
+ break
327
+ if length < 128:
328
+ out += data[i : i + length + 1]
329
+ i += length + 1
330
+ else:
331
+ if i >= n:
332
+ break
333
+ out += bytes([data[i]]) * (257 - length)
334
+ i += 1
335
+ return bytes(out)
336
+
337
+
338
+ def _lzw(data: bytes, early: int = 1) -> bytes:
339
+ out = bytearray()
340
+ table = [bytes([k]) for k in range(256)] + [b"", b""]
341
+ bits, buf, nbits = 9, 0, 0
342
+ previous: bytes | None = None
343
+ for byte in data:
344
+ buf = (buf << 8) | byte
345
+ nbits += 8
346
+ while nbits >= bits:
347
+ code = (buf >> (nbits - bits)) & ((1 << bits) - 1)
348
+ nbits -= bits
349
+ if code == 256:
350
+ table = [bytes([k]) for k in range(256)] + [b"", b""]
351
+ bits, previous = 9, None
352
+ continue
353
+ if code == 257:
354
+ return bytes(out)
355
+ if previous is None:
356
+ if code >= len(table):
357
+ raise PdfError("an LZWDecode stream starts with an undefined code")
358
+ entry = table[code]
359
+ elif code < len(table):
360
+ entry = table[code]
361
+ table.append(previous + entry[:1])
362
+ else:
363
+ entry = previous + previous[:1]
364
+ table.append(entry)
365
+ out += entry
366
+ previous = entry
367
+ if len(table) + early >= (1 << bits) and bits < 12:
368
+ bits += 1
369
+ return bytes(out)
370
+
371
+
372
+ def _png_predictor(data: bytes, colors: int, bpc: int, columns: int) -> bytes:
373
+ bpp = max(1, (colors * bpc + 7) // 8)
374
+ rowlen = (columns * colors * bpc + 7) // 8
375
+ out = bytearray()
376
+ previous = bytearray(rowlen)
377
+ i, n = 0, len(data)
378
+ while i + 1 <= n - 1:
379
+ tag = data[i]
380
+ row = bytearray(data[i + 1 : i + 1 + rowlen])
381
+ if len(row) < rowlen:
382
+ row += bytes(rowlen - len(row))
383
+ i += 1 + rowlen
384
+ if tag == 1:
385
+ for k in range(bpp, rowlen):
386
+ row[k] = (row[k] + row[k - bpp]) & 0xFF
387
+ elif tag == 2:
388
+ for k in range(rowlen):
389
+ row[k] = (row[k] + previous[k]) & 0xFF
390
+ elif tag == 3:
391
+ for k in range(rowlen):
392
+ left = row[k - bpp] if k >= bpp else 0
393
+ row[k] = (row[k] + ((left + previous[k]) >> 1)) & 0xFF
394
+ elif tag == 4:
395
+ for k in range(rowlen):
396
+ a = row[k - bpp] if k >= bpp else 0
397
+ b = previous[k]
398
+ c = previous[k - bpp] if k >= bpp else 0
399
+ p = a + b - c
400
+ pa, pb, pc = abs(p - a), abs(p - b), abs(p - c)
401
+ best = a if (pa <= pb and pa <= pc) else (b if pb <= pc else c)
402
+ row[k] = (row[k] + best) & 0xFF
403
+ out += row
404
+ previous = row
405
+ return bytes(out)
406
+
407
+
408
+ def _tiff_predictor(data: bytes, colors: int, bpc: int, columns: int) -> bytes:
409
+ if bpc != 8:
410
+ return data
411
+ rowlen = columns * colors
412
+ out = bytearray(data)
413
+ for start in range(0, len(out) - rowlen + 1, rowlen):
414
+ for k in range(colors, rowlen):
415
+ out[start + k] = (out[start + k] + out[start + k - colors]) & 0xFF
416
+ return bytes(out)
417
+
418
+
419
+ # --------------------------------------------------------------------------- the object graph
420
+
421
+ _OBJ_RE = re.compile(rb"(?<![0-9])(\d{1,10})[\x00\t\r\n\f ]+(\d{1,5})[\x00\t\r\n\f ]+obj\b")
422
+ _TRAILER_RE = re.compile(rb"trailer[\x00\t\r\n\f ]*(?=<<)")
423
+ _HEADER_RE = re.compile(rb"%PDF-(\d+\.\d+)")
424
+
425
+
426
+ class PdfDocument:
427
+ """Every object in the file, found by scanning rather than by trusting the xref index."""
428
+
429
+ def __init__(self, raw: bytes):
430
+ self.raw = raw
431
+ self.objects: dict[int, object] = {}
432
+ self._decoded: dict[int, bytes] = {}
433
+ self._scan()
434
+ self._expand_object_streams()
435
+
436
+ # -- loading ---------------------------------------------------------------------------
437
+
438
+ def _scan(self) -> None:
439
+ raw = self.raw
440
+ guard = 0
441
+ for m in _OBJ_RE.finditer(raw):
442
+ if m.start() < guard:
443
+ continue # inside a stream's data: a false `obj` match, not an object
444
+ num = int(m.group(1))
445
+ try:
446
+ value, j = _parse(raw, m.end())
447
+ except PdfError:
448
+ continue
449
+ head = _skip_ws(raw, j)
450
+ if isinstance(value, dict) and raw[head : head + 6] == b"stream":
451
+ k = head + 6
452
+ if raw[k : k + 2] == b"\r\n":
453
+ k += 2
454
+ elif raw[k : k + 1] in (b"\n", b"\r"):
455
+ k += 1
456
+ end = self._stream_end(value, k)
457
+ value = Stream(value, raw[k:end])
458
+ after = raw.find(b"endstream", end)
459
+ guard = (after + 9) if after != -1 else end
460
+ self.objects[num] = value
461
+
462
+ def _stream_end(self, d: dict, start: int) -> int:
463
+ """Where the stream data stops. `/Length` when it is honest, `endstream` when it is not.
464
+
465
+ A `/Length` that is an indirect reference cannot be resolved yet — the object it points
466
+ at may not be scanned — so the search is the fallback, and it is also the fallback for a
467
+ `/Length` that simply lies, which real generators do.
468
+ """
469
+ raw = self.raw
470
+ length = d.get("Length")
471
+ if isinstance(length, int) and 0 <= length <= len(raw) - start:
472
+ candidate = start + length
473
+ if re.match(rb"[\x00\t\r\n\f ]*endstream", raw[candidate : candidate + 20]):
474
+ return candidate
475
+ found = raw.find(b"endstream", start)
476
+ end = found if found != -1 else len(raw)
477
+ while end > start and raw[end - 1] in (0x0A, 0x0D):
478
+ end -= 1
479
+ return end
480
+
481
+ def _expand_object_streams(self) -> None:
482
+ for num, obj in list(self.objects.items()):
483
+ if not (isinstance(obj, Stream) and obj.d.get("Type") == "ObjStm"):
484
+ continue
485
+ try:
486
+ data = self.stream_data(num, obj)
487
+ except (PdfError, _ImageData):
488
+ continue
489
+ count = self.resolve(obj.d.get("N")) or 0
490
+ first = self.resolve(obj.d.get("First")) or 0
491
+ if not isinstance(count, int) or not isinstance(first, int):
492
+ continue
493
+ fields = data[:first].split()
494
+ for k in range(0, min(len(fields) - 1, 2 * count), 2):
495
+ try:
496
+ inner, offset = int(fields[k]), int(fields[k + 1])
497
+ except ValueError:
498
+ continue
499
+ if inner in self.objects:
500
+ continue # a direct definition is the newer one; it wins
501
+ try:
502
+ value, _ = _parse(data, first + offset)
503
+ except PdfError:
504
+ continue
505
+ self.objects[inner] = value
506
+
507
+ # -- access ----------------------------------------------------------------------------
508
+
509
+ def resolve(self, obj: object, depth: int = 0) -> object:
510
+ while isinstance(obj, Ref) and depth < 32:
511
+ obj = self.objects.get(obj.num)
512
+ depth += 1
513
+ return obj
514
+
515
+ def as_dict(self, obj: object) -> dict | None:
516
+ obj = self.resolve(obj)
517
+ if isinstance(obj, Stream):
518
+ return obj.d
519
+ return obj if isinstance(obj, dict) else None
520
+
521
+ def stream_data(self, key: object, stream: Stream) -> bytes:
522
+ """Decoded bytes of a stream. Raises `_ImageData` when the filter chain is an image."""
523
+ cache_key = key if isinstance(key, int) else None
524
+ if cache_key is not None and cache_key in self._decoded:
525
+ return self._decoded[cache_key]
526
+ data = stream.raw
527
+ filters = self.resolve(stream.d.get("Filter", stream.d.get("F")))
528
+ if filters is None:
529
+ filters = []
530
+ elif not isinstance(filters, list):
531
+ filters = [filters]
532
+ parms = self.resolve(stream.d.get("DecodeParms", stream.d.get("DP")))
533
+ if not isinstance(parms, list):
534
+ parms = [parms] * len(filters)
535
+ for index, raw_filter in enumerate(filters):
536
+ name = str(self.resolve(raw_filter) or "")
537
+ if name in IMAGE_FILTERS:
538
+ raise _ImageData(name)
539
+ parm = self.as_dict(parms[index]) if index < len(parms) else None
540
+ if name in ("FlateDecode", "Fl"):
541
+ data = _flate(data)
542
+ elif name in ("LZWDecode", "LZW"):
543
+ early = self.resolve((parm or {}).get("EarlyChange", 1))
544
+ data = _lzw(data, 1 if early is None else int(early))
545
+ elif name in ("ASCII85Decode", "A85"):
546
+ data = _ascii85(data)
547
+ elif name in ("ASCIIHexDecode", "AHx"):
548
+ data = _asciihex(data)
549
+ elif name in ("RunLengthDecode", "RL"):
550
+ data = _runlength(data)
551
+ elif name in ("Crypt", ""):
552
+ continue
553
+ else:
554
+ raise PdfError(f"stream filter /{name} is one this reader does not implement")
555
+ if parm:
556
+ predictor = self.resolve(parm.get("Predictor", 1)) or 1
557
+ if isinstance(predictor, int) and predictor > 1:
558
+ colors = int(self.resolve(parm.get("Colors", 1)) or 1)
559
+ bpc = int(self.resolve(parm.get("BitsPerComponent", 8)) or 8)
560
+ columns = int(self.resolve(parm.get("Columns", 1)) or 1)
561
+ if predictor == 2:
562
+ data = _tiff_predictor(data, colors, bpc, columns)
563
+ else:
564
+ data = _png_predictor(data, colors, bpc, columns)
565
+ if cache_key is not None:
566
+ self._decoded[cache_key] = data
567
+ return data
568
+
569
+ def data_of(self, obj: object) -> bytes:
570
+ key = obj.num if isinstance(obj, Ref) else None
571
+ resolved = self.resolve(obj)
572
+ if not isinstance(resolved, Stream):
573
+ raise PdfError("expected a stream and found none")
574
+ return self.stream_data(key, resolved)
575
+
576
+ # -- structure -------------------------------------------------------------------------
577
+
578
+ def trailers(self) -> list[dict]:
579
+ out = []
580
+ for m in _TRAILER_RE.finditer(self.raw):
581
+ try:
582
+ d, _ = _parse_dict(self.raw, m.end())
583
+ except PdfError:
584
+ continue
585
+ out.append(d)
586
+ for obj in self.objects.values():
587
+ if isinstance(obj, Stream) and obj.d.get("Type") == "XRef":
588
+ out.append(obj.d)
589
+ return out
590
+
591
+ def encryption(self) -> dict | None:
592
+ for trailer in self.trailers():
593
+ if "Encrypt" in trailer:
594
+ return self.as_dict(trailer["Encrypt"]) or {}
595
+ return None
596
+
597
+ def catalog(self) -> dict | None:
598
+ for trailer in reversed(self.trailers()):
599
+ root = self.as_dict(trailer.get("Root"))
600
+ if root is not None and "Pages" in root:
601
+ return root
602
+ for num in sorted(self.objects, reverse=True):
603
+ d = self.as_dict(self.objects[num])
604
+ if isinstance(d, dict) and d.get("Type") == "Catalog":
605
+ return d
606
+ return None
607
+
608
+ _INHERITED = ("Resources", "MediaBox", "CropBox", "Rotate")
609
+
610
+ def pages(self) -> list[tuple[dict, dict]]:
611
+ """Every page, in document order, paired with its inherited attributes."""
612
+ out: list[tuple[dict, dict]] = []
613
+ catalog = self.catalog()
614
+ if catalog is not None:
615
+ self._walk(catalog.get("Pages"), {}, out, set(), 0)
616
+ if out:
617
+ return out
618
+ # No usable page tree. Every `/Type/Page` object, in object-number order: not the
619
+ # document's order, and said so by the caller rather than passed off as it.
620
+ for num in sorted(self.objects):
621
+ d = self.as_dict(self.objects[num])
622
+ if isinstance(d, dict) and d.get("Type") == "Page":
623
+ out.append((d, dict(d)))
624
+ return out
625
+
626
+ def _walk(
627
+ self, node: object, inherited: dict, out: list, seen: set, depth: int
628
+ ) -> None:
629
+ if depth > 64 or len(out) > 20000:
630
+ return
631
+ if isinstance(node, Ref):
632
+ if (node.num, node.gen) in seen:
633
+ return
634
+ seen.add((node.num, node.gen))
635
+ d = self.as_dict(node)
636
+ if d is None:
637
+ return
638
+ inherited = {**inherited, **{k: d[k] for k in self._INHERITED if k in d}}
639
+ kids = self.resolve(d.get("Kids"))
640
+ if d.get("Type") == "Page" or (not isinstance(kids, list) and "Contents" in d):
641
+ out.append((d, inherited))
642
+ return
643
+ if isinstance(kids, list):
644
+ for kid in kids:
645
+ self._walk(kid, inherited, out, seen, depth + 1)
646
+
647
+
648
+ # ------------------------------------------------------------------- characters, and vouching
649
+
650
+ # The Adobe Glyph List, restricted to what a Latin document actually uses. A glyph name this
651
+ # table does not hold and that is not a `uniXXXX` form is NOT guessed at — it is unmapped.
652
+ _AGL = {
653
+ "space": " ", "exclam": "!", "quotedbl": '"', "numbersign": "#", "dollar": "$",
654
+ "percent": "%", "ampersand": "&", "quotesingle": "'", "quoteright": "’",
655
+ "quoteleft": "‘", "parenleft": "(", "parenright": ")", "asterisk": "*",
656
+ "plus": "+", "comma": ",", "hyphen": "-", "period": ".", "slash": "/", "zero": "0",
657
+ "one": "1", "two": "2", "three": "3", "four": "4", "five": "5", "six": "6",
658
+ "seven": "7", "eight": "8", "nine": "9", "colon": ":", "semicolon": ";", "less": "<",
659
+ "equal": "=", "greater": ">", "question": "?", "at": "@", "bracketleft": "[",
660
+ "backslash": "\\", "bracketright": "]", "asciicircum": "^", "underscore": "_",
661
+ "grave": "`", "braceleft": "{", "bar": "|", "braceright": "}", "asciitilde": "~",
662
+ "quotedblleft": "“", "quotedblright": "”", "endash": "–",
663
+ "emdash": "—", "bullet": "•", "ellipsis": "…", "fi": "fi", "fl": "fl",
664
+ "ff": "ff", "ffi": "ffi", "ffl": "ffl", "dagger": "†", "daggerdbl": "‡",
665
+ "trademark": "™", "copyright": "©", "registered": "®",
666
+ "degree": "°", "plusminus": "±", "middot": "·", "sterling": "£",
667
+ "yen": "¥", "euro": "€", "cent": "¢", "section": "§",
668
+ "paragraph": "¶", "guillemotleft": "«", "guillemotright": "»",
669
+ "quotesinglbase": "‚", "quotedblbase": "„", "perthousand": "‰",
670
+ "nbspace": " ", "minus": "−", "divide": "÷", "multiply": "×",
671
+ }
672
+ for _letter in "abcdefghijklmnopqrstuvwxyz":
673
+ _AGL[_letter] = _letter
674
+ _AGL[_letter.upper()] = _letter.upper()
675
+
676
+ _UNI_NAME = re.compile(r"^uni([0-9A-Fa-f]{4,6})$")
677
+ _U_NAME = re.compile(r"^u([0-9A-Fa-f]{4,6})$")
678
+
679
+
680
+ def glyph_to_unicode(name: str) -> str | None:
681
+ """A `/Differences` glyph name as a character, or None when nothing in the file says.
682
+
683
+ `g23`, `index7`, `cid100` and a subset font's private names all return None **on purpose**:
684
+ the name is an index into a font, and the file has not said what character it draws.
685
+ """
686
+ if name in _AGL:
687
+ return _AGL[name]
688
+ m = _UNI_NAME.match(name) or _U_NAME.match(name)
689
+ if m:
690
+ try:
691
+ return chr(int(m.group(1), 16))
692
+ except (ValueError, OverflowError):
693
+ return None
694
+ base = name.split(".")[0]
695
+ if base != name and base in _AGL:
696
+ return _AGL[base]
697
+ return None
698
+
699
+
700
+ def _vouchable(text: str) -> bool:
701
+ """Is this character one a caller can be handed? Control, private-use and unassigned are
702
+ exactly the shapes glyph-index noise takes, so they are dropped even when a map produced
703
+ them."""
704
+ for ch in text:
705
+ if ch in "\t\n\r ":
706
+ continue
707
+ if ch == "�":
708
+ return False
709
+ if unicodedata.category(ch) in ("Cc", "Cf", "Cn", "Co", "Cs"):
710
+ return False
711
+ return True
712
+
713
+
714
+ _STANDARD_CODECS = {
715
+ "WinAnsiEncoding": "cp1252",
716
+ "MacRomanEncoding": "mac_roman",
717
+ "PDFDocEncoding": "cp1252",
718
+ }
719
+
720
+
721
+ def _base_encoding_map(name: str) -> dict[int, str]:
722
+ """Codes 0-255 under a named base encoding, ASCII-only when the file names none.
723
+
724
+ A font with no `/Encoding` uses its own built-in one, which the file does not state. ASCII
725
+ is where every Latin text encoding agrees, so ASCII is trusted and the high half is not.
726
+ """
727
+ out: dict[int, str] = {}
728
+ codec = _STANDARD_CODECS.get(name)
729
+ if codec:
730
+ for code in range(32, 256):
731
+ try:
732
+ ch = bytes([code]).decode(codec)
733
+ except UnicodeDecodeError:
734
+ continue
735
+ if _vouchable(ch):
736
+ out[code] = ch
737
+ return out
738
+ for code in range(32, 127):
739
+ out[code] = chr(code)
740
+ return out
741
+
742
+
743
+ def parse_cmap(data: bytes) -> tuple[dict[int, str], set[int]]:
744
+ """A `/ToUnicode` CMap as code -> text, plus the code byte-lengths it declares."""
745
+ mapping: dict[int, str] = {}
746
+ lengths: set[int] = set()
747
+
748
+ def utf16(hexbytes: bytes) -> str:
749
+ raw = hexbytes.decode("ascii")
750
+ if len(raw) % 2:
751
+ raw += "0"
752
+ try:
753
+ return bytes.fromhex(raw).decode("utf-16-be", errors="replace")
754
+ except ValueError:
755
+ return ""
756
+
757
+ for block in re.findall(rb"begincodespacerange(.*?)endcodespacerange", data, re.S):
758
+ for src, _dst in re.findall(rb"<([0-9A-Fa-f]*)>\s*<([0-9A-Fa-f]*)>", block):
759
+ if src:
760
+ lengths.add(max(1, len(src) // 2))
761
+ for block in re.findall(rb"beginbfchar(.*?)endbfchar", data, re.S):
762
+ for src, dst in re.findall(rb"<([0-9A-Fa-f]+)>\s*<([0-9A-Fa-f]*)>", block):
763
+ lengths.add(max(1, len(src) // 2))
764
+ mapping[int(src, 16)] = utf16(dst)
765
+ for block in re.findall(rb"beginbfrange(.*?)endbfrange", data, re.S):
766
+ pattern = rb"<([0-9A-Fa-f]+)>\s*<([0-9A-Fa-f]+)>\s*(?:<([0-9A-Fa-f]*)>|\[(.*?)\])"
767
+ for m in re.finditer(pattern, block, re.S):
768
+ lo, hi = int(m.group(1), 16), int(m.group(2), 16)
769
+ lengths.add(max(1, len(m.group(1)) // 2))
770
+ if hi < lo or hi - lo > 65535:
771
+ continue
772
+ if m.group(3) is not None:
773
+ base = m.group(3)
774
+ prefix, tail = base[:-4], base[-4:] if len(base) >= 4 else base
775
+ try:
776
+ start = int(tail, 16) if tail else 0
777
+ except ValueError:
778
+ continue
779
+ for k in range(hi - lo + 1):
780
+ mapping[lo + k] = utf16(prefix + f"{start + k:04X}".encode())
781
+ else:
782
+ for k, item in enumerate(re.findall(rb"<([0-9A-Fa-f]*)>", m.group(4) or b"")):
783
+ if lo + k <= hi:
784
+ mapping[lo + k] = utf16(item)
785
+ return mapping, lengths
786
+
787
+
788
+ @dataclass
789
+ class _Font:
790
+ """One font resource, and everything needed to turn its codes into vouched characters."""
791
+
792
+ name: str
793
+ basefont: str
794
+ two_byte: bool
795
+ to_unicode: dict[int, str]
796
+ encoding: dict[int, str]
797
+ widths: dict[int, float]
798
+ default_width: float
799
+
800
+ def codes(self, raw: bytes) -> list[int]:
801
+ if self.two_byte:
802
+ if len(raw) % 2:
803
+ raw += b"\x00"
804
+ return [(raw[i] << 8) | raw[i + 1] for i in range(0, len(raw), 2)]
805
+ return list(raw)
806
+
807
+ def char(self, code: int, permissive: bool = False) -> str | None:
808
+ """The character this code means, or None when nothing in the file says.
809
+
810
+ `permissive=True` is the audit path and only the audit path: it decodes an unmapped
811
+ code as `chr(code)`, which is precisely the assumption that turns a subset font's
812
+ glyph indices into mojibake. `read_pdf(vouch=False)` is its only caller.
813
+ """
814
+ text = self.to_unicode.get(code)
815
+ if text is None:
816
+ text = self.encoding.get(code)
817
+ if text is None or not _vouchable(text):
818
+ if not permissive:
819
+ return None
820
+ try:
821
+ return text if text is not None else chr(code)
822
+ except ValueError:
823
+ return "\ufffd"
824
+ return text
825
+
826
+ def width(self, code: int) -> float:
827
+ return self.widths.get(code, self.default_width)
828
+
829
+
830
+ def _build_font(doc: PdfDocument, name: str, node: object) -> _Font:
831
+ d = doc.as_dict(node) or {}
832
+ subtype = str(doc.resolve(d.get("Subtype")) or "")
833
+ basefont = str(doc.resolve(d.get("BaseFont")) or "?")
834
+ to_unicode: dict[int, str] = {}
835
+ lengths: set[int] = set()
836
+ tu = doc.resolve(d.get("ToUnicode"))
837
+ if isinstance(tu, Stream):
838
+ try:
839
+ to_unicode, lengths = parse_cmap(doc.data_of(d.get("ToUnicode")))
840
+ except (PdfError, _ImageData):
841
+ to_unicode, lengths = {}, set()
842
+ if subtype == "Type0":
843
+ encoding_name = str(doc.resolve(d.get("Encoding")) or "")
844
+ # Composite-font codes are two bytes unless the ToUnicode CMap declares otherwise:
845
+ # Identity-H, and every predefined CJK CMap this corpus carries, are two-byte.
846
+ two_byte = lengths != {1}
847
+ del encoding_name
848
+ widths, default_width = _cid_widths(doc, d)
849
+ return _Font(
850
+ name=name,
851
+ basefont=basefont,
852
+ two_byte=two_byte,
853
+ to_unicode=to_unicode,
854
+ encoding={},
855
+ widths=widths,
856
+ # A composite font says nothing about characters without a ToUnicode map: its
857
+ # codes are CIDs, and for Identity-H they are glyph indices in a subset font.
858
+ # `encoding` is therefore left EMPTY, which is what makes `char()` return None
859
+ # for every code — the trust decision is that emptiness and nothing else.
860
+ default_width=default_width,
861
+ )
862
+ encoding_obj = doc.resolve(d.get("Encoding"))
863
+ base_name, differences = "", []
864
+ if isinstance(encoding_obj, Name):
865
+ base_name = str(encoding_obj)
866
+ elif isinstance(encoding_obj, dict):
867
+ base_name = str(doc.resolve(encoding_obj.get("BaseEncoding")) or "")
868
+ differences = doc.resolve(encoding_obj.get("Differences")) or []
869
+ encoding = _base_encoding_map(base_name)
870
+ code = 0
871
+ for item in differences if isinstance(differences, list) else []:
872
+ item = doc.resolve(item)
873
+ if isinstance(item, (int, float)):
874
+ code = int(item)
875
+ elif isinstance(item, Name):
876
+ ch = glyph_to_unicode(str(item))
877
+ if ch is None:
878
+ encoding.pop(code, None) # the file named a glyph, not a character
879
+ else:
880
+ encoding[code] = ch
881
+ code += 1
882
+ widths, default_width = _simple_widths(doc, d)
883
+ return _Font(
884
+ name=name,
885
+ basefont=basefont,
886
+ two_byte=False,
887
+ to_unicode=to_unicode,
888
+ encoding=encoding,
889
+ widths=widths,
890
+ default_width=default_width,
891
+ )
892
+
893
+
894
+ def _simple_widths(doc: PdfDocument, d: dict) -> tuple[dict[int, float], float]:
895
+ widths: dict[int, float] = {}
896
+ first = doc.resolve(d.get("FirstChar"))
897
+ table = doc.resolve(d.get("Widths"))
898
+ if isinstance(first, int) and isinstance(table, list):
899
+ for k, w in enumerate(table):
900
+ w = doc.resolve(w)
901
+ if isinstance(w, (int, float)):
902
+ widths[first + k] = float(w)
903
+ descriptor = doc.as_dict(d.get("FontDescriptor")) or {}
904
+ missing = doc.resolve(descriptor.get("MissingWidth", 500))
905
+ return widths, float(missing) if isinstance(missing, (int, float)) else 500.0
906
+
907
+
908
+ def _cid_widths(doc: PdfDocument, d: dict) -> tuple[dict[int, float], float]:
909
+ descendants = doc.resolve(d.get("DescendantFonts")) or []
910
+ child = doc.as_dict(descendants[0]) if isinstance(descendants, list) and descendants else {}
911
+ child = child or {}
912
+ default = doc.resolve(child.get("DW", 1000))
913
+ widths: dict[int, float] = {}
914
+ table = doc.resolve(child.get("W")) or []
915
+ index = 0
916
+ while isinstance(table, list) and index < len(table):
917
+ start = doc.resolve(table[index])
918
+ following = doc.resolve(table[index + 1]) if index + 1 < len(table) else None
919
+ if isinstance(following, list):
920
+ for k, w in enumerate(following):
921
+ w = doc.resolve(w)
922
+ if isinstance(start, int) and isinstance(w, (int, float)):
923
+ widths[start + k] = float(w)
924
+ index += 2
925
+ elif isinstance(following, int) and index + 2 < len(table):
926
+ w = doc.resolve(table[index + 2])
927
+ if isinstance(start, int) and isinstance(w, (int, float)) and following - start < 65536:
928
+ for cid in range(start, following + 1):
929
+ widths[cid] = float(w)
930
+ index += 3
931
+ else:
932
+ break
933
+ return widths, float(default) if isinstance(default, (int, float)) else 1000.0
934
+
935
+
936
+ # ------------------------------------------------------------- the content-stream interpreter
937
+
938
+
939
+ def _mul(m: tuple, n: tuple) -> tuple:
940
+ a1, b1, c1, d1, e1, f1 = m
941
+ a2, b2, c2, d2, e2, f2 = n
942
+ return (
943
+ a1 * a2 + b1 * c2,
944
+ a1 * b2 + b1 * d2,
945
+ c1 * a2 + d1 * c2,
946
+ c1 * b2 + d1 * d2,
947
+ e1 * a2 + f1 * c2 + e2,
948
+ e1 * b2 + f1 * d2 + f2,
949
+ )
950
+
951
+
952
+ _IDENTITY = (1.0, 0.0, 0.0, 1.0, 0.0, 0.0)
953
+
954
+
955
+ @dataclass
956
+ class _Run:
957
+ y: float
958
+ x: float
959
+ end_x: float
960
+ size: float
961
+ text: str
962
+
963
+
964
+ @dataclass
965
+ class _PageState:
966
+ runs: list[_Run] = field(default_factory=list)
967
+ show_ops: int = 0
968
+ vouched: int = 0
969
+ unmapped: int = 0
970
+ unmapped_fonts: set = field(default_factory=set)
971
+ images: int = 0
972
+ image_bytes: int = 0
973
+ image_kinds: set = field(default_factory=set)
974
+ permissive: bool = False
975
+
976
+
977
+ _INLINE_IMAGE = re.compile(rb"\bID[\x00\t\r\n\f ]")
978
+
979
+
980
+ def _tokenize_content(data: bytes) -> list:
981
+ """Content-stream tokens. Inline images (`BI … ID … EI`) are skipped as the binary they are."""
982
+ out: list = []
983
+ i, n = 0, len(data)
984
+ while True:
985
+ i = _skip_ws(data, i)
986
+ if i >= n:
987
+ return out
988
+ try:
989
+ value, j = _parse(data, i)
990
+ except PdfError:
991
+ # A byte that starts no object. Step over it — and STEPPING is the point: a
992
+ # `continue` that left `i` where it was hung this reader forever on a real file
993
+ # (`Defect_Apr 2026 RELEASE_SIT_AP1827-38933.pdf`, measured 2026-08-20).
994
+ i += 1
995
+ continue
996
+ if j <= i:
997
+ j = i + 1
998
+ if isinstance(value, _Keyword) and value == "BI":
999
+ m = _INLINE_IMAGE.search(data, j)
1000
+ end = data.find(b"EI", m.end() if m else j)
1001
+ out.append(_Keyword("INLINE_IMAGE"))
1002
+ i = (end + 2) if end != -1 else n
1003
+ continue
1004
+ out.append(value)
1005
+ i = j
1006
+
1007
+
1008
+ def _run_content(
1009
+ doc: PdfDocument,
1010
+ data: bytes,
1011
+ resources: dict,
1012
+ state: _PageState,
1013
+ ctm: tuple,
1014
+ depth: int,
1015
+ seen: set,
1016
+ ) -> None:
1017
+ fonts: dict[str, _Font] = {}
1018
+ font_dict = doc.as_dict(resources.get("Font")) or {}
1019
+ xobjects = doc.as_dict(resources.get("XObject")) or {}
1020
+
1021
+ def font_for(key: str) -> _Font:
1022
+ if key not in fonts:
1023
+ if key in font_dict:
1024
+ fonts[key] = _build_font(doc, key, font_dict[key])
1025
+ else:
1026
+ # A font the page never declared. Its codes mean nothing this file states.
1027
+ fonts[key] = _Font(key, "?", False, {}, {}, {}, 500.0)
1028
+ return fonts[key]
1029
+
1030
+ stack: list[tuple] = []
1031
+ tm = tlm = _IDENTITY
1032
+ font: _Font | None = None
1033
+ fsize = 0.0
1034
+ tc = tw = 0.0
1035
+ th = 1.0
1036
+ tl = 0.0
1037
+ rise = 0.0
1038
+ operands: list = []
1039
+
1040
+ def show(raw: bytes) -> None:
1041
+ nonlocal tm
1042
+ if font is None:
1043
+ return
1044
+ state.show_ops += 1
1045
+ placement = _mul(tm, ctm)
1046
+ trm = _mul((fsize * th, 0.0, 0.0, fsize, 0.0, rise), placement)
1047
+ scale = abs(trm[3]) or abs(fsize) or 1.0
1048
+ # The displacement below is computed in TEXT space; the run's extent on the page is
1049
+ # that displacement through the text and current transformation matrices. Using the
1050
+ # CTM alone under-measures every run a `Tm` scales, which shows up as spurious spaces
1051
+ # inside words ("Cus t omer") when the gap rule below compares extents.
1052
+ stretch = abs(placement[0]) or 1.0
1053
+ pieces: list[str] = []
1054
+ advance = 0.0
1055
+ for code in font.codes(raw):
1056
+ ch = font.char(code, state.permissive)
1057
+ if ch is None:
1058
+ state.unmapped += 1
1059
+ state.unmapped_fonts.add(font.basefont or font.name)
1060
+ else:
1061
+ state.vouched += 1
1062
+ pieces.append(ch)
1063
+ step = font.width(code) / 1000.0 * fsize + tc
1064
+ if code == 32 and not font.two_byte:
1065
+ step += tw
1066
+ advance += step * th
1067
+ text = "".join(pieces)
1068
+ if text.strip():
1069
+ x0, y0 = trm[4], trm[5]
1070
+ state.runs.append(
1071
+ _Run(y=y0, x=x0, end_x=x0 + advance * stretch, size=scale, text=text)
1072
+ )
1073
+ tm = _mul((1.0, 0.0, 0.0, 1.0, advance, 0.0), tm)
1074
+
1075
+ def adjust(amount: float) -> None:
1076
+ nonlocal tm
1077
+ tm = _mul((1.0, 0.0, 0.0, 1.0, -amount / 1000.0 * fsize * th, 0.0), tm)
1078
+
1079
+ def numbers(count: int) -> list[float]:
1080
+ got = [v for v in operands[-count:] if isinstance(v, (int, float))]
1081
+ return [float(v) for v in got] if len(got) == count else []
1082
+
1083
+ for token in _tokenize_content(data):
1084
+ if not isinstance(token, _Keyword):
1085
+ operands.append(token)
1086
+ if len(operands) > 64:
1087
+ del operands[:-32]
1088
+ continue
1089
+ op = str(token)
1090
+ if op == "q":
1091
+ stack.append(ctm)
1092
+ elif op == "Q":
1093
+ if stack:
1094
+ ctm = stack.pop()
1095
+ elif op == "cm":
1096
+ got = numbers(6)
1097
+ if got:
1098
+ ctm = _mul(tuple(got), ctm)
1099
+ elif op == "BT":
1100
+ tm = tlm = _IDENTITY
1101
+ elif op == "ET":
1102
+ tm = tlm = _IDENTITY
1103
+ elif op == "Tf":
1104
+ got = [v for v in operands if isinstance(v, Name)]
1105
+ size = numbers(1)
1106
+ if got:
1107
+ font = font_for(str(got[-1]))
1108
+ if size:
1109
+ fsize = size[0]
1110
+ elif op == "Td":
1111
+ got = numbers(2)
1112
+ if got:
1113
+ tlm = _mul((1.0, 0.0, 0.0, 1.0, got[0], got[1]), tlm)
1114
+ tm = tlm
1115
+ elif op == "TD":
1116
+ got = numbers(2)
1117
+ if got:
1118
+ tl = -got[1]
1119
+ tlm = _mul((1.0, 0.0, 0.0, 1.0, got[0], got[1]), tlm)
1120
+ tm = tlm
1121
+ elif op == "Tm":
1122
+ got = numbers(6)
1123
+ if got:
1124
+ tm = tlm = tuple(got)
1125
+ elif op == "T*":
1126
+ tlm = _mul((1.0, 0.0, 0.0, 1.0, 0.0, -tl), tlm)
1127
+ tm = tlm
1128
+ elif op == "TL":
1129
+ got = numbers(1)
1130
+ if got:
1131
+ tl = got[0]
1132
+ elif op == "Tc":
1133
+ got = numbers(1)
1134
+ if got:
1135
+ tc = got[0]
1136
+ elif op == "Tw":
1137
+ got = numbers(1)
1138
+ if got:
1139
+ tw = got[0]
1140
+ elif op == "Tz":
1141
+ got = numbers(1)
1142
+ if got:
1143
+ th = got[0] / 100.0
1144
+ elif op == "Ts":
1145
+ got = numbers(1)
1146
+ if got:
1147
+ rise = got[0]
1148
+ elif op == "Tj":
1149
+ if operands and isinstance(operands[-1], bytes):
1150
+ show(operands[-1])
1151
+ elif op == "'":
1152
+ tlm = _mul((1.0, 0.0, 0.0, 1.0, 0.0, -tl), tlm)
1153
+ tm = tlm
1154
+ if operands and isinstance(operands[-1], bytes):
1155
+ show(operands[-1])
1156
+ elif op == '"':
1157
+ got = [v for v in operands[-3:-1] if isinstance(v, (int, float))]
1158
+ if len(got) == 2:
1159
+ tw, tc = float(got[0]), float(got[1])
1160
+ tlm = _mul((1.0, 0.0, 0.0, 1.0, 0.0, -tl), tlm)
1161
+ tm = tlm
1162
+ if operands and isinstance(operands[-1], bytes):
1163
+ show(operands[-1])
1164
+ elif op == "TJ":
1165
+ array = operands[-1] if operands and isinstance(operands[-1], list) else []
1166
+ for item in array:
1167
+ if isinstance(item, bytes):
1168
+ show(item)
1169
+ elif isinstance(item, (int, float)):
1170
+ adjust(float(item))
1171
+ elif op == "INLINE_IMAGE":
1172
+ state.images += 1
1173
+ state.image_kinds.add("inline")
1174
+ elif op == "Do":
1175
+ got = [v for v in operands if isinstance(v, Name)]
1176
+ if got:
1177
+ _do_xobject(doc, str(got[-1]), xobjects, state, ctm, depth, seen)
1178
+ operands = []
1179
+
1180
+
1181
+ def _do_xobject(
1182
+ doc: PdfDocument, key: str, xobjects: dict, state: _PageState, ctm: tuple, depth: int, seen: set
1183
+ ) -> None:
1184
+ node = xobjects.get(key)
1185
+ resolved = doc.resolve(node)
1186
+ if not isinstance(resolved, Stream):
1187
+ return
1188
+ subtype = str(doc.resolve(resolved.d.get("Subtype")) or "")
1189
+ if subtype == "Image":
1190
+ state.images += 1
1191
+ size = doc.resolve(resolved.d.get("Length"))
1192
+ state.image_bytes += int(size) if isinstance(size, int) else len(resolved.raw)
1193
+ filters = doc.resolve(resolved.d.get("Filter"))
1194
+ for name in filters if isinstance(filters, list) else [filters]:
1195
+ state.image_kinds.add(str(doc.resolve(name) or "raw"))
1196
+ return
1197
+ if subtype != "Form" or depth >= 8:
1198
+ return
1199
+ marker = (node.num, node.gen) if isinstance(node, Ref) else id(resolved)
1200
+ if marker in seen:
1201
+ return
1202
+ seen.add(marker)
1203
+ try:
1204
+ data = doc.data_of(node)
1205
+ except _ImageData as image:
1206
+ state.images += 1
1207
+ state.image_kinds.add(image.filter_name)
1208
+ return
1209
+ except PdfError:
1210
+ return
1211
+ inner = doc.as_dict(resolved.d.get("Resources")) or {}
1212
+ matrix = doc.resolve(resolved.d.get("Matrix"))
1213
+ local = ctm
1214
+ if isinstance(matrix, list) and len(matrix) == 6:
1215
+ try:
1216
+ local = _mul(tuple(float(doc.resolve(v)) for v in matrix), ctm)
1217
+ except (TypeError, ValueError):
1218
+ local = ctm
1219
+ _run_content(doc, data, inner, state, local, depth + 1, seen)
1220
+ seen.discard(marker)
1221
+
1222
+
1223
+ def _rows_from_runs(runs: list[_Run], rotate: int) -> tuple[str, ...]:
1224
+ """Glyph runs, grouped into rows by baseline and ordered by x. A slicing unit, not layout."""
1225
+ if not runs:
1226
+ return ()
1227
+ if rotate % 360 in (90, 270):
1228
+ runs = [_Run(y=r.x, x=-r.y, end_x=-r.y + (r.end_x - r.x), size=r.size, text=r.text)
1229
+ for r in runs]
1230
+ runs.sort(key=lambda r: (-r.y, r.x))
1231
+ rows: list[list[_Run]] = []
1232
+ current: list[_Run] = []
1233
+ baseline = 0.0
1234
+ for run in runs:
1235
+ tolerance = max(1.5, 0.35 * run.size)
1236
+ if current and abs(run.y - baseline) <= tolerance:
1237
+ current.append(run)
1238
+ else:
1239
+ if current:
1240
+ rows.append(current)
1241
+ current = [run]
1242
+ baseline = run.y
1243
+ if current:
1244
+ rows.append(current)
1245
+ out: list[str] = []
1246
+ for row in rows:
1247
+ row.sort(key=lambda r: r.x)
1248
+ pieces: list[str] = []
1249
+ previous_end: float | None = None
1250
+ for run in row:
1251
+ if previous_end is not None and run.x - previous_end > 0.22 * max(run.size, 1.0):
1252
+ pieces.append(" ")
1253
+ pieces.append(run.text)
1254
+ previous_end = max(run.end_x, run.x)
1255
+ line = "".join(pieces).replace("\t", " ").replace("\r", " ").replace("\n", " ").strip()
1256
+ if line:
1257
+ out.append(line)
1258
+ return tuple(out)
1259
+
1260
+
1261
+ # ------------------------------------------------------------------------------- the API
1262
+
1263
+
1264
+ @dataclass(frozen=True)
1265
+ class PdfPage:
1266
+ """One page: its rows, and the counts behind everything the rows do not carry."""
1267
+
1268
+ number: int
1269
+ rows: tuple[str, ...]
1270
+ show_ops: int
1271
+ vouched: int
1272
+ unmapped: int
1273
+ unmapped_fonts: tuple[str, ...]
1274
+ images: int
1275
+ image_bytes: int
1276
+ image_kinds: tuple[str, ...]
1277
+
1278
+ @property
1279
+ def characters(self) -> int:
1280
+ return sum(len(row.strip()) for row in self.rows)
1281
+
1282
+
1283
+ @dataclass(frozen=True)
1284
+ class PdfText:
1285
+ version: str
1286
+ pages: tuple[PdfPage, ...]
1287
+
1288
+ @property
1289
+ def vouched(self) -> int:
1290
+ return sum(page.vouched for page in self.pages)
1291
+
1292
+ @property
1293
+ def unmapped(self) -> int:
1294
+ return sum(page.unmapped for page in self.pages)
1295
+
1296
+ @property
1297
+ def show_ops(self) -> int:
1298
+ return sum(page.show_ops for page in self.pages)
1299
+
1300
+ @property
1301
+ def images(self) -> int:
1302
+ return sum(page.images for page in self.pages)
1303
+
1304
+ @property
1305
+ def unmapped_fonts(self) -> tuple[str, ...]:
1306
+ out: set[str] = set()
1307
+ for page in self.pages:
1308
+ out.update(page.unmapped_fonts)
1309
+ return tuple(sorted(out))
1310
+
1311
+
1312
+ def read_pdf(path: str | Path, vouch: bool = True) -> PdfText:
1313
+ """Every page of `path`, as rows this reader can vouch for.
1314
+
1315
+ `vouch=False` renders the characters it CANNOT vouch for as well, mapping an unmapped code
1316
+ to the character its raw value would suggest. That output is mojibake by construction and
1317
+ is never what `docread` asks for — it exists so the audit in
1318
+ `.shiftwork/probes/J25-D3-pdf-coverage.py` can *demonstrate* what the guard is suppressing
1319
+ rather than assert it. Nothing in the runtime calls it with `vouch=False`.
1320
+ """
1321
+ path = Path(path)
1322
+ raw = path.read_bytes()
1323
+ header = _HEADER_RE.match(raw[:1024]) or _HEADER_RE.search(raw[:1024])
1324
+ if header is None:
1325
+ raise PdfError("it does not begin with a %PDF- header, so it is not a PDF")
1326
+ version = header.group(1).decode("ascii")
1327
+ doc = PdfDocument(raw)
1328
+ encryption = doc.encryption()
1329
+ if encryption is not None:
1330
+ handler = encryption.get("Filter", "?")
1331
+ raise PdfError(
1332
+ f"it is an ENCRYPTED PDF (/Encrypt, security handler /{handler}) and this reader "
1333
+ "decrypts nothing — no file on the corpus it was measured against was encrypted, "
1334
+ "so decryption here would be unmeasured code"
1335
+ )
1336
+ page_nodes = doc.pages()
1337
+ if not page_nodes:
1338
+ raise PdfError(
1339
+ f"its structure yielded no page at all: {len(doc.objects)} objects were parsed and "
1340
+ "none of them is a page tree or a /Type/Page"
1341
+ )
1342
+ pages: list[PdfPage] = []
1343
+ for index, (node, inherited) in enumerate(page_nodes):
1344
+ state = _PageState()
1345
+ resources = doc.as_dict(inherited.get("Resources", node.get("Resources"))) or {}
1346
+ for chunk in _content_chunks(doc, node):
1347
+ try:
1348
+ _run_content(doc, chunk, resources, state, _IDENTITY, 0, set())
1349
+ except PdfError:
1350
+ continue
1351
+ except RecursionError:
1352
+ continue
1353
+ rotate = doc.resolve(inherited.get("Rotate", node.get("Rotate", 0)))
1354
+ rows = _rows_from_runs(state.runs, int(rotate) if isinstance(rotate, int) else 0)
1355
+ if not vouch:
1356
+ rows = _unvouched_rows(doc, node, resources)
1357
+ pages.append(
1358
+ PdfPage(
1359
+ number=index + 1,
1360
+ rows=rows,
1361
+ show_ops=state.show_ops,
1362
+ vouched=state.vouched,
1363
+ unmapped=state.unmapped,
1364
+ unmapped_fonts=tuple(sorted(state.unmapped_fonts)),
1365
+ images=state.images,
1366
+ image_bytes=state.image_bytes,
1367
+ image_kinds=tuple(sorted(state.image_kinds)),
1368
+ )
1369
+ )
1370
+ return PdfText(version=version, pages=tuple(pages))
1371
+
1372
+
1373
+ def _content_chunks(doc: PdfDocument, node: dict) -> list[bytes]:
1374
+ contents = node.get("Contents")
1375
+ targets = contents if isinstance(doc.resolve(contents), list) else [contents]
1376
+ if isinstance(doc.resolve(contents), list):
1377
+ targets = doc.resolve(contents)
1378
+ out: list[bytes] = []
1379
+ for target in targets:
1380
+ try:
1381
+ out.append(doc.data_of(target))
1382
+ except (PdfError, _ImageData):
1383
+ continue
1384
+ return out
1385
+
1386
+
1387
+ def _unvouched_rows(doc: PdfDocument, node: dict, resources: dict) -> tuple[str, ...]:
1388
+ """What the page WOULD render if every code were decoded regardless of provenance.
1389
+
1390
+ Audit-only, and deliberately crude: an unmapped code becomes `chr(code)`, which is exactly
1391
+ the assumption a reader makes when it decodes a subset font's glyph indices as if they were
1392
+ characters. `read_pdf(vouch=False)` is the only caller, and nothing in the runtime calls
1393
+ that. It exists so the mojibake guard can be DEMONSTRATED rather than asserted.
1394
+ """
1395
+ audit = _PageState(permissive=True)
1396
+ for chunk in _content_chunks(doc, node):
1397
+ try:
1398
+ _run_content(doc, chunk, resources, audit, _IDENTITY, 0, set())
1399
+ except (PdfError, RecursionError):
1400
+ continue
1401
+ rotate = doc.resolve(node.get("Rotate", 0))
1402
+ return _rows_from_runs(audit.runs, int(rotate) if isinstance(rotate, int) else 0)