glyphive 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. glyphive-0.1.0/.gitignore +32 -0
  2. glyphive-0.1.0/CHANGELOG.md +250 -0
  3. glyphive-0.1.0/CONTRIBUTING.md +54 -0
  4. glyphive-0.1.0/LICENSE +21 -0
  5. glyphive-0.1.0/PKG-INFO +304 -0
  6. glyphive-0.1.0/README.md +230 -0
  7. glyphive-0.1.0/RELEASENOTES.md +86 -0
  8. glyphive-0.1.0/THIRD_PARTY_LICENSES.md +32 -0
  9. glyphive-0.1.0/docs/api/archive.md +6 -0
  10. glyphive-0.1.0/docs/api/codec.md +10 -0
  11. glyphive-0.1.0/docs/api/compression.md +6 -0
  12. glyphive-0.1.0/docs/api/layout.md +6 -0
  13. glyphive-0.1.0/docs/api/ocr.md +6 -0
  14. glyphive-0.1.0/docs/api/overview.md +35 -0
  15. glyphive-0.1.0/docs/api/render.md +6 -0
  16. glyphive-0.1.0/docs/api/restore.md +6 -0
  17. glyphive-0.1.0/docs/benchmarks.md +180 -0
  18. glyphive-0.1.0/docs/changelog.md +3 -0
  19. glyphive-0.1.0/docs/guides/create.md +229 -0
  20. glyphive-0.1.0/docs/guides/ocr.md +154 -0
  21. glyphive-0.1.0/docs/guides/restore.md +182 -0
  22. glyphive-0.1.0/docs/guides/wire-format.md +156 -0
  23. glyphive-0.1.0/docs/index.md +55 -0
  24. glyphive-0.1.0/examples/create_restore.py +50 -0
  25. glyphive-0.1.0/examples/gallery/docx-consolas-10pt.docx +0 -0
  26. glyphive-0.1.0/examples/gallery/docx-none-compression.docx +0 -0
  27. glyphive-0.1.0/examples/gallery/manifest.md +22 -0
  28. glyphive-0.1.0/examples/gallery/pdf-centered-spaced.pdf +0 -0
  29. glyphive-0.1.0/examples/gallery/pdf-courier-11pt.pdf +0 -0
  30. glyphive-0.1.0/examples/gallery/pdf-courier-8pt-safe.pdf +0 -0
  31. glyphive-0.1.0/examples/gallery/pdf-minimal-margins.pdf +0 -0
  32. glyphive-0.1.0/examples/gallery/pdf-ocrb-6pt-dense.pdf +0 -0
  33. glyphive-0.1.0/examples/gallery/pdf-pinned-line-width.pdf +0 -0
  34. glyphive-0.1.0/examples/gallery/pdf-simple-low-redundancy.pdf +0 -0
  35. glyphive-0.1.0/examples/gallery/text-default.txt +408 -0
  36. glyphive-0.1.0/examples/gallery/text-none.txt +978 -0
  37. glyphive-0.1.0/examples/gallery.py +324 -0
  38. glyphive-0.1.0/mkdocs.yml +62 -0
  39. glyphive-0.1.0/package.py +254 -0
  40. glyphive-0.1.0/pyproject.toml +86 -0
  41. glyphive-0.1.0/src/glyphive/__init__.py +23 -0
  42. glyphive-0.1.0/src/glyphive/__main__.py +6 -0
  43. glyphive-0.1.0/src/glyphive/archive.py +670 -0
  44. glyphive-0.1.0/src/glyphive/assets/__init__.py +1 -0
  45. glyphive-0.1.0/src/glyphive/assets/fonts/__init__.py +1 -0
  46. glyphive-0.1.0/src/glyphive/assets/fonts/dejavu_sans_mono/DejaVuSansMono.ttf +0 -0
  47. glyphive-0.1.0/src/glyphive/assets/fonts/dejavu_sans_mono/LICENSE +187 -0
  48. glyphive-0.1.0/src/glyphive/assets/fonts/dejavu_sans_mono/__init__.py +1 -0
  49. glyphive-0.1.0/src/glyphive/assets/fonts/ocr_b/LICENSE.md +94 -0
  50. glyphive-0.1.0/src/glyphive/assets/fonts/ocr_b/OCR-B.ttf +0 -0
  51. glyphive-0.1.0/src/glyphive/assets/fonts/ocr_b/__init__.py +1 -0
  52. glyphive-0.1.0/src/glyphive/cli/__init__.py +68 -0
  53. glyphive-0.1.0/src/glyphive/cli/_common.py +411 -0
  54. glyphive-0.1.0/src/glyphive/cli/create.py +462 -0
  55. glyphive-0.1.0/src/glyphive/cli/extract.py +148 -0
  56. glyphive-0.1.0/src/glyphive/cli/inspect.py +161 -0
  57. glyphive-0.1.0/src/glyphive/cli/list.py +124 -0
  58. glyphive-0.1.0/src/glyphive/codec/__init__.py +37 -0
  59. glyphive-0.1.0/src/glyphive/codec/_base.py +101 -0
  60. glyphive-0.1.0/src/glyphive/codec/base16c.py +1335 -0
  61. glyphive-0.1.0/src/glyphive/codec/pagers.py +121 -0
  62. glyphive-0.1.0/src/glyphive/codec/radix.py +96 -0
  63. glyphive-0.1.0/src/glyphive/compression/__init__.py +40 -0
  64. glyphive-0.1.0/src/glyphive/compression/_base.py +141 -0
  65. glyphive-0.1.0/src/glyphive/compression/gzip.py +39 -0
  66. glyphive-0.1.0/src/glyphive/compression/none.py +24 -0
  67. glyphive-0.1.0/src/glyphive/compression/zstd.py +69 -0
  68. glyphive-0.1.0/src/glyphive/layout.py +1219 -0
  69. glyphive-0.1.0/src/glyphive/plugins.py +142 -0
  70. glyphive-0.1.0/src/glyphive/py.typed +1 -0
  71. glyphive-0.1.0/src/glyphive/render/__init__.py +91 -0
  72. glyphive-0.1.0/src/glyphive/render/_base.py +133 -0
  73. glyphive-0.1.0/src/glyphive/render/formats/__init__.py +15 -0
  74. glyphive-0.1.0/src/glyphive/render/formats/docx.py +99 -0
  75. glyphive-0.1.0/src/glyphive/render/formats/pdf.py +405 -0
  76. glyphive-0.1.0/src/glyphive/render/formats/qr.py +172 -0
  77. glyphive-0.1.0/src/glyphive/render/formats/text.py +39 -0
  78. glyphive-0.1.0/src/glyphive/restore/__init__.py +19 -0
  79. glyphive-0.1.0/src/glyphive/restore/decode.py +258 -0
  80. glyphive-0.1.0/src/glyphive/restore/document_images.py +154 -0
  81. glyphive-0.1.0/src/glyphive/restore/ocr/__init__.py +122 -0
  82. glyphive-0.1.0/src/glyphive/restore/ocr/_base.py +82 -0
  83. glyphive-0.1.0/src/glyphive/restore/ocr/providers/__init__.py +12 -0
  84. glyphive-0.1.0/src/glyphive/restore/ocr/providers/_image.py +19 -0
  85. glyphive-0.1.0/src/glyphive/restore/ocr/providers/easyocr.py +32 -0
  86. glyphive-0.1.0/src/glyphive/restore/ocr/providers/paddle.py +40 -0
  87. glyphive-0.1.0/src/glyphive/restore/ocr/providers/tesseract.py +58 -0
  88. glyphive-0.1.0/src/glyphive/restore/qr.py +223 -0
  89. glyphive-0.1.0/src/glyphive/restore/unarchive.py +452 -0
  90. glyphive-0.1.0/tests/test_archive_roundtrip.py +234 -0
  91. glyphive-0.1.0/tests/test_cli.py +901 -0
  92. glyphive-0.1.0/tests/test_codec.py +583 -0
  93. glyphive-0.1.0/tests/test_compression.py +91 -0
  94. glyphive-0.1.0/tests/test_document_inputs.py +190 -0
  95. glyphive-0.1.0/tests/test_ignore.py +47 -0
  96. glyphive-0.1.0/tests/test_layout_metadata.py +437 -0
  97. glyphive-0.1.0/tests/test_metadata_permissions.py +153 -0
  98. glyphive-0.1.0/tests/test_ocr_registry.py +67 -0
  99. glyphive-0.1.0/tests/test_ocr_transcripts.py +188 -0
  100. glyphive-0.1.0/tests/test_package.py +146 -0
  101. glyphive-0.1.0/tests/test_pagers.py +50 -0
  102. glyphive-0.1.0/tests/test_plugins.py +170 -0
  103. glyphive-0.1.0/tests/test_qr.py +118 -0
  104. glyphive-0.1.0/tests/test_renderers.py +404 -0
  105. glyphive-0.1.0/tests/test_streaming_archive.py +89 -0
  106. glyphive-0.1.0/tests/test_streaming_memory.py +74 -0
  107. glyphive-0.1.0/tests/test_streaming_restore.py +388 -0
  108. glyphive-0.1.0/tools/document_to_images.py +29 -0
  109. glyphive-0.1.0/tools/ocr_font_report.py +909 -0
@@ -0,0 +1,32 @@
1
+ # Agent config must never be committed (global rule)
2
+ AGENTS.md
3
+ .agents/
4
+ CLAUDE*
5
+ .claude
6
+
7
+ # Python
8
+ __pycache__/
9
+ *.py[cod]
10
+ *.egg-info/
11
+ .eggs/
12
+ build/
13
+ dist/
14
+ site/
15
+ .venv/
16
+ .pytest_cache/
17
+ .hypothesis/
18
+ .mypy_cache/
19
+ .ruff_cache/
20
+ .coverage*
21
+ htmlcov/
22
+
23
+ # Local scratch / tool output
24
+ run.log
25
+ tmp/
26
+
27
+ # OS
28
+ .DS_Store
29
+ Thumbs.db
30
+ .idea/
31
+ .vscode/
32
+ *.traineddata
@@ -0,0 +1,250 @@
1
+ # Changelog
2
+
3
+ All notable changes to this project will be documented in this file.
4
+
5
+ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
6
+ and this project adheres to [Semantic Versioning](https://semver.org/).
7
+
8
+ ## [Unreleased]
9
+
10
+ ## [0.1.0] - 2026-07-18
11
+
12
+ The first public release provides an end-to-end path from a file tree to
13
+ OCR-friendly printable pages and back to a verified tree.
14
+
15
+ ### Changed
16
+
17
+ - **Compact `#!glyphive` header line**: the display-only summary is now
18
+ `#!glyphive v<N> <codec>[,<comp>] files=.. bytes=.. pages=..` — a bare `v<N>`
19
+ token, codec and compression collapsed to one positional `codec[,comp]` token,
20
+ and `sha256`/`meta` dropped entirely (they live in the protected `H` frames).
21
+ Fewer characters on page 1 means less to OCR and less display-line overflow.
22
+ Any line beginning with `#!` is now treated as a comment on the read path, so
23
+ documents can carry arbitrary `#!` notes. (Pre-release format change; no
24
+ backward compatibility with the old header grammar.)
25
+ - **`--descan auto` retry now sweeps a `0.6`/`0.8` blur ladder** (was a single
26
+ `0.6` pass): the widest glyphs (e.g. Courier 12 pt) can need `~0.8` to decode
27
+ from a raw photo, so the auto-retry covers both. The per-line CRC merge keeps
28
+ this strictly additive — extra blur passes only ever recover more lines.
29
+ - **Default PDF font is now `dejavu-sans-mono`** (was Courier). DejaVu was one of
30
+ only two fonts that held up on real photographed scans under `tesseract-glyphive`
31
+ and passes the byte-for-byte restore gate, so it is preferred for recovery
32
+ robustness. Trade-offs, accepted deliberately: it renders ~26% fewer usable
33
+ bytes/page than Courier at 8 pt+ (more pages per document) and embeds ~340 KB per
34
+ PDF (Courier is a zero-embedding core font). Pass `--font courier` for the denser,
35
+ zero-embed alternative. The synthetic density comparison and the override rationale
36
+ are in `benchmarks/results/FONT_CANDIDATES.md`.
37
+ - **Footer-hash mismatches are now advisory, not warnings**: an OCR restore
38
+ almost always produces a footer-hash mismatch (OCR inserts interior spaces
39
+ that change the page text hash while the `L`/`P` lines still decode via
40
+ CRC/RS), so logging it at WARNING cried wolf on every clean restore. These are
41
+ now collected separately (`meta["_footer_hash_notes"]`) and logged at INFO
42
+ (shown with `-v`); genuine page-integrity issues (missing/reconstructed pages)
43
+ stay at WARNING. `glyphive inspect` reports the advisory count. Correctness is
44
+ unchanged — it never rested on the page footer hash.
45
+ - **Faster decode on clean input**: decode now skips Reed-Solomon entirely on
46
+ erasure-free blocks (the common case — text transcripts and good scans are
47
+ mostly clean), since a line whose per-line CRC matched is already trusted and
48
+ RS has nothing to correct. A partially-damaged stream RS-corrects only the
49
+ blocks that actually contain a bad/missing line. Correctness is unchanged: the
50
+ per-line CRC oracle plus the whole-document SHA-256 gate still catch any
51
+ corruption loudly. (Profiling put RS at ~75% of clean-decode time; the exact
52
+ speedup is measured in CI, not stated here.)
53
+ - **Codec identifier renamed `g1` -> `base16c-crc16-rs`**: exposes the
54
+ composable parts (16-char OCR-safe alphabet / CRC-16 / Reed-Solomon) instead
55
+ of an opaque version tag. `codec/g1.py` -> `codec/base16c.py`; `G1Codec` ->
56
+ `Base16CCodec`; `--codec` default and `codec.names()` updated. Nothing was
57
+ published under the old name, so this is an in-place rename with no
58
+ migration path or dual-format compatibility.
59
+ - **Dense preset documented**: the bundled `ocr-b` font (SIL OFL 1.1, OCR-B by
60
+ Raisty) at 6pt measures 5,050 usable bytes/page — 23% denser than the
61
+ Courier 8pt `safe` default — and remains safe on both tested engines
62
+ (Tesseract and PaddleOCR). Select it with `--font ocr-b --font-size 6`; the
63
+ shipped default stays Courier 8pt per the project's font-selection design.
64
+
65
+ ### Fixed
66
+
67
+ - **PDF frames no longer silently shrink to fit**: a machine/data frame
68
+ (`H`/`L`/`P`/`T`) that overflows the printable width now raises a clear error
69
+ naming the overflow instead of quietly reducing its font size to fit.
70
+ Silent shrinking distorted glyphs (hurting OCR) and hid a misconfigured
71
+ font/size/margin/line-width. The display-only `#!glyphive` human header is
72
+ exempt and still scales to fit (restore never trusts it).
73
+ - **Overwrite publication now rolls back on partial failure**: restoring onto
74
+ an existing destination with `--overwrite` moves each existing final file
75
+ aside into a private backup before replacing it. If any staged move fails
76
+ partway through (disk full, permission error, etc.), every file already
77
+ replaced is restored from its backup and any newly created file is removed
78
+ -- the destination is left exactly as it was, never half-migrated.
79
+ - **Machine header now Reed-Solomon protected**: the real Tesseract 5.4.0
80
+ end-to-end gate (`create` -> rasterize 300 DPI -> `extract --from-images`
81
+ -> diff) found that `--compression zstd` deterministically failed restore:
82
+ a wider header caused Tesseract to misread one `H` frame's two duplicate
83
+ copies identically, so CRC+duplication alone could not recover it.
84
+ `layout.py` now adds one Reed-Solomon parity chunk over the header
85
+ envelope so restore reconstructs a single damaged chunk instead of only
86
+ detecting it; corruption spanning more than one distinct chunk still fails
87
+ loud. Both `--compression none` and `--compression zstd` now restore
88
+ byte-for-byte through the real OCR gate.
89
+ - **Large-document parity overhead**: `parity_ratio` now targets aggregate
90
+ Reed-Solomon parity across all GF(255) blocks. The default is approximately
91
+ 12% for large streams instead of repeating a capped whole-stream budget per
92
+ block and inflating parity to roughly 65%.
93
+
94
+ ### Added
95
+
96
+ - **`create --no-header`**: omit the display-only `#!glyphive` summary line from
97
+ page 1 for the tightest possible page. Restore needs nothing from it (all
98
+ authoritative metadata comes from the CRC-protected `H` frames), so a
99
+ `--no-header` document restores byte-for-byte identically.
100
+ - **Bundled `dejavu-sans-mono` PDF font**: DejaVu Sans Mono 2.37 (permissive
101
+ DejaVu Fonts License) is now a selectable bundled font
102
+ (`--font dejavu-sans-mono`), embedded in PDF output. It was one of only two
103
+ fonts (with Courier) that held up under the `tesseract-glyphive` profile in
104
+ real scan/restore testing — offered as a strong option; a byte-for-byte
105
+ restore gate is pending before it is recommended over Courier. Adds ~335 KB
106
+ to the wheel. (Also fixes a latent bug where a second bundled font would have
107
+ collided on the hardcoded `"OCR-B"` FPDF family name.)
108
+ - **`glyphive inspect` — recovery-headroom report**: a read-only subcommand
109
+ that reports how much damage a document can survive without fully decoding or
110
+ verifying it (so it works on a partially damaged scan a restore would reject,
111
+ and writes nothing). Shows data/parity page counts (whole-page recovery
112
+ budget), the realized per-line Reed-Solomon `nsym` (scattered-damage budget),
113
+ and which pages are present/missing/reconstructable. `--json` for machine
114
+ output; `--strict` exits non-zero on an already-unrecoverable document. Backed
115
+ by a new pure `codec.base16c.describe_line_stream` shape oracle.
116
+ - **`--line-width auto|max|<int>` spellings** on `create`: `auto` (default) is
117
+ the OCR-measured-safe capacity (≤60); `max` fills the largest row that
118
+ physically fits the font/size/margins (may exceed 60, not OCR-verified, and
119
+ an error on formats without font metrics); an integer above the safe cap now
120
+ requires `--force`. The renderer interface gains a public
121
+ `geometric_payload_capacity` hook (uncapped fit) alongside the safety-capped
122
+ `payload_capacity`.
123
+ - **De-scan blur for photographed input (`--descan`, auto by default)**:
124
+ `extract`/`list` apply a Gaussian blur to image and rasterized-PDF input
125
+ before OCR. The default `--descan auto` does a single sharp pass, then
126
+ automatically retries once with a light `0.6` blur if that fails to decode
127
+ image/PDF input — raw phone photos are often too sharp/noisy for the frame
128
+ CRC/RS and otherwise fail, and the retry costs an extra OCR pass only on
129
+ failure. `--descan 0` disables the auto-retry; an explicit list
130
+ (`--descan 0,0.6,1.0`) OCRs each image at every radius and **merges the
131
+ CRC-valid lines across passes** — different blurs recover different lines and
132
+ the per-line CRC makes combining them safe, so a document no single blur can
133
+ fully read may still restore from the union. Automatic OCR-engine selection
134
+ also prefers the constrained `tesseract-glyphive` profile over plain
135
+ `tesseract` (measured substantially higher scan-restore success).
136
+ - **Whole-page recovery (`--parity-pages K`)**: `create` can emit K extra
137
+ document-level Reed-Solomon parity pages over the data pages, so a document
138
+ survives up to K wholly lost / unscannable / destroyed pages — not just the
139
+ scattered per-line OCR errors the per-line RS already fixes. Restore
140
+ reconstructs missing data pages from the parity pages and reports which it
141
+ rebuilt. Default 0 (off, byte-identical to before). Data pages + K must not
142
+ exceed 255. Independent of `--parity-ratio`. The header gains a `pgpar`
143
+ token (omitted when 0) and the protected machine envelope records K and the
144
+ page block size. Separately, a *missing* page no longer hard-fails up front
145
+ even with K=0: the codec's document-wide RS is given a chance to recover it
146
+ from the surviving pages when the per-line parity budget allows.
147
+ - **Progress logging for `create`/`extract`**: both commands now log
148
+ intermediate stage events (`archived`, `compressed`, `encoded`, `rendered`
149
+ for `create`; `staged`, `published` for `extract`) instead of only a final
150
+ one-line summary, sparsely rate-limited so a large tree doesn't flood the
151
+ log. The underlying `on_progress` callback is also available to library
152
+ callers of `restore_document_spooled`/`unarchive_spool`.
153
+ - **Isolated QR transport primitives**: the optional `qr` extra provides
154
+ deterministic, versioned 1000-byte envelopes, level-H Segno PNG generation,
155
+ and Pillow/ZXing-C++ raw-byte decoding without OpenCV. Extract/list accept
156
+ explicit `--from-qr` image or directory input; ordinary image OCR behavior is
157
+ unchanged. Explicit `--format qr` writes six symbols per Letter page;
158
+ `--format hybrid` writes one symbol plus its transcript slice per page. A
159
+ plain `.pdf` suffix continues to select the ordinary PDF renderer.
160
+ - **Constrained Tesseract profile** (`tesseract-glyphive`): an opt-in OCR
161
+ provider using PSM 6, Glyphive's exact machine alphabet, and disabled general
162
+ language dictionaries. The existing `tesseract` provider remains unchanged.
163
+ - **Bounded archive and compression primitives**: archive records can now be
164
+ written and parsed as fixed-size chunks, and the built-in none/gzip/zstd
165
+ methods support binary stream adapters. Existing one-shot APIs remain
166
+ available while create/restore migrate to disk-backed spools. Create now
167
+ streams archive, compression, FEC, pagination, and renderer output through
168
+ temporary spools; `--temp-dir` and `--chunk-size` control spool placement and
169
+ I/O granularity. Restore stream-decompresses into a size-limited spool,
170
+ validates the global digest and archive framing, stages files privately, and
171
+ only then publishes final paths; `--max-output-bytes` caps expansion.
172
+ Transcript parsing spools normalized frames in one pass, while
173
+ `base16c-crc16-rs` retains only compact line offsets and processes
174
+ Reed-Solomon codewords blockwise.
175
+ - **Explicit plugin discovery** (`glyphive.plugins`): trusted installed
176
+ distributions can provide typed codecs, compression methods, render formats,
177
+ or OCR providers through four documented entry-point groups. Discovery is
178
+ opt-in through the library API or global `--plugins` CLI flag, deterministic,
179
+ cached, and reports broken candidates without changing normal imports.
180
+ - **Standalone zipapp packaging** (`package.py`): build a universal
181
+ `glyphive.pyz` containing the required runtime dependencies and the core
182
+ text/none/gzip feature set, or explicitly named platform-specific artifacts
183
+ with optional integrations. Lightweight OCR Python shims are optional;
184
+ heavyweight OCR engines and models remain external.
185
+ - **Codec registry** (`glyphive.codec`): typed named lookup with the built-in
186
+ `base16c-crc16-rs` codec.
187
+ - **Compression registry** (`glyphive.compression`): named `none`, `gzip`, and
188
+ lazy optional `zstd` methods.
189
+ - **Renderer and OCR registries** (`glyphive.render`,
190
+ `glyphive.restore.ocr`): text, PDF, Word, and optional OCR providers use
191
+ explicit registries with lazy backend imports.
192
+ - **Codec `base16c-crc16-rs`**: the measured-safe `ABCDHKLMPRTVXY34` alphabet
193
+ (16 symbols, 4-bit packing), a full CRC-16 per line, masked five-character
194
+ indices, and document-wide interleaved Reed-Solomon parity. Scattered OCR
195
+ errors can self-heal; correctness is judged by CRC and parity rather than
196
+ speculative character substitution. (Named for its composable parts —
197
+ 16-symbol OCR-safe alphabet / CRC-16 / Reed-Solomon — instead of an opaque
198
+ `g1` version tag; renamed before the first release, so no migration shim
199
+ was needed.)
200
+ - **Binary-safe archive stream** (`glyphive.archive`): length-prefixed records
201
+ for arbitrary bytes, deterministic ordering, root-level
202
+ `.gitignore`/`.ignore` filtering, empty-directory records, and `none`, `gzip`,
203
+ or `zstd` whole-stream compression.
204
+ - **Protected layout metadata** (`glyphive.layout`): safe-alphabet `H` header
205
+ frames and `T` page-footer frames carry authoritative document metadata,
206
+ page identities, and hashes. Unrestricted human-readable summaries are
207
+ display aids only.
208
+ - **Text, PDF, and Word renderers** with selectable font family and size. PDF
209
+ output uses built-in FPDF font families; Word output accepts installed Word
210
+ font names.
211
+ - **Verified restore pipeline** (`glyphive.restore`): transcript decode,
212
+ whole-document SHA-256 validation, path-traversal-safe extraction, and an
213
+ optional multi-provider OCR layer for image input.
214
+ - **Tar-like CLI**: `create` (`c`), `extract` (`x`), and `list` (`t`) commands
215
+ with codec, compression, metadata, renderer, and OCR selectors. Leading
216
+ `-c`, `-x`, and `-t` mode flags work without a positional command.
217
+ - **Automatic document input**: `extract` and `list` classify each direct-child
218
+ input independently by magic bytes, extension, then UTF-8 text, so transcript,
219
+ image, PDF, and DOCX pages can be mixed in deterministic order. A conversion
220
+ helper renders PDF pages or diagnostic DOCX transcript pages to PNG with
221
+ configurable DPI and blur; DOCX restore uses `python-docx`, not an office suite.
222
+ - **Capacity-aware rendering**: PDF/DOCX row budgets follow font size and page
223
+ margins; `--minimal-margins` uses a compact 12-point profile, and long PDF
224
+ display headers fit rather than clip. Output format is inferred from `-f`
225
+ when `--format` is omitted.
226
+ - **Bundled OCR-B PDF font**: `--font ocr-b` embeds a pinned, unmodified
227
+ SIL-OFL-1.1 font; PDF output also accepts explicit `.ttf`/`.otf` paths.
228
+ - **Documentation and examples**: task-focused create, restore, wire-format,
229
+ OCR, benchmark, and API pages plus a runnable create/restore example.
230
+
231
+ ### Changed
232
+
233
+ - Require `pathlib_next>=0.8.1`, including its Python 3.9 import and local path
234
+ walking fixes.
235
+ - A directory supplied to CLI `-f` expands its direct child files in stable
236
+ filename order.
237
+
238
+ ### Known limitations
239
+
240
+ - `create` archives one directory (or `.`) at a time; wrap multiple inputs in a
241
+ directory.
242
+ - Ignore files are read at the archived-tree root only; nested `.gitignore`
243
+ files are not applied.
244
+ - A fully lost or unscannable page is detected and reported, not reconstructed.
245
+ - Protected header/footer metadata detects corruption but is not itself
246
+ Reed-Solomon-corrected.
247
+ - QR-code output is not implemented.
248
+
249
+ [Unreleased]: https://github.com/jose-pr/glyphive/compare/v0.1.0...HEAD
250
+ [0.1.0]: https://github.com/jose-pr/glyphive/releases/tag/v0.1.0
@@ -0,0 +1,54 @@
1
+ # Contributing to Glyphive
2
+
3
+ Thanks for helping improve Glyphive. The project is alpha: wire-format and CLI
4
+ changes are welcome when they include tests and clear documentation.
5
+
6
+ ## Development setup
7
+
8
+ ```bash
9
+ git clone https://github.com/jose-pr/glyphive.git
10
+ cd glyphive
11
+ python -m venv .venv/dev
12
+ python -m pip install -e ".[all,dev,docs]"
13
+ ```
14
+
15
+ Activate the environment using the command appropriate for your shell, or call
16
+ its Python executable directly.
17
+
18
+ ## Checks
19
+
20
+ Run the automated suite and strict documentation build before opening a pull
21
+ request:
22
+
23
+ ```bash
24
+ python -m pytest -q
25
+ python -m mkdocs build --strict
26
+ ```
27
+
28
+ Core timing checks are separate from correctness tests:
29
+
30
+ ```bash
31
+ python benchmarks/run.py
32
+ ```
33
+
34
+ Local timings are sanity checks, not publishable performance evidence. Keep
35
+ performance claims tied to comparable CI results. OCR sweeps and other
36
+ compute-heavy experiments should run on an appropriate remote machine and must
37
+ record engine, model, font, size, DPI, platform, and corpus provenance.
38
+
39
+ ## Changes and commits
40
+
41
+ - Add focused tests for behavior changes.
42
+ - Update the relevant guide and `CHANGELOG.md` for user-visible changes.
43
+ - Do not commit generated `site/`, `dist/`, temporary OCR output, or agent files.
44
+ - Use `type: description` commits, such as `feat:`, `fix:`, `docs:`, `test:`,
45
+ or `chore:`.
46
+ - Keep unrelated changes in separate commits.
47
+
48
+ ## Pull requests and bug reports
49
+
50
+ Open a focused pull request describing the behavior, its tests, and any wire or
51
+ density impact. Bug reports should include the Glyphive and Python versions, the
52
+ operating system, the exact command, expected and observed behavior, and a small
53
+ reproducer when possible. Remove private archive contents from transcripts and
54
+ images before attaching them publicly.
glyphive-0.1.0/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Jose A.
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,304 @@
1
+ Metadata-Version: 2.4
2
+ Name: glyphive
3
+ Version: 0.1.0
4
+ Summary: Archive file trees to compact, OCR-friendly printable pages (text/PDF/docx) and restore them from scans.
5
+ Project-URL: Homepage, https://github.com/jose-pr/glyphive/
6
+ Project-URL: Documentation, https://jose-pr.github.io/glyphive/
7
+ Project-URL: Issues, https://github.com/jose-pr/glyphive/issues
8
+ Project-URL: Repository, https://github.com/jose-pr/glyphive
9
+ Author: Jose A.
10
+ License-Expression: MIT
11
+ License-File: LICENSE
12
+ License-File: src/glyphive/assets/fonts/dejavu_sans_mono/LICENSE
13
+ License-File: src/glyphive/assets/fonts/ocr_b/LICENSE.md
14
+ Keywords: archive,backup,ocr,paper-backup,print,reed-solomon
15
+ Classifier: Development Status :: 3 - Alpha
16
+ Classifier: Environment :: Console
17
+ Classifier: Intended Audience :: Developers
18
+ Classifier: Intended Audience :: End Users/Desktop
19
+ Classifier: Operating System :: OS Independent
20
+ Classifier: Programming Language :: Python :: 3
21
+ Classifier: Programming Language :: Python :: 3.9
22
+ Classifier: Programming Language :: Python :: 3.10
23
+ Classifier: Programming Language :: Python :: 3.11
24
+ Classifier: Programming Language :: Python :: 3.12
25
+ Classifier: Programming Language :: Python :: 3.13
26
+ Classifier: Programming Language :: Python :: 3.14
27
+ Classifier: Topic :: System :: Archiving :: Backup
28
+ Classifier: Topic :: Utilities
29
+ Classifier: Typing :: Typed
30
+ Requires-Python: >=3.9
31
+ Requires-Dist: duho>=0.2.0
32
+ Requires-Dist: pathlib-next>=0.8.1
33
+ Requires-Dist: pathspec
34
+ Requires-Dist: reedsolo
35
+ Provides-Extra: all
36
+ Requires-Dist: fpdf2; extra == 'all'
37
+ Requires-Dist: pillow; extra == 'all'
38
+ Requires-Dist: pypdfium2; extra == 'all'
39
+ Requires-Dist: pytesseract; extra == 'all'
40
+ Requires-Dist: python-docx; extra == 'all'
41
+ Requires-Dist: segno==1.6.6; extra == 'all'
42
+ Requires-Dist: zstandard; extra == 'all'
43
+ Requires-Dist: zxing-cpp==2.3.0; (python_version < '3.10') and extra == 'all'
44
+ Requires-Dist: zxing-cpp==3.1.0; (python_version >= '3.10') and extra == 'all'
45
+ Provides-Extra: dev
46
+ Requires-Dist: build; extra == 'dev'
47
+ Requires-Dist: hatchling; extra == 'dev'
48
+ Requires-Dist: pytest; extra == 'dev'
49
+ Requires-Dist: pytest-cov; extra == 'dev'
50
+ Requires-Dist: twine; extra == 'dev'
51
+ Provides-Extra: docs
52
+ Requires-Dist: mkdocs; extra == 'docs'
53
+ Requires-Dist: mkdocs-material; extra == 'docs'
54
+ Requires-Dist: mkdocstrings[python]; extra == 'docs'
55
+ Provides-Extra: document-input
56
+ Requires-Dist: pillow; extra == 'document-input'
57
+ Requires-Dist: pypdfium2; extra == 'document-input'
58
+ Requires-Dist: python-docx; extra == 'document-input'
59
+ Provides-Extra: docx
60
+ Requires-Dist: python-docx; extra == 'docx'
61
+ Provides-Extra: ocr
62
+ Requires-Dist: pillow; extra == 'ocr'
63
+ Requires-Dist: pytesseract; extra == 'ocr'
64
+ Provides-Extra: pdf
65
+ Requires-Dist: fpdf2; extra == 'pdf'
66
+ Provides-Extra: qr
67
+ Requires-Dist: pillow; extra == 'qr'
68
+ Requires-Dist: segno==1.6.6; extra == 'qr'
69
+ Requires-Dist: zxing-cpp==2.3.0; (python_version < '3.10') and extra == 'qr'
70
+ Requires-Dist: zxing-cpp==3.1.0; (python_version >= '3.10') and extra == 'qr'
71
+ Provides-Extra: zstd
72
+ Requires-Dist: zstandard; extra == 'zstd'
73
+ Description-Content-Type: text/markdown
74
+
75
+ # glyphive
76
+
77
+ [![Status: alpha](https://img.shields.io/badge/status-alpha-orange.svg)](CHANGELOG.md)
78
+ [![Python 3.9+](https://img.shields.io/badge/python-3.9%2B-blue.svg)](pyproject.toml)
79
+ [![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE)
80
+ [![Docs](https://img.shields.io/badge/docs-latest-blue.svg)](https://jose-pr.github.io/glyphive/)
81
+ [![CI](https://img.shields.io/github/actions/workflow/status/jose-pr/glyphive/test.yml)](https://github.com/jose-pr/glyphive/actions/workflows/test.yml)
82
+
83
+ Archive a file tree to **compact, OCR-friendly, printable pages** and restore
84
+ it from a transcript or scan. Glyphive keeps the paper representation
85
+ human-readable while using checksums and error correction to detect or repair
86
+ the character errors that make ordinary base64-on-paper fragile.
87
+
88
+ > **Status: alpha.** The wire format and CLI may change before 1.0.
89
+
90
+ ## Features
91
+
92
+ - **Printable text, PDF, or Word output** with selectable font family and size.
93
+ - **Bundled OCR-B PDF option** under the SIL Open Font License, plus custom
94
+ `.ttf`/`.otf` PDF font paths for measured channels.
95
+ - **Measured OCR-safe `base16c-crc16-rs` alphabet** (`ABCDHKLMPRTVXY34`) with no confusable
96
+ character aliases.
97
+ - **Localized integrity checks** on every encoded line and protected page
98
+ metadata.
99
+ - **Document-wide Reed-Solomon parity** for correcting scattered OCR errors.
100
+ - **Optional whole-page recovery** (`--parity-pages K`, default off): survives
101
+ up to K wholly lost/unscannable pages, independent of the per-line parity.
102
+ - **Binary-safe archives** for nested trees, empty directories, and arbitrary
103
+ file contents.
104
+ - **Deterministic restore** from text; optional OCR providers are loaded only
105
+ when image input is requested.
106
+
107
+ ## Installation
108
+
109
+ ```bash
110
+ pip install glyphive
111
+ ```
112
+
113
+ Optional features:
114
+
115
+ | Extra | Adds | Needed for |
116
+ | --- | --- | --- |
117
+ | `pdf` | `fpdf2` | PDF output |
118
+ | `docx` | `python-docx` | Word (`.docx`) output |
119
+ | `zstd` | `zstandard` | zstd compression |
120
+ | `ocr` | `Pillow`, `pytesseract` | Tesseract image bridge; the Tesseract program is installed separately |
121
+ | `qr` | `segno`, `zxing-cpp`, `Pillow` | QR envelope generation and image decoding without OpenCV |
122
+ | `all` | all packages above | All lightweight integrations |
123
+
124
+ Glyphive requires `pathlib_next>=0.8.1` and Python 3.9 or newer.
125
+
126
+ ## Quick start
127
+
128
+ Create a text archive from one directory, inspect it, and restore it:
129
+
130
+ ```bash
131
+ glyphive create -f backup.txt --compression gzip -C project .
132
+ glyphive list -f backup.txt
133
+ glyphive inspect -f backup.txt # recovery-headroom report (read-only)
134
+ glyphive extract -f backup.txt -C restored
135
+ ```
136
+
137
+ Restore or inspect an already-generated GQ1 QR image set explicitly with
138
+ `glyphive extract -f qr-pages/ --from-qr -C restored` or
139
+ `glyphive list -f qr-pages/ --from-qr`. Ordinary image input continues through
140
+ OCR; QR mode requires `glyphive[qr]` and rejects mixed, duplicate, corrupt, or
141
+ incomplete symbol sets before writing files.
142
+
143
+ Create QR-only or hybrid human-readable/QR PDF pages with an explicit format:
144
+
145
+ ```bash
146
+ glyphive create -f backup.pdf --format qr -C project .
147
+ glyphive create -f backup-hybrid.pdf --format hybrid -C project .
148
+ ```
149
+
150
+ A `.pdf` filename without `--format` remains the ordinary text PDF renderer;
151
+ the suffix cannot distinguish PDF, QR, and hybrid presentations.
152
+
153
+ Tar-style mode flags are equivalent when a positional command is inconvenient:
154
+
155
+ ```bash
156
+ glyphive -c -f backup.txt -C project .
157
+ glyphive -t -f backup.txt
158
+ glyphive -x -f backup.txt -C restored
159
+ ```
160
+
161
+ ### Installed plugins
162
+
163
+ Glyphive can explicitly discover implementations supplied by installed Python
164
+ distributions. Pass the global `--plugins` flag to opt in for that invocation:
165
+
166
+ ```bash
167
+ glyphive --plugins create -f backup.txt --codec vendor_codec -C project .
168
+ ```
169
+
170
+ Plugin distributions register a concrete typed class under one of these entry
171
+ point groups:
172
+
173
+ | Entry-point group | Required base class |
174
+ | --- | --- |
175
+ | `glyphive.codecs` | `glyphive.codec.Codec` |
176
+ | `glyphive.compression` | `glyphive.compression.CompressionMethod` |
177
+ | `glyphive.render_formats` | `glyphive.render.RenderFormat` |
178
+ | `glyphive.ocr_providers` | `glyphive.restore.ocr.OcrProvider` |
179
+
180
+ The entry-point name must exactly match the class's lowercase registry `name`.
181
+ Library callers use `glyphive.plugins.discover()`; normal imports and registry
182
+ lookups never discover plugins. Discovery is deterministic and cached, and a
183
+ bad candidate is reported without preventing valid candidates from loading.
184
+
185
+ Installed plugin code executes with the same permissions as Glyphive and is
186
+ not sandboxed. Use `--plugins` only when you trust every installed distribution
187
+ that declares one of these groups. Discovery does not download or update code.
188
+
189
+ Create a PDF instead:
190
+
191
+ ```bash
192
+ pip install "glyphive[pdf,zstd]"
193
+ glyphive create -f backup.pdf --compression zstd -C project .
194
+ ```
195
+
196
+ Creation uses bounded-memory disk spools; `--temp-dir` selects their location
197
+ and `--chunk-size` tunes sequential I/O for unusually constrained systems.
198
+
199
+ Restore scans or generated documents with Tesseract. Input type is detected
200
+ from file contents first and the extension second:
201
+
202
+ ```bash
203
+ pip install "glyphive[ocr,document-input]"
204
+ glyphive extract -f scans/ --ocr-engine tesseract -C restored
205
+ glyphive list -f backup.pdf --ocr-engine tesseract
206
+ ```
207
+
208
+ Restore verifies global integrity before publishing staged files. Advanced
209
+ resource controls include `--temp-dir`, `--chunk-size`, and
210
+ `--max-output-bytes` on both `extract` and `list`. Transcript frames,
211
+ compressed payload, decompressed archive, and staged files are disk-backed;
212
+ Reed-Solomon correction retains only compact offsets and one codeword at a time.
213
+
214
+ The operating-system Tesseract executable and language data must also be
215
+ installed. PDF input uses `pypdfium2`; Glyphive-generated DOCX transcripts are
216
+ read directly with `python-docx`, without Microsoft Word or LibreOffice. See the
217
+ [create guide](https://jose-pr.github.io/glyphive/guides/create/),
218
+ [restore guide](https://jose-pr.github.io/glyphive/guides/restore/), and
219
+ [OCR guide](https://jose-pr.github.io/glyphive/guides/ocr/) for details.
220
+
221
+ ## Format at a glance
222
+
223
+ The default `base16c-crc16-rs` format uses exactly 16 payload symbols, or 4 bits per printed
224
+ character. Each data (`L`) or parity (`P`) line has a masked index and a full
225
+ CRC-16 encoded in the same safe alphabet:
226
+
227
+ ```text
228
+ L<5 safe index chars> <up to 60 safe payload chars> #<4 safe CRC chars>
229
+ ```
230
+
231
+ CRC-protected `H` header frames carry the codec, compression method, page
232
+ count, and whole-document SHA-256. Each page has a protected `T` footer with
233
+ its page number and a truncated page hash. Human-facing `#!glyphive` and
234
+ `PAGE n/total` text is display-only; restore trusts the protected frames.
235
+
236
+ Small, scattered line errors become known erasures and can be repaired by the
237
+ document-wide Reed-Solomon parity. A missing page is reported rather than
238
+ guessed, and the document-wide RS may still recover it from the surviving
239
+ pages when the budget allows. `create --parity-pages K` (default 0, off)
240
+ opts into a separate, guaranteed layer: K extra pages of document-level
241
+ parity that survive up to K wholly lost pages regardless of the per-line
242
+ budget. Read the [wire-format guide](https://jose-pr.github.io/glyphive/guides/wire-format/)
243
+ for the complete layering and framing rules.
244
+
245
+ ## Standalone zipapp
246
+
247
+ Releases may include a self-contained `glyphive.pyz`. Build the universal core
248
+ artifact with:
249
+
250
+ ```bash
251
+ python package.py --out dist/glyphive.pyz
252
+ python dist/glyphive.pyz --help
253
+ ```
254
+
255
+ The universal zipapp includes text output with `none` and `gzip` compression.
256
+ Optional, platform-specific artifacts can be built with `--extras`; use
257
+ `python package.py --help` for the declared choices.
258
+
259
+ ## API overview
260
+
261
+ | Module | Purpose |
262
+ | --- | --- |
263
+ | `glyphive.archive` | Serialize and inspect deterministic archive streams, including chunked reader/writer primitives |
264
+ | `glyphive.codec` | Resolve printable codecs; includes `base16c-crc16-rs` |
265
+ | `glyphive.compression` | Resolve `none`, `gzip`, and optional `zstd` compression |
266
+ | `glyphive.layout` | Paginate frames and verify protected page metadata |
267
+ | `glyphive.plugins` | Explicitly discover trusted installed entry points |
268
+ | `glyphive.render` | Render pages as text, PDF, or Word |
269
+ | `glyphive.restore` | Decode documents and safely restore file trees |
270
+ | `glyphive.restore.ocr` | Select OCR providers and OCR page images |
271
+
272
+ Full generated API documentation is available on the
273
+ [documentation site](https://jose-pr.github.io/glyphive/api/overview/).
274
+
275
+ ## Development
276
+
277
+ See [CONTRIBUTING.md](CONTRIBUTING.md) for setup, test, documentation, and pull
278
+ request guidance.
279
+
280
+ Use a project-local virtual environment, then run the lightweight suite:
281
+
282
+ ```bash
283
+ python -m venv .venv/dev
284
+ .venv/dev/Scripts/python -m pip install -e ".[all,dev,docs]"
285
+ .venv/dev/Scripts/python -m pytest -q
286
+ .venv/dev/Scripts/python -m mkdocs build --strict
287
+ ```
288
+
289
+ On POSIX, replace `.venv/dev/Scripts/python` with
290
+ `.venv/dev/bin/python`. OCR sweeps and performance measurements are separate
291
+ manual workloads; see [Benchmarks](https://jose-pr.github.io/glyphive/benchmarks/).
292
+
293
+ ### Releasing
294
+
295
+ Glyphive follows [Semantic Versioning](https://semver.org/) and keeps a
296
+ [`CHANGELOG.md`](CHANGELOG.md). Pushing a `v*` tag runs the release validation,
297
+ package build, publication, and documentation deployment workflow.
298
+
299
+ ## License
300
+
301
+ MIT — see [LICENSE](LICENSE).
302
+
303
+ The bundled OCR-B font retains its SIL Open Font License 1.1; see
304
+ [third-party licenses](THIRD_PARTY_LICENSES.md).