yom-format 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (199) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +86 -0
  3. package/dist/cli.d.ts +30 -0
  4. package/dist/cli.d.ts.map +1 -0
  5. package/dist/cli.js +344 -0
  6. package/dist/cli.js.map +1 -0
  7. package/dist/index.d.ts +36 -0
  8. package/dist/index.d.ts.map +1 -0
  9. package/dist/index.js +25 -0
  10. package/dist/index.js.map +1 -0
  11. package/dist/pipeline/avif.d.ts +3 -0
  12. package/dist/pipeline/avif.d.ts.map +1 -0
  13. package/dist/pipeline/avif.js +30 -0
  14. package/dist/pipeline/avif.js.map +1 -0
  15. package/dist/pipeline/convert.d.ts +28 -0
  16. package/dist/pipeline/convert.d.ts.map +1 -0
  17. package/dist/pipeline/convert.js +111 -0
  18. package/dist/pipeline/convert.js.map +1 -0
  19. package/dist/pipeline/index.d.ts +17 -0
  20. package/dist/pipeline/index.d.ts.map +1 -0
  21. package/dist/pipeline/index.js +17 -0
  22. package/dist/pipeline/index.js.map +1 -0
  23. package/dist/pipeline/ocrLayoutBuilder.d.ts +10 -0
  24. package/dist/pipeline/ocrLayoutBuilder.d.ts.map +1 -0
  25. package/dist/pipeline/ocrLayoutBuilder.js +62 -0
  26. package/dist/pipeline/ocrLayoutBuilder.js.map +1 -0
  27. package/dist/pipeline/pdfDocument.d.ts +12 -0
  28. package/dist/pipeline/pdfDocument.d.ts.map +1 -0
  29. package/dist/pipeline/pdfDocument.js +76 -0
  30. package/dist/pipeline/pdfDocument.js.map +1 -0
  31. package/dist/pipeline/renderPage.d.ts +10 -0
  32. package/dist/pipeline/renderPage.d.ts.map +1 -0
  33. package/dist/pipeline/renderPage.js +36 -0
  34. package/dist/pipeline/renderPage.js.map +1 -0
  35. package/dist/pipeline/textExtract.d.ts +29 -0
  36. package/dist/pipeline/textExtract.d.ts.map +1 -0
  37. package/dist/pipeline/textExtract.js +48 -0
  38. package/dist/pipeline/textExtract.js.map +1 -0
  39. package/dist/yom/blobLimits.d.ts +15 -0
  40. package/dist/yom/blobLimits.d.ts.map +1 -0
  41. package/dist/yom/blobLimits.js +25 -0
  42. package/dist/yom/blobLimits.js.map +1 -0
  43. package/dist/yom/byteSource.d.ts +56 -0
  44. package/dist/yom/byteSource.d.ts.map +1 -0
  45. package/dist/yom/byteSource.js +111 -0
  46. package/dist/yom/byteSource.js.map +1 -0
  47. package/dist/yom/bytes.d.ts +2 -0
  48. package/dist/yom/bytes.d.ts.map +1 -0
  49. package/dist/yom/bytes.js +13 -0
  50. package/dist/yom/bytes.js.map +1 -0
  51. package/dist/yom/constants.d.ts +34 -0
  52. package/dist/yom/constants.d.ts.map +1 -0
  53. package/dist/yom/constants.js +82 -0
  54. package/dist/yom/constants.js.map +1 -0
  55. package/dist/yom/errors.d.ts +44 -0
  56. package/dist/yom/errors.d.ts.map +1 -0
  57. package/dist/yom/errors.js +44 -0
  58. package/dist/yom/errors.js.map +1 -0
  59. package/dist/yom/flatbufferVerify.d.ts +21 -0
  60. package/dist/yom/flatbufferVerify.d.ts.map +1 -0
  61. package/dist/yom/flatbufferVerify.js +255 -0
  62. package/dist/yom/flatbufferVerify.js.map +1 -0
  63. package/dist/yom/generated/yomu/yom/blob-kind.d.ts +9 -0
  64. package/dist/yom/generated/yomu/yom/blob-kind.d.ts.map +1 -0
  65. package/dist/yom/generated/yomu/yom/blob-kind.js +12 -0
  66. package/dist/yom/generated/yomu/yom/blob-kind.js.map +1 -0
  67. package/dist/yom/generated/yomu/yom/blob-ref.d.ts +45 -0
  68. package/dist/yom/generated/yomu/yom/blob-ref.d.ts.map +1 -0
  69. package/dist/yom/generated/yomu/yom/blob-ref.js +141 -0
  70. package/dist/yom/generated/yomu/yom/blob-ref.js.map +1 -0
  71. package/dist/yom/generated/yomu/yom/book-metadata.d.ts +32 -0
  72. package/dist/yom/generated/yomu/yom/book-metadata.d.ts.map +1 -0
  73. package/dist/yom/generated/yomu/yom/book-metadata.js +89 -0
  74. package/dist/yom/generated/yomu/yom/book-metadata.js.map +1 -0
  75. package/dist/yom/generated/yomu/yom/compression.d.ts +5 -0
  76. package/dist/yom/generated/yomu/yom/compression.d.ts.map +1 -0
  77. package/dist/yom/generated/yomu/yom/compression.js +8 -0
  78. package/dist/yom/generated/yomu/yom/compression.js.map +1 -0
  79. package/dist/yom/generated/yomu/yom/main-index.d.ts +63 -0
  80. package/dist/yom/generated/yomu/yom/main-index.d.ts.map +1 -0
  81. package/dist/yom/generated/yomu/yom/main-index.js +208 -0
  82. package/dist/yom/generated/yomu/yom/main-index.js.map +1 -0
  83. package/dist/yom/generated/yomu/yom/ocrblock.d.ts +32 -0
  84. package/dist/yom/generated/yomu/yom/ocrblock.d.ts.map +1 -0
  85. package/dist/yom/generated/yomu/yom/ocrblock.js +84 -0
  86. package/dist/yom/generated/yomu/yom/ocrblock.js.map +1 -0
  87. package/dist/yom/generated/yomu/yom/ocrlayout.d.ts +27 -0
  88. package/dist/yom/generated/yomu/yom/ocrlayout.d.ts.map +1 -0
  89. package/dist/yom/generated/yomu/yom/ocrlayout.js +76 -0
  90. package/dist/yom/generated/yomu/yom/ocrlayout.js.map +1 -0
  91. package/dist/yom/generated/yomu/yom/ocrline.d.ts +35 -0
  92. package/dist/yom/generated/yomu/yom/ocrline.d.ts.map +1 -0
  93. package/dist/yom/generated/yomu/yom/ocrline.js +95 -0
  94. package/dist/yom/generated/yomu/yom/ocrline.js.map +1 -0
  95. package/dist/yom/generated/yomu/yom/ocrpage.d.ts +46 -0
  96. package/dist/yom/generated/yomu/yom/ocrpage.d.ts.map +1 -0
  97. package/dist/yom/generated/yomu/yom/ocrpage.js +147 -0
  98. package/dist/yom/generated/yomu/yom/ocrpage.js.map +1 -0
  99. package/dist/yom/generated/yomu/yom/ocrtoken.d.ts +44 -0
  100. package/dist/yom/generated/yomu/yom/ocrtoken.d.ts.map +1 -0
  101. package/dist/yom/generated/yomu/yom/ocrtoken.js +128 -0
  102. package/dist/yom/generated/yomu/yom/ocrtoken.js.map +1 -0
  103. package/dist/yom/generated/yomu/yom/outline-entry.d.ts +31 -0
  104. package/dist/yom/generated/yomu/yom/outline-entry.d.ts.map +1 -0
  105. package/dist/yom/generated/yomu/yom/outline-entry.js +92 -0
  106. package/dist/yom/generated/yomu/yom/outline-entry.js.map +1 -0
  107. package/dist/yom/generated/yomu/yom/page-entry.d.ts +95 -0
  108. package/dist/yom/generated/yomu/yom/page-entry.d.ts.map +1 -0
  109. package/dist/yom/generated/yomu/yom/page-entry.js +288 -0
  110. package/dist/yom/generated/yomu/yom/page-entry.js.map +1 -0
  111. package/dist/yom/generated/yomu/yom/pdfexport-metadata.d.ts +32 -0
  112. package/dist/yom/generated/yomu/yom/pdfexport-metadata.d.ts.map +1 -0
  113. package/dist/yom/generated/yomu/yom/pdfexport-metadata.js +96 -0
  114. package/dist/yom/generated/yomu/yom/pdfexport-metadata.js.map +1 -0
  115. package/dist/yom/generated/yomu/yom/reading-direction.d.ts +5 -0
  116. package/dist/yom/generated/yomu/yom/reading-direction.d.ts.map +1 -0
  117. package/dist/yom/generated/yomu/yom/reading-direction.js +8 -0
  118. package/dist/yom/generated/yomu/yom/reading-direction.js.map +1 -0
  119. package/dist/yom/generated/yomu/yom/search-term-ref.d.ts +34 -0
  120. package/dist/yom/generated/yomu/yom/search-term-ref.d.ts.map +1 -0
  121. package/dist/yom/generated/yomu/yom/search-term-ref.js +99 -0
  122. package/dist/yom/generated/yomu/yom/search-term-ref.js.map +1 -0
  123. package/dist/yom/generated/yomu/yom.d.ts +6 -0
  124. package/dist/yom/generated/yomu/yom.d.ts.map +1 -0
  125. package/dist/yom/generated/yomu/yom.js +8 -0
  126. package/dist/yom/generated/yomu/yom.js.map +1 -0
  127. package/dist/yom/hash.d.ts +4 -0
  128. package/dist/yom/hash.d.ts.map +1 -0
  129. package/dist/yom/hash.js +21 -0
  130. package/dist/yom/hash.js.map +1 -0
  131. package/dist/yom/header.d.ts +44 -0
  132. package/dist/yom/header.d.ts.map +1 -0
  133. package/dist/yom/header.js +115 -0
  134. package/dist/yom/header.js.map +1 -0
  135. package/dist/yom/reader.d.ts +57 -0
  136. package/dist/yom/reader.d.ts.map +1 -0
  137. package/dist/yom/reader.js +141 -0
  138. package/dist/yom/reader.js.map +1 -0
  139. package/dist/yom/searchIndex.d.ts +41 -0
  140. package/dist/yom/searchIndex.d.ts.map +1 -0
  141. package/dist/yom/searchIndex.js +133 -0
  142. package/dist/yom/searchIndex.js.map +1 -0
  143. package/dist/yom/textNormalizer.d.ts +19 -0
  144. package/dist/yom/textNormalizer.d.ts.map +1 -0
  145. package/dist/yom/textNormalizer.js +77 -0
  146. package/dist/yom/textNormalizer.js.map +1 -0
  147. package/dist/yom/validator.d.ts +14 -0
  148. package/dist/yom/validator.d.ts.map +1 -0
  149. package/dist/yom/validator.js +197 -0
  150. package/dist/yom/validator.js.map +1 -0
  151. package/dist/yom/writer.d.ts +76 -0
  152. package/dist/yom/writer.d.ts.map +1 -0
  153. package/dist/yom/writer.js +154 -0
  154. package/dist/yom/writer.js.map +1 -0
  155. package/dist/yom/zstd.d.ts +14 -0
  156. package/dist/yom/zstd.d.ts.map +1 -0
  157. package/dist/yom/zstd.js +43 -0
  158. package/dist/yom/zstd.js.map +1 -0
  159. package/package.json +65 -0
  160. package/src/cli.ts +372 -0
  161. package/src/index.ts +109 -0
  162. package/src/pipeline/avif.ts +33 -0
  163. package/src/pipeline/convert.ts +148 -0
  164. package/src/pipeline/index.ts +28 -0
  165. package/src/pipeline/ocrLayoutBuilder.ts +73 -0
  166. package/src/pipeline/pdfDocument.ts +86 -0
  167. package/src/pipeline/renderPage.ts +48 -0
  168. package/src/pipeline/textExtract.ts +64 -0
  169. package/src/worker/convert.worker.ts +54 -0
  170. package/src/yom/blobLimits.ts +30 -0
  171. package/src/yom/byteSource.ts +144 -0
  172. package/src/yom/bytes.ts +11 -0
  173. package/src/yom/constants.ts +89 -0
  174. package/src/yom/errors.ts +43 -0
  175. package/src/yom/flatbufferVerify.ts +298 -0
  176. package/src/yom/generated/yomu/yom/blob-kind.ts +12 -0
  177. package/src/yom/generated/yomu/yom/blob-ref.ts +184 -0
  178. package/src/yom/generated/yomu/yom/book-metadata.ts +121 -0
  179. package/src/yom/generated/yomu/yom/compression.ts +8 -0
  180. package/src/yom/generated/yomu/yom/main-index.ts +257 -0
  181. package/src/yom/generated/yomu/yom/ocrblock.ts +117 -0
  182. package/src/yom/generated/yomu/yom/ocrlayout.ts +102 -0
  183. package/src/yom/generated/yomu/yom/ocrline.ts +131 -0
  184. package/src/yom/generated/yomu/yom/ocrpage.ts +191 -0
  185. package/src/yom/generated/yomu/yom/ocrtoken.ts +173 -0
  186. package/src/yom/generated/yomu/yom/outline-entry.ts +124 -0
  187. package/src/yom/generated/yomu/yom/page-entry.ts +388 -0
  188. package/src/yom/generated/yomu/yom/pdfexport-metadata.ts +128 -0
  189. package/src/yom/generated/yomu/yom/reading-direction.ts +8 -0
  190. package/src/yom/generated/yomu/yom/search-term-ref.ts +134 -0
  191. package/src/yom/generated/yomu/yom.ts +9 -0
  192. package/src/yom/hash.ts +20 -0
  193. package/src/yom/header.ts +132 -0
  194. package/src/yom/reader.ts +163 -0
  195. package/src/yom/searchIndex.ts +178 -0
  196. package/src/yom/textNormalizer.ts +92 -0
  197. package/src/yom/validator.ts +245 -0
  198. package/src/yom/writer.ts +337 -0
  199. package/src/yom/zstd.ts +47 -0
@@ -0,0 +1,154 @@
1
+ import { Builder } from "flatbuffers";
2
+ import { BlobKind } from "./generated/yomu/yom/blob-kind.js";
3
+ import { Compression } from "./generated/yomu/yom/compression.js";
4
+ import { BlobRefT } from "./generated/yomu/yom/blob-ref.js";
5
+ import { PageEntryT } from "./generated/yomu/yom/page-entry.js";
6
+ import { BookMetadataT } from "./generated/yomu/yom/book-metadata.js";
7
+ import { SearchTermRefT } from "./generated/yomu/yom/search-term-ref.js";
8
+ import { OutlineEntryT } from "./generated/yomu/yom/outline-entry.js";
9
+ import { PDFExportMetadataT } from "./generated/yomu/yom/pdfexport-metadata.js";
10
+ import { MainIndexT } from "./generated/yomu/yom/main-index.js";
11
+ import { YOM_FORMAT_VERSION, YOM_HASH_LENGTH, YOM_HEADER_SIZE, YOM_MAGIC, requiredAlignment, requiredCompression, } from "./constants.js";
12
+ import { encodeFooter, encodeHeader, zeroingHashes } from "./header.js";
13
+ import { sha256 } from "./hash.js";
14
+ import { zstdCompress } from "./zstd.js";
15
+ import { concatBytes } from "./bytes.js";
16
+ export class YOMWriterError extends Error {
17
+ }
18
+ async function planBlob(input) {
19
+ const alignment = requiredAlignment(input.kind);
20
+ const uncompressedLength = BigInt(input.data.length);
21
+ const hash = await sha256(input.data);
22
+ const storedBytes = input.compression === Compression.None ? input.data : await zstdCompress(input.data);
23
+ return { kind: input.kind, compression: input.compression, alignment, storedBytes, uncompressedLength, hash };
24
+ }
25
+ function validatePolicy(blob, expectedKind) {
26
+ if (blob.kind !== expectedKind) {
27
+ throw new YOMWriterError(`blob kind mismatch: expected ${expectedKind}, got ${blob.kind}`);
28
+ }
29
+ const required = requiredCompression(blob.kind);
30
+ if (blob.compression !== required) {
31
+ throw new YOMWriterError(`compression policy violation for kind ${blob.kind}: declared ${blob.compression}, required ${required}`);
32
+ }
33
+ }
34
+ export async function writeYOM(input) {
35
+ if (input.pages.length === 0)
36
+ throw new YOMWriterError("YOM package must have at least one page");
37
+ const blobPlans = [];
38
+ const pageBlobIndices = [];
39
+ for (const page of input.pages) {
40
+ validatePolicy(page.pageImage, BlobKind.PageImage);
41
+ validatePolicy(page.thumbnail, BlobKind.ThumbnailImage);
42
+ const imageIndex = blobPlans.length;
43
+ blobPlans.push(await planBlob(page.pageImage));
44
+ const thumbnailIndex = blobPlans.length;
45
+ blobPlans.push(await planBlob(page.thumbnail));
46
+ pageBlobIndices.push({ image: imageIndex, thumbnail: thumbnailIndex });
47
+ }
48
+ let ocrIndex = -1;
49
+ if (input.ocrLayout) {
50
+ validatePolicy(input.ocrLayout, BlobKind.OCRLayout);
51
+ ocrIndex = blobPlans.length;
52
+ blobPlans.push(await planBlob(input.ocrLayout));
53
+ }
54
+ let normalizedTextIndex = -1;
55
+ if (input.normalizedText) {
56
+ validatePolicy(input.normalizedText, BlobKind.NormalizedText);
57
+ normalizedTextIndex = blobPlans.length;
58
+ blobPlans.push(await planBlob(input.normalizedText));
59
+ }
60
+ let searchDictionaryIndex = -1;
61
+ if (input.searchDictionary) {
62
+ validatePolicy(input.searchDictionary, BlobKind.SearchDictionary);
63
+ searchDictionaryIndex = blobPlans.length;
64
+ blobPlans.push(await planBlob(input.searchDictionary));
65
+ }
66
+ let searchPostingsIndex = -1;
67
+ if (input.searchPostings) {
68
+ validatePolicy(input.searchPostings, BlobKind.SearchPostings);
69
+ searchPostingsIndex = blobPlans.length;
70
+ blobPlans.push(await planBlob(input.searchPostings));
71
+ }
72
+ // FlatBuffers scalar fields are fixed-width, so the encoded index size does
73
+ // not depend on the actual blob offset *values* -- only on how many
74
+ // blobs/pages/strings/etc. exist. Build once with placeholder offsets to
75
+ // learn the index size, then again with the real offsets.
76
+ const placeholderOffsets = blobPlans.map(() => 0n);
77
+ const dryRunIndex = buildIndexBytes(input, blobPlans, placeholderOffsets, pageBlobIndices, ocrIndex, normalizedTextIndex, searchDictionaryIndex, searchPostingsIndex);
78
+ const blobAreaStart = BigInt(YOM_HEADER_SIZE) + BigInt(dryRunIndex.length);
79
+ const blobOffsets = computeBlobOffsets(blobPlans, blobAreaStart);
80
+ const indexBytes = buildIndexBytes(input, blobPlans, blobOffsets, pageBlobIndices, ocrIndex, normalizedTextIndex, searchDictionaryIndex, searchPostingsIndex);
81
+ if (indexBytes.length !== dryRunIndex.length) {
82
+ throw new YOMWriterError("YOM index size must not depend on blob offset values");
83
+ }
84
+ const bodyChunks = [];
85
+ let cursor = blobAreaStart;
86
+ for (let i = 0; i < blobPlans.length; i++) {
87
+ const plan = blobPlans[i];
88
+ const offset = blobOffsets[i];
89
+ if (offset < cursor)
90
+ throw new YOMWriterError("blob offsets must be monotonically increasing");
91
+ if (offset > cursor) {
92
+ bodyChunks.push(new Uint8Array(Number(offset - cursor)));
93
+ }
94
+ bodyChunks.push(plan.storedBytes);
95
+ cursor = offset + BigInt(plan.storedBytes.length);
96
+ }
97
+ const body = concatBytes(bodyChunks);
98
+ const indexHash = await sha256(indexBytes);
99
+ const header = {
100
+ magic: YOM_MAGIC,
101
+ formatVersion: YOM_FORMAT_VERSION,
102
+ indexOffset: BigInt(YOM_HEADER_SIZE),
103
+ indexCompressedLength: BigInt(indexBytes.length),
104
+ indexUncompressedLength: BigInt(indexBytes.length),
105
+ indexHash,
106
+ fileHash: new Uint8Array(YOM_HASH_LENGTH),
107
+ };
108
+ // File hash is computed with both hash fields zeroed (see header.ts doc
109
+ // comment), then patched into the real header/footer afterwards.
110
+ const hashingHeader = zeroingHashes(header);
111
+ const fileForHashing = concatBytes([encodeHeader(hashingHeader), indexBytes, body, encodeFooter(hashingHeader)]);
112
+ const fileHash = await sha256(fileForHashing);
113
+ const finalHeader = { ...header, fileHash };
114
+ return concatBytes([encodeHeader(finalHeader), indexBytes, body, encodeFooter(finalHeader)]);
115
+ }
116
+ function computeBlobOffsets(blobPlans, blobAreaStart) {
117
+ const offsets = [];
118
+ let cursor = blobAreaStart;
119
+ for (const plan of blobPlans) {
120
+ const alignment = BigInt(Math.max(plan.alignment, 1));
121
+ const remainder = cursor % alignment;
122
+ if (remainder !== 0n)
123
+ cursor += alignment - remainder;
124
+ offsets.push(cursor);
125
+ cursor += BigInt(plan.storedBytes.length);
126
+ }
127
+ return offsets;
128
+ }
129
+ function buildIndexBytes(input, blobPlans, blobOffsets, pageBlobIndices, ocrIndex, normalizedTextIndex, searchDictionaryIndex, searchPostingsIndex) {
130
+ const builder = new Builder(1024);
131
+ // Without this, the builder omits any scalar field whose value equals its
132
+ // schema default (e.g. a placeholder blob offset of 0), which would make
133
+ // the dry-run index a different size than the real index -- breaking the
134
+ // "index size does not depend on blob offset values" invariant above.
135
+ builder.forceDefaults(true);
136
+ const pages = input.pages.map((page, index) => {
137
+ const { image: imageIndex, thumbnail: thumbnailIndex } = pageBlobIndices[index];
138
+ return new PageEntryT(index, page.pageLabel, page.pixelWidth, page.pixelHeight, page.dpi, page.colorSpace, page.colorProfileRef, page.mediaBox, page.cropBox, page.contentBox, page.rotation, page.readingDirection, imageIndex, thumbnailIndex, Array.from(blobPlans[imageIndex].hash));
139
+ });
140
+ const blobs = blobPlans.map((plan, i) => new BlobRefT(plan.kind, blobOffsets[i], BigInt(plan.storedBytes.length), plan.uncompressedLength, plan.compression, plan.alignment, Array.from(plan.hash)));
141
+ const searchTerms = input.searchTerms.map((term) => new SearchTermRefT(term.term, term.dictionaryOffset, term.dictionaryLength, term.postingsOffset, term.postingsLength));
142
+ const pdfExport = input.pdfExport
143
+ ? new PDFExportMetadataT(input.pdfExport.pageLabels, buildOutline(input.pdfExport.outline))
144
+ : null;
145
+ const book = new BookMetadataT(input.book.title, input.pages.length, input.book.readingDirection, input.book.createdAtUnix);
146
+ const mainIndex = new MainIndexT(YOM_FORMAT_VERSION, book, pages, blobs, ocrIndex, normalizedTextIndex, searchDictionaryIndex, searchPostingsIndex, searchTerms, pdfExport);
147
+ const offset = mainIndex.pack(builder);
148
+ builder.finish(offset);
149
+ return builder.asUint8Array();
150
+ }
151
+ function buildOutline(entries) {
152
+ return entries.map((entry) => new OutlineEntryT(entry.title, entry.pageId, buildOutline(entry.children)));
153
+ }
154
+ //# sourceMappingURL=writer.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"writer.js","sourceRoot":"","sources":["../../src/yom/writer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AACtC,OAAO,EAAE,QAAQ,EAAE,MAAM,mCAAmC,CAAC;AAC7D,OAAO,EAAE,WAAW,EAAE,MAAM,qCAAqC,CAAC;AAElE,OAAO,EAAE,QAAQ,EAAE,MAAM,kCAAkC,CAAC;AAC5D,OAAO,EAAE,UAAU,EAAE,MAAM,oCAAoC,CAAC;AAChE,OAAO,EAAE,aAAa,EAAE,MAAM,uCAAuC,CAAC;AACtE,OAAO,EAAE,cAAc,EAAE,MAAM,yCAAyC,CAAC;AACzE,OAAO,EAAE,aAAa,EAAE,MAAM,uCAAuC,CAAC;AACtE,OAAO,EAAE,kBAAkB,EAAE,MAAM,4CAA4C,CAAC;AAChF,OAAO,EAAE,UAAU,EAAE,MAAM,oCAAoC,CAAC;AAChE,OAAO,EACL,kBAAkB,EAClB,eAAe,EACf,eAAe,EACf,SAAS,EACT,iBAAiB,EACjB,mBAAmB,GACpB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,aAAa,EAAkB,MAAM,aAAa,CAAC;AACxF,OAAO,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AACnC,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AACzC,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AA8EzC,MAAM,OAAO,cAAe,SAAQ,KAAK;CAAG;AAY5C,KAAK,UAAU,QAAQ,CAAC,KAAgB;IACtC,MAAM,SAAS,GAAG,iBAAiB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAChD,MAAM,kBAAkB,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACrD,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACtC,MAAM,WAAW,GAAG,KAAK,CAAC,WAAW,KAAK,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACzG,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,WAAW,EAAE,KAAK,CAAC,WAAW,EAAE,SAAS,EAAE,WAAW,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAC;AAChH,CAAC;AAED,SAAS,cAAc,CAAC,IAAe,EAAE,YAAsB;IAC7D,IAAI,IAAI,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;QAC/B,MAAM,IAAI,cAAc,CAAC,gCAAgC,YAAY,SAAS,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;IAC7F,CAAC;IACD,MAAM,QAAQ,GAAG,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAChD,IAAI,IAAI,CAAC,WAAW,KAAK,QAAQ,EAAE,CAAC;QAClC,MAAM,IAAI,cAAc,CACtB,yCAAyC,IAAI,CAAC,IAAI,cAAc,IAAI,CAAC,WAAW,cAAc,QAAQ,EAAE,CACzG,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,KAAkB;IAC/C,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,MAAM,IAAI,cAAc,CAAC,yCAAyC,CAAC,CAAC;IAElG,MAAM,SAAS,GAAe,EAAE,CAAC;IACjC,MAAM,eAAe,GAAgD,EAAE,CAAC;IAExE,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;QAC/B,cAAc,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,SAAS,CAAC,CAAC;QACnD,cAAc,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,cAAc,CAAC,CAAC;QACxD,MAAM,UAAU,GAAG,SAAS,CAAC,MAAM,CAAC;QACpC,SAAS,CAAC,IAAI,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;QAC/C,MAAM,cAAc,GAAG,SAAS,CAAC,MAAM,CAAC;QACxC,SAAS,CAAC,IAAI,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;QAC/C,eAAe,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE,SAAS,EAAE,cAAc,EAAE,CAAC,CAAC;IACzE,CAAC;IAED,IAAI,QAAQ,GAAG,CAAC,CAAC,CAAC;IAClB,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;QACpB,cAAc,CAAC,KAAK,CAAC,SAAS,EAAE,QAAQ,CAAC,SAAS,CAAC,CAAC;QACpD,QAAQ,GAAG,SAAS,CAAC,MAAM,CAAC;QAC5B,SAAS,CAAC,IAAI,CAAC,MAAM,QAAQ,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC;IAClD,CAAC;IACD,IAAI,mBAAmB,GAAG,CAAC,CAAC,CAAC;IAC7B,IAAI,KAAK,CAAC,cAAc,EAAE,CAAC;QACzB,cAAc,CAAC,KAAK,CAAC,cAAc,EAAE,QAAQ,CAAC,cAAc,CAAC,CAAC;QAC9D,mBAAmB,GAAG,SAAS,CAAC,MAAM,CAAC;QACvC,SAAS,CAAC,IAAI,CAAC,MAAM,QAAQ,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC;IACvD,CAAC;IACD,IAAI,qBAAqB,GAAG,CAAC,CAAC,CAAC;IAC/B,IAAI,KAAK,CAAC,gBAAgB,EAAE,CAAC;QAC3B,cAAc,CAAC,KAAK,CAAC,gBAAgB,EAAE,QAAQ,CAAC,gBAAgB,CAAC,CAAC;QAClE,qBAAqB,GAAG,SAAS,CAAC,MAAM,CAAC;QACzC,SAAS,CAAC,IAAI,CAAC,MAAM,QAAQ,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAC;IACzD,CAAC;IACD,IAAI,mBAAmB,GAAG,CAAC,CAAC,CAAC;IAC7B,IAAI,KAAK,CAAC,cAAc,EAAE,CAAC;QACzB,cAAc,CAAC,KAAK,CAAC,cAAc,EAAE,QAAQ,CAAC,cAAc,CAAC,CAAC;QAC9D,mBAAmB,GAAG,SAAS,CAAC,MAAM,CAAC;QACvC,SAAS,CAAC,IAAI,CAAC,MAAM,QAAQ,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC;IACvD,CAAC;IAED,4EAA4E;IAC5E,oEAAoE;IACpE,yEAAyE;IACzE,0DAA0D;IAC1D,MAAM,kBAAkB,GAAG,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;IACnD,MAAM,WAAW,GAAG,eAAe,CACjC,KAAK,EACL,SAAS,EACT,kBAAkB,EAClB,eAAe,EACf,QAAQ,EACR,mBAAmB,EACnB,qBAAqB,EACrB,mBAAmB,CACpB,CAAC;IAEF,MAAM,aAAa,GAAG,MAAM,CAAC,eAAe,CAAC,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;IAC3E,MAAM,WAAW,GAAG,kBAAkB,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC;IAEjE,MAAM,UAAU,GAAG,eAAe,CAChC,KAAK,EACL,SAAS,EACT,WAAW,EACX,eAAe,EACf,QAAQ,EACR,mBAAmB,EACnB,qBAAqB,EACrB,mBAAmB,CACpB,CAAC;IACF,IAAI,UAAU,CAAC,MAAM,KAAK,WAAW,CAAC,MAAM,EAAE,CAAC;QAC7C,MAAM,IAAI,cAAc,CAAC,sDAAsD,CAAC,CAAC;IACnF,CAAC;IAED,MAAM,UAAU,GAAiB,EAAE,CAAC;IACpC,IAAI,MAAM,GAAG,aAAa,CAAC;IAC3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC1C,MAAM,IAAI,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;QAC1B,MAAM,MAAM,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;QAC9B,IAAI,MAAM,GAAG,MAAM;YAAE,MAAM,IAAI,cAAc,CAAC,+CAA+C,CAAC,CAAC;QAC/F,IAAI,MAAM,GAAG,MAAM,EAAE,CAAC;YACpB,UAAU,CAAC,IAAI,CAAC,IAAI,UAAU,CAAC,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QAC3D,CAAC;QACD,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAClC,MAAM,GAAG,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;IACpD,CAAC;IACD,MAAM,IAAI,GAAG,WAAW,CAAC,UAAU,CAAC,CAAC;IAErC,MAAM,SAAS,GAAG,MAAM,MAAM,CAAC,UAAU,CAAC,CAAC;IAC3C,MAAM,MAAM,GAAc;QACxB,KAAK,EAAE,SAAS;QAChB,aAAa,EAAE,kBAAkB;QACjC,WAAW,EAAE,MAAM,CAAC,eAAe,CAAC;QACpC,qBAAqB,EAAE,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC;QAChD,uBAAuB,EAAE,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC;QAClD,SAAS;QACT,QAAQ,EAAE,IAAI,UAAU,CAAC,eAAe,CAAC;KAC1C,CAAC;IAEF,wEAAwE;IACxE,iEAAiE;IACjE,MAAM,aAAa,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;IAC5C,MAAM,cAAc,GAAG,WAAW,CAAC,CAAC,YAAY,CAAC,aAAa,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,YAAY,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;IACjH,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,cAAc,CAAC,CAAC;IAE9C,MAAM,WAAW,GAAc,EAAE,GAAG,MAAM,EAAE,QAAQ,EAAE,CAAC;IACvD,OAAO,WAAW,CAAC,CAAC,YAAY,CAAC,WAAW,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,YAAY,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;AAC/F,CAAC;AAED,SAAS,kBAAkB,CAAC,SAAqB,EAAE,aAAqB;IACtE,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,MAAM,GAAG,aAAa,CAAC;IAC3B,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;QAC7B,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC;QACtD,MAAM,SAAS,GAAG,MAAM,GAAG,SAAS,CAAC;QACrC,IAAI,SAAS,KAAK,EAAE;YAAE,MAAM,IAAI,SAAS,GAAG,SAAS,CAAC;QACtD,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACrB,MAAM,IAAI,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;IAC5C,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,eAAe,CACtB,KAAkB,EAClB,SAAqB,EACrB,WAAqB,EACrB,eAA4D,EAC5D,QAAgB,EAChB,mBAA2B,EAC3B,qBAA6B,EAC7B,mBAA2B;IAE3B,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAClC,0EAA0E;IAC1E,yEAAyE;IACzE,yEAAyE;IACzE,sEAAsE;IACtE,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;IAE5B,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;QAC5C,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,SAAS,EAAE,cAAc,EAAE,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAChF,OAAO,IAAI,UAAU,CACnB,KAAK,EACL,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,UAAU,EACf,IAAI,CAAC,WAAW,EAChB,IAAI,CAAC,GAAG,EACR,IAAI,CAAC,UAAU,EACf,IAAI,CAAC,eAAe,EACpB,IAAI,CAAC,QAAQ,EACb,IAAI,CAAC,OAAO,EACZ,IAAI,CAAC,UAAU,EACf,IAAI,CAAC,QAAQ,EACb,IAAI,CAAC,gBAAgB,EACrB,UAAU,EACV,cAAc,EACd,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CACvC,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,MAAM,KAAK,GAAG,SAAS,CAAC,GAAG,CACzB,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CACV,IAAI,QAAQ,CACV,IAAI,CAAC,IAAI,EACT,WAAW,CAAC,CAAC,CAAC,EACd,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,EAC/B,IAAI,CAAC,kBAAkB,EACvB,IAAI,CAAC,WAAW,EAChB,IAAI,CAAC,SAAS,EACd,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CACtB,CACJ,CAAC;IAEF,MAAM,WAAW,GAAG,KAAK,CAAC,WAAW,CAAC,GAAG,CACvC,CAAC,IAAI,EAAE,EAAE,CACP,IAAI,cAAc,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,gBAAgB,EAAE,IAAI,CAAC,gBAAgB,EAAE,IAAI,CAAC,cAAc,EAAE,IAAI,CAAC,cAAc,CAAC,CACxH,CAAC;IAEF,MAAM,SAAS,GAAG,KAAK,CAAC,SAAS;QAC/B,CAAC,CAAC,IAAI,kBAAkB,CAAC,KAAK,CAAC,SAAS,CAAC,UAAU,EAAE,YAAY,CAAC,KAAK,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QAC3F,CAAC,CAAC,IAAI,CAAC;IAET,MAAM,IAAI,GAAG,IAAI,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,gBAAgB,EAAE,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAE5H,MAAM,SAAS,GAAG,IAAI,UAAU,CAC9B,kBAAkB,EAClB,IAAI,EACJ,KAAK,EACL,KAAK,EACL,QAAQ,EACR,mBAAmB,EACnB,qBAAqB,EACrB,mBAAmB,EACnB,WAAW,EACX,SAAS,CACV,CAAC;IAEF,MAAM,MAAM,GAAG,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACvC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACvB,OAAO,OAAO,CAAC,YAAY,EAAE,CAAC;AAChC,CAAC;AAED,SAAS,YAAY,CAAC,OAAuB;IAC3C,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,aAAa,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,YAAY,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;AAC5G,CAAC"}
@@ -0,0 +1,14 @@
1
+ export declare function zstdCompress(data: Uint8Array): Promise<Uint8Array>;
2
+ /**
3
+ * Decompresses a zstd frame and asserts the output is exactly `expectedSize`
4
+ * bytes (the blob's index-declared `uncompressed_length`), converging on the
5
+ * Swift `ZstdCodec.decompress(_, originalSize:)` discipline.
6
+ *
7
+ * `@bokuweb/zstd-wasm`'s `decompress` takes no target-size argument (it sizes
8
+ * its output buffer from the frame's own declared content size), so the guard
9
+ * against a size-lying frame is this exact-length check plus the caller's
10
+ * declared-size cap (`enforceBlobSizePolicy`). A mismatch throws
11
+ * `CorruptBlobError` before the bytes are used, ahead of the SHA-256 gate.
12
+ */
13
+ export declare function zstdDecompress(data: Uint8Array, expectedSize: number): Promise<Uint8Array>;
14
+ //# sourceMappingURL=zstd.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"zstd.d.ts","sourceRoot":"","sources":["../../src/yom/zstd.ts"],"names":[],"mappings":"AAuBA,wBAAsB,YAAY,CAAC,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC,CAGxE;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,cAAc,CAAC,IAAI,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CAOhG"}
@@ -0,0 +1,43 @@
1
+ import { init, compress, decompress } from "@bokuweb/zstd-wasm";
2
+ import { CorruptBlobError } from "./errors.js";
3
+ /**
4
+ * Thin wrapper over `@bokuweb/zstd-wasm`, the Web analogue of the Swift
5
+ * implementation in `swift/Sources/YOMFormat/ZstdCodec.swift`'s vendored
6
+ * libzstd wrapper. Every YOM
7
+ * blob is bounded and self-describing (zstd frames carry their own content
8
+ * size), so the one-shot simple API is enough -- no streaming/dictionary
9
+ * support is needed here either.
10
+ */
11
+ let initPromise = null;
12
+ function ensureInit() {
13
+ if (!initPromise) {
14
+ initPromise = init();
15
+ }
16
+ return initPromise;
17
+ }
18
+ /** Compression level mirrors `ZstdCodec.compressionLevel` (favor decode speed). */
19
+ const COMPRESSION_LEVEL = 3;
20
+ export async function zstdCompress(data) {
21
+ await ensureInit();
22
+ return compress(data, COMPRESSION_LEVEL);
23
+ }
24
+ /**
25
+ * Decompresses a zstd frame and asserts the output is exactly `expectedSize`
26
+ * bytes (the blob's index-declared `uncompressed_length`), converging on the
27
+ * Swift `ZstdCodec.decompress(_, originalSize:)` discipline.
28
+ *
29
+ * `@bokuweb/zstd-wasm`'s `decompress` takes no target-size argument (it sizes
30
+ * its output buffer from the frame's own declared content size), so the guard
31
+ * against a size-lying frame is this exact-length check plus the caller's
32
+ * declared-size cap (`enforceBlobSizePolicy`). A mismatch throws
33
+ * `CorruptBlobError` before the bytes are used, ahead of the SHA-256 gate.
34
+ */
35
+ export async function zstdDecompress(data, expectedSize) {
36
+ await ensureInit();
37
+ const out = decompress(data);
38
+ if (out.length !== expectedSize) {
39
+ throw new CorruptBlobError(`zstd output length ${out.length} != declared ${expectedSize}`);
40
+ }
41
+ return out;
42
+ }
43
+ //# sourceMappingURL=zstd.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"zstd.js","sourceRoot":"","sources":["../../src/yom/zstd.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAChE,OAAO,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAE/C;;;;;;;GAOG;AACH,IAAI,WAAW,GAAyB,IAAI,CAAC;AAE7C,SAAS,UAAU;IACjB,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,WAAW,GAAG,IAAI,EAAE,CAAC;IACvB,CAAC;IACD,OAAO,WAAW,CAAC;AACrB,CAAC;AAED,mFAAmF;AACnF,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAE5B,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,IAAgB;IACjD,MAAM,UAAU,EAAE,CAAC;IACnB,OAAO,QAAQ,CAAC,IAAI,EAAE,iBAAiB,CAAC,CAAC;AAC3C,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,IAAgB,EAAE,YAAoB;IACzE,MAAM,UAAU,EAAE,CAAC;IACnB,MAAM,GAAG,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;IAC7B,IAAI,GAAG,CAAC,MAAM,KAAK,YAAY,EAAE,CAAC;QAChC,MAAM,IAAI,gBAAgB,CAAC,sBAAsB,GAAG,CAAC,MAAM,gBAAgB,YAAY,EAAE,CAAC,CAAC;IAC7F,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC"}
package/package.json ADDED
@@ -0,0 +1,65 @@
1
+ {
2
+ "name": "yom-format",
3
+ "version": "0.1.0",
4
+ "type": "module",
5
+ "description": "Read-only packfile format for scanned books: AVIF pages, FlatBuffers index, OCR + full-text search in a single .yom file",
6
+ "license": "MIT",
7
+ "author": "juntaki",
8
+ "homepage": "https://github.com/juntaki/yom-format",
9
+ "repository": {
10
+ "type": "git",
11
+ "url": "github:juntaki/yom-format",
12
+ "directory": "web"
13
+ },
14
+ "keywords": [
15
+ "yom",
16
+ "ebook",
17
+ "packfile",
18
+ "flatbuffers",
19
+ "avif",
20
+ "ocr",
21
+ "full-text-search",
22
+ "pdf"
23
+ ],
24
+ "sideEffects": false,
25
+ "engines": {
26
+ "node": ">=20"
27
+ },
28
+ "files": [
29
+ "dist",
30
+ "src"
31
+ ],
32
+ "main": "./dist/index.js",
33
+ "types": "./dist/index.d.ts",
34
+ "bin": {
35
+ "yom": "./dist/cli.js"
36
+ },
37
+ "exports": {
38
+ ".": {
39
+ "types": "./dist/index.d.ts",
40
+ "default": "./dist/index.js"
41
+ },
42
+ "./pipeline": {
43
+ "types": "./dist/pipeline/index.d.ts",
44
+ "default": "./dist/pipeline/index.js"
45
+ }
46
+ },
47
+ "scripts": {
48
+ "test": "vitest run",
49
+ "typecheck": "tsc -b --noEmit",
50
+ "build": "tsc -p tsconfig.build.json",
51
+ "prepublishOnly": "npm run typecheck && npm run test -- --run && npm run build"
52
+ },
53
+ "dependencies": {
54
+ "@bokuweb/zstd-wasm": "^0.0.27",
55
+ "@jsquash/avif": "^2.1.1",
56
+ "flatbuffers": "^25.9.23",
57
+ "pdfjs-dist": "^6.1.200"
58
+ },
59
+ "devDependencies": {
60
+ "@types/node": "^22.20.1",
61
+ "typescript": "^5.7.3",
62
+ "vite": "^6.0.7",
63
+ "vitest": "^2.1.8"
64
+ }
65
+ }
package/src/cli.ts ADDED
@@ -0,0 +1,372 @@
1
+ #!/usr/bin/env node
2
+ /**
3
+ * `yom` -- a small, dependency-light CLI for inspecting, verifying, and
4
+ * extracting from existing `.yom` packfiles.
5
+ *
6
+ * Commands: info | verify | ls | extract | meta. `verify` also accepts an
7
+ * HTTP(S) URL (exercising the Gap-1 ranged reader end-to-end); the others
8
+ * operate on a local file. PDF -> .yom conversion is intentionally NOT here:
9
+ * that pipeline is browser-only. Use the web convert tool or the iOS app.
10
+ *
11
+ * Command handlers are exported for testability; the process entry point at
12
+ * the bottom only runs when the file is executed directly.
13
+ */
14
+ import { readFile, writeFile } from "node:fs/promises";
15
+ import { pathToFileURL } from "node:url";
16
+ import { YOMPackfileReader } from "./yom/reader.js";
17
+ import { validateYOM } from "./yom/validator.js";
18
+ import { parseHeaderFields } from "./yom/header.js";
19
+ import { YOM_HEADER_SIZE } from "./yom/constants.js";
20
+ import { pdfExportPresent } from "./yom/flatbufferVerify.js";
21
+ import { ReadingDirection } from "./yom/generated/yomu/yom/reading-direction.js";
22
+ import type { MainIndex } from "./yom/generated/yomu/yom/main-index.js";
23
+ import type { OutlineEntry } from "./yom/generated/yomu/yom/outline-entry.js";
24
+
25
+ export interface CliIO {
26
+ out(line: string): void;
27
+ err(line: string): void;
28
+ }
29
+
30
+ interface ParsedArgs {
31
+ positionals: string[];
32
+ json: boolean;
33
+ page?: number;
34
+ kind?: string;
35
+ out?: string;
36
+ }
37
+
38
+ const VALUE_FLAGS = new Set(["--page", "--kind", "-o", "--out"]);
39
+
40
+ function parseArgs(args: string[]): ParsedArgs {
41
+ const parsed: ParsedArgs = { positionals: [], json: false };
42
+ for (let i = 0; i < args.length; i++) {
43
+ const arg = args[i];
44
+ if (arg === "--json") {
45
+ parsed.json = true;
46
+ } else if (VALUE_FLAGS.has(arg)) {
47
+ const value = args[++i];
48
+ if (value === undefined) throw new CliUsageError(`missing value for ${arg}`);
49
+ if (arg === "--page") parsed.page = Number(value);
50
+ else if (arg === "--kind") parsed.kind = value;
51
+ else parsed.out = value; // -o | --out
52
+ } else if (arg.startsWith("-")) {
53
+ throw new CliUsageError(`unknown flag ${arg}`);
54
+ } else {
55
+ parsed.positionals.push(arg);
56
+ }
57
+ }
58
+ return parsed;
59
+ }
60
+
61
+ class CliUsageError extends Error {}
62
+
63
+ const USAGE = `yom -- inspect and extract from .yom packfiles
64
+
65
+ Usage:
66
+ yom info <file.yom> [--json]
67
+ yom verify <file.yom | url> [--json]
68
+ yom ls <file.yom> [--json]
69
+ yom extract <file.yom> --page N --kind image|thumb|ocr|text -o <out>
70
+ yom meta <file.yom>
71
+
72
+ Note: PDF -> .yom conversion is browser-only; use the web convert tool or the
73
+ iOS app. There is no 'convert' command.`;
74
+
75
+ function isUrl(target: string): boolean {
76
+ return /^https?:\/\//i.test(target);
77
+ }
78
+
79
+ async function loadLocalBytes(path: string): Promise<Uint8Array> {
80
+ const buf = await readFile(path);
81
+ return new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength);
82
+ }
83
+
84
+ function readingDirectionName(value: number): string {
85
+ return ReadingDirection[value] ?? String(value);
86
+ }
87
+
88
+ /** `info` -- magic, version, sizes, page count, index summary. */
89
+ export async function cmdInfo(args: ParsedArgs, io: CliIO): Promise<number> {
90
+ const target = args.positionals[0];
91
+ if (!target) throw new CliUsageError("info requires a <file.yom>");
92
+ const bytes = await loadLocalBytes(target);
93
+ const reader = await YOMPackfileReader.open(bytes);
94
+ const header = parseHeaderFields(bytes.subarray(0, YOM_HEADER_SIZE));
95
+ const index = reader.mainIndex;
96
+
97
+ const info = {
98
+ magic: new TextDecoder().decode(header.magic),
99
+ formatVersion: header.formatVersion,
100
+ fileSize: bytes.length,
101
+ indexOffset: Number(header.indexOffset),
102
+ indexLength: Number(header.indexUncompressedLength),
103
+ pageCount: reader.pageCount,
104
+ blobCount: index.blobsLength(),
105
+ title: reader.title,
106
+ hasOcr: index.ocrLayoutBlobIndex() >= 0,
107
+ hasNormalizedText: index.normalizedTextBlobIndex() >= 0,
108
+ hasSearch: index.searchDictionaryBlobIndex() >= 0,
109
+ searchTermCount: index.searchTermsLength(),
110
+ };
111
+
112
+ if (args.json) {
113
+ io.out(JSON.stringify(info, null, 2));
114
+ } else {
115
+ io.out(`magic: ${info.magic}`);
116
+ io.out(`formatVersion: ${info.formatVersion}`);
117
+ io.out(`fileSize: ${info.fileSize}`);
118
+ io.out(`indexOffset: ${info.indexOffset}`);
119
+ io.out(`indexLength: ${info.indexLength}`);
120
+ io.out(`title: ${info.title}`);
121
+ io.out(`pageCount: ${info.pageCount}`);
122
+ io.out(`blobCount: ${info.blobCount}`);
123
+ io.out(`hasOcr: ${info.hasOcr}`);
124
+ io.out(`hasNormalizedText:${info.hasNormalizedText}`);
125
+ io.out(`hasSearch: ${info.hasSearch}`);
126
+ io.out(`searchTermCount: ${info.searchTermCount}`);
127
+ }
128
+ return 0;
129
+ }
130
+
131
+ /** `verify` -- full validation for a local file; per-blob ranged verify for a URL. */
132
+ export async function cmdVerify(args: ParsedArgs, io: CliIO): Promise<number> {
133
+ const target = args.positionals[0];
134
+ if (!target) throw new CliUsageError("verify requires a <file.yom | url>");
135
+
136
+ try {
137
+ if (isUrl(target)) {
138
+ // Exercise the ranged reader end-to-end: open reads header+index (with
139
+ // bounded index verify), then each blob read verifies its own SHA-256.
140
+ const reader = await YOMPackfileReader.openURL(target, { fallbackToFullFetch: true });
141
+ for (let i = 0; i < reader.mainIndex.blobsLength(); i++) {
142
+ await reader.readBlob(i);
143
+ }
144
+ report(io, args.json, { ok: true, target, mode: "range-per-blob" });
145
+ } else {
146
+ const bytes = await loadLocalBytes(target);
147
+ await validateYOM(bytes);
148
+ report(io, args.json, { ok: true, target, mode: "full" });
149
+ }
150
+ return 0;
151
+ } catch (err) {
152
+ const reason = err instanceof Error ? err.message : String(err);
153
+ report(io, args.json, { ok: false, target, reason });
154
+ return 1;
155
+ }
156
+ }
157
+
158
+ /** `ls` -- per-page index, label, dimensions, and page-image blob size. */
159
+ export async function cmdLs(args: ParsedArgs, io: CliIO): Promise<number> {
160
+ const target = args.positionals[0];
161
+ if (!target) throw new CliUsageError("ls requires a <file.yom>");
162
+ const bytes = await loadLocalBytes(target);
163
+ const reader = await YOMPackfileReader.open(bytes);
164
+ const index = reader.mainIndex;
165
+
166
+ const rows = [];
167
+ for (let i = 0; i < reader.pageCount; i++) {
168
+ const page = reader.page(i);
169
+ const imageBlob = index.blobs(page.pageImageBlobIndex());
170
+ rows.push({
171
+ page: i,
172
+ label: page.pageLabel() ?? "",
173
+ pixelWidth: page.pixelWidth(),
174
+ pixelHeight: page.pixelHeight(),
175
+ imageBlobIndex: page.pageImageBlobIndex(),
176
+ imageBlobBytes: imageBlob ? Number(imageBlob.compressedLength()) : 0,
177
+ });
178
+ }
179
+ const hasOcr = index.ocrLayoutBlobIndex() >= 0;
180
+ const hasNormalizedText = index.normalizedTextBlobIndex() >= 0;
181
+
182
+ if (args.json) {
183
+ io.out(JSON.stringify({ pageCount: reader.pageCount, hasOcr, hasNormalizedText, pages: rows }, null, 2));
184
+ } else {
185
+ io.out(`pages: ${reader.pageCount} hasOcr: ${hasOcr} hasNormalizedText: ${hasNormalizedText}`);
186
+ io.out("idx label WxH imageBytes");
187
+ for (const r of rows) {
188
+ io.out(
189
+ `${String(r.page).padEnd(4)} ${r.label.padEnd(16)} ${`${r.pixelWidth}x${r.pixelHeight}`.padEnd(13)} ${r.imageBlobBytes}`,
190
+ );
191
+ }
192
+ }
193
+ return 0;
194
+ }
195
+
196
+ /** `extract` -- write one blob (page image/thumbnail, or file-wide ocr/text) to disk. */
197
+ export async function cmdExtract(args: ParsedArgs, io: CliIO): Promise<number> {
198
+ const target = args.positionals[0];
199
+ if (!target) throw new CliUsageError("extract requires a <file.yom>");
200
+ if (!args.kind) throw new CliUsageError("extract requires --kind image|thumb|ocr|text");
201
+ if (!args.out) throw new CliUsageError("extract requires -o <out>");
202
+ const bytes = await loadLocalBytes(target);
203
+ const reader = await YOMPackfileReader.open(bytes);
204
+ const index = reader.mainIndex;
205
+
206
+ let data: Uint8Array;
207
+ switch (args.kind) {
208
+ case "image":
209
+ requirePage(args);
210
+ data = await reader.pageImage(args.page!);
211
+ break;
212
+ case "thumb":
213
+ requirePage(args);
214
+ data = await reader.thumbnailImage(args.page!);
215
+ break;
216
+ case "ocr":
217
+ data = await readFileWideBlob(reader, index.ocrLayoutBlobIndex(), "ocr");
218
+ break;
219
+ case "text":
220
+ data = await readFileWideBlob(reader, index.normalizedTextBlobIndex(), "normalized text");
221
+ break;
222
+ default:
223
+ throw new CliUsageError(`unknown --kind ${args.kind} (expected image|thumb|ocr|text)`);
224
+ }
225
+
226
+ await writeFile(args.out, data);
227
+ io.out(`wrote ${data.length} bytes to ${args.out}`);
228
+ return 0;
229
+ }
230
+
231
+ /** `meta` -- book metadata + page labels + outline (when present) as JSON. */
232
+ export async function cmdMeta(args: ParsedArgs, io: CliIO): Promise<number> {
233
+ const target = args.positionals[0];
234
+ if (!target) throw new CliUsageError("meta requires a <file.yom>");
235
+ const bytes = await loadLocalBytes(target);
236
+ const reader = await YOMPackfileReader.open(bytes);
237
+ const index = reader.mainIndex;
238
+ const book = index.book();
239
+
240
+ const pageLabels: string[] = [];
241
+ for (let i = 0; i < reader.pageCount; i++) {
242
+ pageLabels.push(reader.page(i).pageLabel() ?? "");
243
+ }
244
+
245
+ const header = parseHeaderFields(bytes.subarray(0, YOM_HEADER_SIZE));
246
+ const indexOffset = Number(header.indexOffset);
247
+ const indexBytes = bytes.subarray(indexOffset, indexOffset + Number(header.indexUncompressedLength));
248
+
249
+ const meta = {
250
+ title: reader.title,
251
+ pageCount: book ? book.pageCount() : reader.pageCount,
252
+ readingDirection: book ? readingDirectionName(book.readingDirection()) : null,
253
+ createdAtUnix: book ? book.createdAtUnix().toString() : null,
254
+ pageLabels,
255
+ outline: pdfExportPresent(indexBytes) ? readOutline(index) : [],
256
+ };
257
+
258
+ io.out(JSON.stringify(meta, null, 2));
259
+ return 0;
260
+ }
261
+
262
+ interface OutlineNode {
263
+ title: string;
264
+ pageId: number;
265
+ children: OutlineNode[];
266
+ }
267
+
268
+ function readOutline(index: MainIndex): OutlineNode[] {
269
+ const pdf = index.pdfExport();
270
+ if (!pdf) return [];
271
+ const nodes: OutlineNode[] = [];
272
+ for (let i = 0; i < pdf.outlineLength(); i++) {
273
+ const entry = pdf.outline(i);
274
+ if (entry) nodes.push(readOutlineEntry(entry));
275
+ }
276
+ return nodes;
277
+ }
278
+
279
+ function readOutlineEntry(entry: OutlineEntry): OutlineNode {
280
+ const children: OutlineNode[] = [];
281
+ for (let i = 0; i < entry.childrenLength(); i++) {
282
+ const child = entry.children(i);
283
+ if (child) children.push(readOutlineEntry(child));
284
+ }
285
+ return { title: entry.title() ?? "", pageId: entry.pageId(), children };
286
+ }
287
+
288
+ function requirePage(args: ParsedArgs): void {
289
+ if (args.page === undefined || !Number.isInteger(args.page) || args.page < 0) {
290
+ throw new CliUsageError("extract --kind image|thumb requires --page N (a non-negative integer)");
291
+ }
292
+ }
293
+
294
+ async function readFileWideBlob(reader: YOMPackfileReader, blobIndex: number, label: string): Promise<Uint8Array> {
295
+ if (blobIndex < 0) throw new CliUsageError(`this file has no ${label} blob`);
296
+ return reader.readBlob(blobIndex);
297
+ }
298
+
299
+ function report(io: CliIO, json: boolean, payload: Record<string, unknown>): void {
300
+ if (json) {
301
+ io.out(JSON.stringify(payload));
302
+ return;
303
+ }
304
+ if (payload.ok) io.out(`OK ${payload.target}${payload.mode ? ` (${payload.mode})` : ""}`);
305
+ else io.err(`FAIL ${payload.target}: ${payload.reason}`);
306
+ }
307
+
308
+ /**
309
+ * Dispatches a parsed argv to a command handler. Returns the process exit
310
+ * code (0 ok, non-zero on failure / bad args). Never throws for usage
311
+ * errors -- they are reported and mapped to exit code 2.
312
+ */
313
+ export async function runCli(argv: string[], io: CliIO): Promise<number> {
314
+ const command = argv[0];
315
+ if (!command || command === "--help" || command === "-h" || command === "help") {
316
+ io.out(USAGE);
317
+ return command ? 0 : 2;
318
+ }
319
+
320
+ let args: ParsedArgs;
321
+ try {
322
+ args = parseArgs(argv.slice(1));
323
+ } catch (err) {
324
+ io.err(err instanceof Error ? err.message : String(err));
325
+ io.err(USAGE);
326
+ return 2;
327
+ }
328
+
329
+ try {
330
+ switch (command) {
331
+ case "info":
332
+ return await cmdInfo(args, io);
333
+ case "verify":
334
+ return await cmdVerify(args, io);
335
+ case "ls":
336
+ return await cmdLs(args, io);
337
+ case "extract":
338
+ return await cmdExtract(args, io);
339
+ case "meta":
340
+ return await cmdMeta(args, io);
341
+ case "convert":
342
+ io.err("PDF -> .yom conversion is browser-only; use the web convert tool or the iOS app.");
343
+ return 2;
344
+ default:
345
+ io.err(`unknown command: ${command}`);
346
+ io.err(USAGE);
347
+ return 2;
348
+ }
349
+ } catch (err) {
350
+ if (err instanceof CliUsageError) {
351
+ io.err(err.message);
352
+ return 2;
353
+ }
354
+ io.err(err instanceof Error ? err.message : String(err));
355
+ return 1;
356
+ }
357
+ }
358
+
359
+ const consoleIO: CliIO = {
360
+ out: (line) => process.stdout.write(`${line}\n`),
361
+ err: (line) => process.stderr.write(`${line}\n`),
362
+ };
363
+
364
+ const invokedDirectly = process.argv[1] !== undefined && import.meta.url === pathToFileURL(process.argv[1]).href;
365
+ if (invokedDirectly) {
366
+ runCli(process.argv.slice(2), consoleIO)
367
+ .then((code) => process.exit(code))
368
+ .catch((err) => {
369
+ consoleIO.err(err instanceof Error ? err.message : String(err));
370
+ process.exit(1);
371
+ });
372
+ }