yom-format 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +86 -0
- package/dist/cli.d.ts +30 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +344 -0
- package/dist/cli.js.map +1 -0
- package/dist/index.d.ts +36 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +25 -0
- package/dist/index.js.map +1 -0
- package/dist/pipeline/avif.d.ts +3 -0
- package/dist/pipeline/avif.d.ts.map +1 -0
- package/dist/pipeline/avif.js +30 -0
- package/dist/pipeline/avif.js.map +1 -0
- package/dist/pipeline/convert.d.ts +28 -0
- package/dist/pipeline/convert.d.ts.map +1 -0
- package/dist/pipeline/convert.js +111 -0
- package/dist/pipeline/convert.js.map +1 -0
- package/dist/pipeline/index.d.ts +17 -0
- package/dist/pipeline/index.d.ts.map +1 -0
- package/dist/pipeline/index.js +17 -0
- package/dist/pipeline/index.js.map +1 -0
- package/dist/pipeline/ocrLayoutBuilder.d.ts +10 -0
- package/dist/pipeline/ocrLayoutBuilder.d.ts.map +1 -0
- package/dist/pipeline/ocrLayoutBuilder.js +62 -0
- package/dist/pipeline/ocrLayoutBuilder.js.map +1 -0
- package/dist/pipeline/pdfDocument.d.ts +12 -0
- package/dist/pipeline/pdfDocument.d.ts.map +1 -0
- package/dist/pipeline/pdfDocument.js +76 -0
- package/dist/pipeline/pdfDocument.js.map +1 -0
- package/dist/pipeline/renderPage.d.ts +10 -0
- package/dist/pipeline/renderPage.d.ts.map +1 -0
- package/dist/pipeline/renderPage.js +36 -0
- package/dist/pipeline/renderPage.js.map +1 -0
- package/dist/pipeline/textExtract.d.ts +29 -0
- package/dist/pipeline/textExtract.d.ts.map +1 -0
- package/dist/pipeline/textExtract.js +48 -0
- package/dist/pipeline/textExtract.js.map +1 -0
- package/dist/yom/blobLimits.d.ts +15 -0
- package/dist/yom/blobLimits.d.ts.map +1 -0
- package/dist/yom/blobLimits.js +25 -0
- package/dist/yom/blobLimits.js.map +1 -0
- package/dist/yom/byteSource.d.ts +56 -0
- package/dist/yom/byteSource.d.ts.map +1 -0
- package/dist/yom/byteSource.js +111 -0
- package/dist/yom/byteSource.js.map +1 -0
- package/dist/yom/bytes.d.ts +2 -0
- package/dist/yom/bytes.d.ts.map +1 -0
- package/dist/yom/bytes.js +13 -0
- package/dist/yom/bytes.js.map +1 -0
- package/dist/yom/constants.d.ts +34 -0
- package/dist/yom/constants.d.ts.map +1 -0
- package/dist/yom/constants.js +82 -0
- package/dist/yom/constants.js.map +1 -0
- package/dist/yom/errors.d.ts +44 -0
- package/dist/yom/errors.d.ts.map +1 -0
- package/dist/yom/errors.js +44 -0
- package/dist/yom/errors.js.map +1 -0
- package/dist/yom/flatbufferVerify.d.ts +21 -0
- package/dist/yom/flatbufferVerify.d.ts.map +1 -0
- package/dist/yom/flatbufferVerify.js +255 -0
- package/dist/yom/flatbufferVerify.js.map +1 -0
- package/dist/yom/generated/yomu/yom/blob-kind.d.ts +9 -0
- package/dist/yom/generated/yomu/yom/blob-kind.d.ts.map +1 -0
- package/dist/yom/generated/yomu/yom/blob-kind.js +12 -0
- package/dist/yom/generated/yomu/yom/blob-kind.js.map +1 -0
- package/dist/yom/generated/yomu/yom/blob-ref.d.ts +45 -0
- package/dist/yom/generated/yomu/yom/blob-ref.d.ts.map +1 -0
- package/dist/yom/generated/yomu/yom/blob-ref.js +141 -0
- package/dist/yom/generated/yomu/yom/blob-ref.js.map +1 -0
- package/dist/yom/generated/yomu/yom/book-metadata.d.ts +32 -0
- package/dist/yom/generated/yomu/yom/book-metadata.d.ts.map +1 -0
- package/dist/yom/generated/yomu/yom/book-metadata.js +89 -0
- package/dist/yom/generated/yomu/yom/book-metadata.js.map +1 -0
- package/dist/yom/generated/yomu/yom/compression.d.ts +5 -0
- package/dist/yom/generated/yomu/yom/compression.d.ts.map +1 -0
- package/dist/yom/generated/yomu/yom/compression.js +8 -0
- package/dist/yom/generated/yomu/yom/compression.js.map +1 -0
- package/dist/yom/generated/yomu/yom/main-index.d.ts +63 -0
- package/dist/yom/generated/yomu/yom/main-index.d.ts.map +1 -0
- package/dist/yom/generated/yomu/yom/main-index.js +208 -0
- package/dist/yom/generated/yomu/yom/main-index.js.map +1 -0
- package/dist/yom/generated/yomu/yom/ocrblock.d.ts +32 -0
- package/dist/yom/generated/yomu/yom/ocrblock.d.ts.map +1 -0
- package/dist/yom/generated/yomu/yom/ocrblock.js +84 -0
- package/dist/yom/generated/yomu/yom/ocrblock.js.map +1 -0
- package/dist/yom/generated/yomu/yom/ocrlayout.d.ts +27 -0
- package/dist/yom/generated/yomu/yom/ocrlayout.d.ts.map +1 -0
- package/dist/yom/generated/yomu/yom/ocrlayout.js +76 -0
- package/dist/yom/generated/yomu/yom/ocrlayout.js.map +1 -0
- package/dist/yom/generated/yomu/yom/ocrline.d.ts +35 -0
- package/dist/yom/generated/yomu/yom/ocrline.d.ts.map +1 -0
- package/dist/yom/generated/yomu/yom/ocrline.js +95 -0
- package/dist/yom/generated/yomu/yom/ocrline.js.map +1 -0
- package/dist/yom/generated/yomu/yom/ocrpage.d.ts +46 -0
- package/dist/yom/generated/yomu/yom/ocrpage.d.ts.map +1 -0
- package/dist/yom/generated/yomu/yom/ocrpage.js +147 -0
- package/dist/yom/generated/yomu/yom/ocrpage.js.map +1 -0
- package/dist/yom/generated/yomu/yom/ocrtoken.d.ts +44 -0
- package/dist/yom/generated/yomu/yom/ocrtoken.d.ts.map +1 -0
- package/dist/yom/generated/yomu/yom/ocrtoken.js +128 -0
- package/dist/yom/generated/yomu/yom/ocrtoken.js.map +1 -0
- package/dist/yom/generated/yomu/yom/outline-entry.d.ts +31 -0
- package/dist/yom/generated/yomu/yom/outline-entry.d.ts.map +1 -0
- package/dist/yom/generated/yomu/yom/outline-entry.js +92 -0
- package/dist/yom/generated/yomu/yom/outline-entry.js.map +1 -0
- package/dist/yom/generated/yomu/yom/page-entry.d.ts +95 -0
- package/dist/yom/generated/yomu/yom/page-entry.d.ts.map +1 -0
- package/dist/yom/generated/yomu/yom/page-entry.js +288 -0
- package/dist/yom/generated/yomu/yom/page-entry.js.map +1 -0
- package/dist/yom/generated/yomu/yom/pdfexport-metadata.d.ts +32 -0
- package/dist/yom/generated/yomu/yom/pdfexport-metadata.d.ts.map +1 -0
- package/dist/yom/generated/yomu/yom/pdfexport-metadata.js +96 -0
- package/dist/yom/generated/yomu/yom/pdfexport-metadata.js.map +1 -0
- package/dist/yom/generated/yomu/yom/reading-direction.d.ts +5 -0
- package/dist/yom/generated/yomu/yom/reading-direction.d.ts.map +1 -0
- package/dist/yom/generated/yomu/yom/reading-direction.js +8 -0
- package/dist/yom/generated/yomu/yom/reading-direction.js.map +1 -0
- package/dist/yom/generated/yomu/yom/search-term-ref.d.ts +34 -0
- package/dist/yom/generated/yomu/yom/search-term-ref.d.ts.map +1 -0
- package/dist/yom/generated/yomu/yom/search-term-ref.js +99 -0
- package/dist/yom/generated/yomu/yom/search-term-ref.js.map +1 -0
- package/dist/yom/generated/yomu/yom.d.ts +6 -0
- package/dist/yom/generated/yomu/yom.d.ts.map +1 -0
- package/dist/yom/generated/yomu/yom.js +8 -0
- package/dist/yom/generated/yomu/yom.js.map +1 -0
- package/dist/yom/hash.d.ts +4 -0
- package/dist/yom/hash.d.ts.map +1 -0
- package/dist/yom/hash.js +21 -0
- package/dist/yom/hash.js.map +1 -0
- package/dist/yom/header.d.ts +44 -0
- package/dist/yom/header.d.ts.map +1 -0
- package/dist/yom/header.js +115 -0
- package/dist/yom/header.js.map +1 -0
- package/dist/yom/reader.d.ts +57 -0
- package/dist/yom/reader.d.ts.map +1 -0
- package/dist/yom/reader.js +141 -0
- package/dist/yom/reader.js.map +1 -0
- package/dist/yom/searchIndex.d.ts +41 -0
- package/dist/yom/searchIndex.d.ts.map +1 -0
- package/dist/yom/searchIndex.js +133 -0
- package/dist/yom/searchIndex.js.map +1 -0
- package/dist/yom/textNormalizer.d.ts +19 -0
- package/dist/yom/textNormalizer.d.ts.map +1 -0
- package/dist/yom/textNormalizer.js +77 -0
- package/dist/yom/textNormalizer.js.map +1 -0
- package/dist/yom/validator.d.ts +14 -0
- package/dist/yom/validator.d.ts.map +1 -0
- package/dist/yom/validator.js +197 -0
- package/dist/yom/validator.js.map +1 -0
- package/dist/yom/writer.d.ts +76 -0
- package/dist/yom/writer.d.ts.map +1 -0
- package/dist/yom/writer.js +154 -0
- package/dist/yom/writer.js.map +1 -0
- package/dist/yom/zstd.d.ts +14 -0
- package/dist/yom/zstd.d.ts.map +1 -0
- package/dist/yom/zstd.js +43 -0
- package/dist/yom/zstd.js.map +1 -0
- package/package.json +65 -0
- package/src/cli.ts +372 -0
- package/src/index.ts +109 -0
- package/src/pipeline/avif.ts +33 -0
- package/src/pipeline/convert.ts +148 -0
- package/src/pipeline/index.ts +28 -0
- package/src/pipeline/ocrLayoutBuilder.ts +73 -0
- package/src/pipeline/pdfDocument.ts +86 -0
- package/src/pipeline/renderPage.ts +48 -0
- package/src/pipeline/textExtract.ts +64 -0
- package/src/worker/convert.worker.ts +54 -0
- package/src/yom/blobLimits.ts +30 -0
- package/src/yom/byteSource.ts +144 -0
- package/src/yom/bytes.ts +11 -0
- package/src/yom/constants.ts +89 -0
- package/src/yom/errors.ts +43 -0
- package/src/yom/flatbufferVerify.ts +298 -0
- package/src/yom/generated/yomu/yom/blob-kind.ts +12 -0
- package/src/yom/generated/yomu/yom/blob-ref.ts +184 -0
- package/src/yom/generated/yomu/yom/book-metadata.ts +121 -0
- package/src/yom/generated/yomu/yom/compression.ts +8 -0
- package/src/yom/generated/yomu/yom/main-index.ts +257 -0
- package/src/yom/generated/yomu/yom/ocrblock.ts +117 -0
- package/src/yom/generated/yomu/yom/ocrlayout.ts +102 -0
- package/src/yom/generated/yomu/yom/ocrline.ts +131 -0
- package/src/yom/generated/yomu/yom/ocrpage.ts +191 -0
- package/src/yom/generated/yomu/yom/ocrtoken.ts +173 -0
- package/src/yom/generated/yomu/yom/outline-entry.ts +124 -0
- package/src/yom/generated/yomu/yom/page-entry.ts +388 -0
- package/src/yom/generated/yomu/yom/pdfexport-metadata.ts +128 -0
- package/src/yom/generated/yomu/yom/reading-direction.ts +8 -0
- package/src/yom/generated/yomu/yom/search-term-ref.ts +134 -0
- package/src/yom/generated/yomu/yom.ts +9 -0
- package/src/yom/hash.ts +20 -0
- package/src/yom/header.ts +132 -0
- package/src/yom/reader.ts +163 -0
- package/src/yom/searchIndex.ts +178 -0
- package/src/yom/textNormalizer.ts +92 -0
- package/src/yom/validator.ts +245 -0
- package/src/yom/writer.ts +337 -0
- package/src/yom/zstd.ts +47 -0
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import * as flatbuffers from 'flatbuffers';
|
|
2
|
+
export declare class SearchTermRef implements flatbuffers.IUnpackableObject<SearchTermRefT> {
|
|
3
|
+
bb: flatbuffers.ByteBuffer | null;
|
|
4
|
+
bb_pos: number;
|
|
5
|
+
__init(i: number, bb: flatbuffers.ByteBuffer): SearchTermRef;
|
|
6
|
+
static getRootAsSearchTermRef(bb: flatbuffers.ByteBuffer, obj?: SearchTermRef): SearchTermRef;
|
|
7
|
+
static getSizePrefixedRootAsSearchTermRef(bb: flatbuffers.ByteBuffer, obj?: SearchTermRef): SearchTermRef;
|
|
8
|
+
term(): string | null;
|
|
9
|
+
term(optionalEncoding: flatbuffers.Encoding): string | Uint8Array | null;
|
|
10
|
+
dictionaryOffset(): bigint;
|
|
11
|
+
dictionaryLength(): bigint;
|
|
12
|
+
postingsOffset(): bigint;
|
|
13
|
+
postingsLength(): bigint;
|
|
14
|
+
static startSearchTermRef(builder: flatbuffers.Builder): void;
|
|
15
|
+
static addTerm(builder: flatbuffers.Builder, termOffset: flatbuffers.Offset): void;
|
|
16
|
+
static addDictionaryOffset(builder: flatbuffers.Builder, dictionaryOffset: bigint): void;
|
|
17
|
+
static addDictionaryLength(builder: flatbuffers.Builder, dictionaryLength: bigint): void;
|
|
18
|
+
static addPostingsOffset(builder: flatbuffers.Builder, postingsOffset: bigint): void;
|
|
19
|
+
static addPostingsLength(builder: flatbuffers.Builder, postingsLength: bigint): void;
|
|
20
|
+
static endSearchTermRef(builder: flatbuffers.Builder): flatbuffers.Offset;
|
|
21
|
+
static createSearchTermRef(builder: flatbuffers.Builder, termOffset: flatbuffers.Offset, dictionaryOffset: bigint, dictionaryLength: bigint, postingsOffset: bigint, postingsLength: bigint): flatbuffers.Offset;
|
|
22
|
+
unpack(): SearchTermRefT;
|
|
23
|
+
unpackTo(_o: SearchTermRefT): void;
|
|
24
|
+
}
|
|
25
|
+
export declare class SearchTermRefT implements flatbuffers.IGeneratedObject {
|
|
26
|
+
term: string | Uint8Array | null;
|
|
27
|
+
dictionaryOffset: bigint;
|
|
28
|
+
dictionaryLength: bigint;
|
|
29
|
+
postingsOffset: bigint;
|
|
30
|
+
postingsLength: bigint;
|
|
31
|
+
constructor(term?: string | Uint8Array | null, dictionaryOffset?: bigint, dictionaryLength?: bigint, postingsOffset?: bigint, postingsLength?: bigint);
|
|
32
|
+
pack(builder: flatbuffers.Builder): flatbuffers.Offset;
|
|
33
|
+
}
|
|
34
|
+
//# sourceMappingURL=search-term-ref.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"search-term-ref.d.ts","sourceRoot":"","sources":["../../../../../src/yom/generated/yomu/yom/search-term-ref.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,WAAW,MAAM,aAAa,CAAC;AAI3C,qBAAa,aAAc,YAAW,WAAW,CAAC,iBAAiB,CAAC,cAAc,CAAC;IACjF,EAAE,EAAE,WAAW,CAAC,UAAU,GAAC,IAAI,CAAQ;IACvC,MAAM,SAAK;IACX,MAAM,CAAC,CAAC,EAAC,MAAM,EAAE,EAAE,EAAC,WAAW,CAAC,UAAU,GAAE,aAAa;IAM3D,MAAM,CAAC,sBAAsB,CAAC,EAAE,EAAC,WAAW,CAAC,UAAU,EAAE,GAAG,CAAC,EAAC,aAAa,GAAE,aAAa;IAI1F,MAAM,CAAC,kCAAkC,CAAC,EAAE,EAAC,WAAW,CAAC,UAAU,EAAE,GAAG,CAAC,EAAC,aAAa,GAAE,aAAa;IAKtG,IAAI,IAAG,MAAM,GAAC,IAAI;IAClB,IAAI,CAAC,gBAAgB,EAAC,WAAW,CAAC,QAAQ,GAAE,MAAM,GAAC,UAAU,GAAC,IAAI;IAMlE,gBAAgB,IAAG,MAAM;IAKzB,gBAAgB,IAAG,MAAM;IAKzB,cAAc,IAAG,MAAM;IAKvB,cAAc,IAAG,MAAM;IAKvB,MAAM,CAAC,kBAAkB,CAAC,OAAO,EAAC,WAAW,CAAC,OAAO;IAIrD,MAAM,CAAC,OAAO,CAAC,OAAO,EAAC,WAAW,CAAC,OAAO,EAAE,UAAU,EAAC,WAAW,CAAC,MAAM;IAIzE,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAC,WAAW,CAAC,OAAO,EAAE,gBAAgB,EAAC,MAAM;IAI/E,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAC,WAAW,CAAC,OAAO,EAAE,gBAAgB,EAAC,MAAM;IAI/E,MAAM,CAAC,iBAAiB,CAAC,OAAO,EAAC,WAAW,CAAC,OAAO,EAAE,cAAc,EAAC,MAAM;IAI3E,MAAM,CAAC,iBAAiB,CAAC,OAAO,EAAC,WAAW,CAAC,OAAO,EAAE,cAAc,EAAC,MAAM;IAI3E,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAC,WAAW,CAAC,OAAO,GAAE,WAAW,CAAC,MAAM;IAKvE,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAC,WAAW,CAAC,OAAO,EAAE,UAAU,EAAC,WAAW,CAAC,MAAM,EAAE,gBAAgB,EAAC,MAAM,EAAE,gBAAgB,EAAC,MAAM,EAAE,cAAc,EAAC,MAAM,EAAE,cAAc,EAAC,MAAM,GAAE,WAAW,CAAC,MAAM;IAUzM,MAAM,IAAI,cAAc;IAWxB,QAAQ,CAAC,EAAE,EAAE,cAAc,GAAG,IAAI;CAOjC;AAED,qBAAa,cAAe,YAAW,WAAW,CAAC,gBAAgB;IAE1D,IAAI,EAAE,MAAM,GAAC,UAAU,GAAC,IAAI;IAC5B,gBAAgB,EAAE,MAAM;IACxB,gBAAgB,EAAE,MAAM;IACxB,cAAc,EAAE,MAAM;IACtB,cAAc,EAAE,MAAM;gBAJtB,IAAI,GAAE,MAAM,GAAC,UAAU,GAAC,IAAW,EACnC,gBAAgB,GAAE,MAAoB,EACtC,gBAAgB,GAAE,MAAoB,EACtC,cAAc,GAAE,MAAoB,EACpC,cAAc,GAAE,MAAoB;IAI7C,IAAI,CAAC,OAAO,EAAC,WAAW,CAAC,OAAO,GAAG,WAAW,CAAC,MAAM;CAWpD"}
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
// automatically generated by the FlatBuffers compiler, do not modify
|
|
2
|
+
/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
|
|
3
|
+
import * as flatbuffers from 'flatbuffers';
|
|
4
|
+
export class SearchTermRef {
|
|
5
|
+
bb = null;
|
|
6
|
+
bb_pos = 0;
|
|
7
|
+
__init(i, bb) {
|
|
8
|
+
this.bb_pos = i;
|
|
9
|
+
this.bb = bb;
|
|
10
|
+
return this;
|
|
11
|
+
}
|
|
12
|
+
static getRootAsSearchTermRef(bb, obj) {
|
|
13
|
+
return (obj || new SearchTermRef()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
|
|
14
|
+
}
|
|
15
|
+
static getSizePrefixedRootAsSearchTermRef(bb, obj) {
|
|
16
|
+
bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH);
|
|
17
|
+
return (obj || new SearchTermRef()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
|
|
18
|
+
}
|
|
19
|
+
term(optionalEncoding) {
|
|
20
|
+
const offset = this.bb.__offset(this.bb_pos, 4);
|
|
21
|
+
return offset ? this.bb.__string(this.bb_pos + offset, optionalEncoding) : null;
|
|
22
|
+
}
|
|
23
|
+
dictionaryOffset() {
|
|
24
|
+
const offset = this.bb.__offset(this.bb_pos, 6);
|
|
25
|
+
return offset ? this.bb.readUint64(this.bb_pos + offset) : BigInt('0');
|
|
26
|
+
}
|
|
27
|
+
dictionaryLength() {
|
|
28
|
+
const offset = this.bb.__offset(this.bb_pos, 8);
|
|
29
|
+
return offset ? this.bb.readUint64(this.bb_pos + offset) : BigInt('0');
|
|
30
|
+
}
|
|
31
|
+
postingsOffset() {
|
|
32
|
+
const offset = this.bb.__offset(this.bb_pos, 10);
|
|
33
|
+
return offset ? this.bb.readUint64(this.bb_pos + offset) : BigInt('0');
|
|
34
|
+
}
|
|
35
|
+
postingsLength() {
|
|
36
|
+
const offset = this.bb.__offset(this.bb_pos, 12);
|
|
37
|
+
return offset ? this.bb.readUint64(this.bb_pos + offset) : BigInt('0');
|
|
38
|
+
}
|
|
39
|
+
static startSearchTermRef(builder) {
|
|
40
|
+
builder.startObject(5);
|
|
41
|
+
}
|
|
42
|
+
static addTerm(builder, termOffset) {
|
|
43
|
+
builder.addFieldOffset(0, termOffset, 0);
|
|
44
|
+
}
|
|
45
|
+
static addDictionaryOffset(builder, dictionaryOffset) {
|
|
46
|
+
builder.addFieldInt64(1, dictionaryOffset, BigInt('0'));
|
|
47
|
+
}
|
|
48
|
+
static addDictionaryLength(builder, dictionaryLength) {
|
|
49
|
+
builder.addFieldInt64(2, dictionaryLength, BigInt('0'));
|
|
50
|
+
}
|
|
51
|
+
static addPostingsOffset(builder, postingsOffset) {
|
|
52
|
+
builder.addFieldInt64(3, postingsOffset, BigInt('0'));
|
|
53
|
+
}
|
|
54
|
+
static addPostingsLength(builder, postingsLength) {
|
|
55
|
+
builder.addFieldInt64(4, postingsLength, BigInt('0'));
|
|
56
|
+
}
|
|
57
|
+
static endSearchTermRef(builder) {
|
|
58
|
+
const offset = builder.endObject();
|
|
59
|
+
return offset;
|
|
60
|
+
}
|
|
61
|
+
static createSearchTermRef(builder, termOffset, dictionaryOffset, dictionaryLength, postingsOffset, postingsLength) {
|
|
62
|
+
SearchTermRef.startSearchTermRef(builder);
|
|
63
|
+
SearchTermRef.addTerm(builder, termOffset);
|
|
64
|
+
SearchTermRef.addDictionaryOffset(builder, dictionaryOffset);
|
|
65
|
+
SearchTermRef.addDictionaryLength(builder, dictionaryLength);
|
|
66
|
+
SearchTermRef.addPostingsOffset(builder, postingsOffset);
|
|
67
|
+
SearchTermRef.addPostingsLength(builder, postingsLength);
|
|
68
|
+
return SearchTermRef.endSearchTermRef(builder);
|
|
69
|
+
}
|
|
70
|
+
unpack() {
|
|
71
|
+
return new SearchTermRefT(this.term(), this.dictionaryOffset(), this.dictionaryLength(), this.postingsOffset(), this.postingsLength());
|
|
72
|
+
}
|
|
73
|
+
unpackTo(_o) {
|
|
74
|
+
_o.term = this.term();
|
|
75
|
+
_o.dictionaryOffset = this.dictionaryOffset();
|
|
76
|
+
_o.dictionaryLength = this.dictionaryLength();
|
|
77
|
+
_o.postingsOffset = this.postingsOffset();
|
|
78
|
+
_o.postingsLength = this.postingsLength();
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
export class SearchTermRefT {
|
|
82
|
+
term;
|
|
83
|
+
dictionaryOffset;
|
|
84
|
+
dictionaryLength;
|
|
85
|
+
postingsOffset;
|
|
86
|
+
postingsLength;
|
|
87
|
+
constructor(term = null, dictionaryOffset = BigInt('0'), dictionaryLength = BigInt('0'), postingsOffset = BigInt('0'), postingsLength = BigInt('0')) {
|
|
88
|
+
this.term = term;
|
|
89
|
+
this.dictionaryOffset = dictionaryOffset;
|
|
90
|
+
this.dictionaryLength = dictionaryLength;
|
|
91
|
+
this.postingsOffset = postingsOffset;
|
|
92
|
+
this.postingsLength = postingsLength;
|
|
93
|
+
}
|
|
94
|
+
pack(builder) {
|
|
95
|
+
const term = (this.term !== null ? builder.createString(this.term) : 0);
|
|
96
|
+
return SearchTermRef.createSearchTermRef(builder, term, this.dictionaryOffset, this.dictionaryLength, this.postingsOffset, this.postingsLength);
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
//# sourceMappingURL=search-term-ref.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"search-term-ref.js","sourceRoot":"","sources":["../../../../../src/yom/generated/yomu/yom/search-term-ref.ts"],"names":[],"mappings":"AAAA,qEAAqE;AAErE,oIAAoI;AAEpI,OAAO,KAAK,WAAW,MAAM,aAAa,CAAC;AAI3C,MAAM,OAAO,aAAa;IACxB,EAAE,GAAgC,IAAI,CAAC;IACvC,MAAM,GAAG,CAAC,CAAC;IACX,MAAM,CAAC,CAAQ,EAAE,EAAyB;QAC1C,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;QAChB,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,CAAC,sBAAsB,CAAC,EAAyB,EAAE,GAAkB;QACzE,OAAO,CAAC,GAAG,IAAI,IAAI,aAAa,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,SAAS,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;IAC9F,CAAC;IAED,MAAM,CAAC,kCAAkC,CAAC,EAAyB,EAAE,GAAkB;QACrF,EAAE,CAAC,WAAW,CAAC,EAAE,CAAC,QAAQ,EAAE,GAAG,WAAW,CAAC,kBAAkB,CAAC,CAAC;QAC/D,OAAO,CAAC,GAAG,IAAI,IAAI,aAAa,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,SAAS,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;IAC9F,CAAC;IAID,IAAI,CAAC,gBAAqB;QACxB,MAAM,MAAM,GAAG,IAAI,CAAC,EAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QACjD,OAAO,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,EAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,GAAG,MAAM,EAAE,gBAAgB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IACnF,CAAC;IAED,gBAAgB;QACd,MAAM,MAAM,GAAG,IAAI,CAAC,EAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QACjD,OAAO,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,EAAG,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IAC1E,CAAC;IAED,gBAAgB;QACd,MAAM,MAAM,GAAG,IAAI,CAAC,EAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QACjD,OAAO,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,EAAG,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IAC1E,CAAC;IAED,cAAc;QACZ,MAAM,MAAM,GAAG,IAAI,CAAC,EAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAClD,OAAO,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,EAAG,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IAC1E,CAAC;IAED,cAAc;QACZ,MAAM,MAAM,GAAG,IAAI,CAAC,EAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAClD,OAAO,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,EAAG,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IAC1E,CAAC;IAED,MAAM,CAAC,kBAAkB,CAAC,OAA2B;QACnD,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;IACzB,CAAC;IAED,MAAM,CAAC,OAAO,CAAC,OAA2B,EAAE,UAA6B;QACvE,OAAO,CAAC,cAAc,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC;IAC3C,CAAC;IAED,MAAM,CAAC,mBAAmB,CAAC,OAA2B,EAAE,gBAAuB;QAC7E,OAAO,CAAC,aAAa,CAAC,CAAC,EAAE,gBAAgB,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;IAC1D,CAAC;IAED,MAAM,CAAC,mBAAmB,CAAC,OAA2B,EAAE,gBAAuB;QAC7E,OAAO,CAAC,aAAa,CAAC,CAAC,EAAE,gBAAgB,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;IAC1D,CAAC;IAED,MAAM,CAAC,iBAAiB,CAAC,OAA2B,EAAE,cAAqB;QACzE,OAAO,CAAC,aAAa,CAAC,CAAC,EAAE,cAAc,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;IACxD,CAAC;IAED,MAAM,CAAC,iBAAiB,CAAC,OAA2B,EAAE,cAAqB;QACzE,OAAO,CAAC,aAAa,CAAC,CAAC,EAAE,cAAc,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;IACxD,CAAC;IAED,MAAM,CAAC,gBAAgB,CAAC,OAA2B;QACjD,MAAM,MAAM,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC;QACnC,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,MAAM,CAAC,mBAAmB,CAAC,OAA2B,EAAE,UAA6B,EAAE,gBAAuB,EAAE,gBAAuB,EAAE,cAAqB,EAAE,cAAqB;QACnL,aAAa,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;QAC1C,aAAa,CAAC,OAAO,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;QAC3C,aAAa,CAAC,mBAAmB,CAAC,OAAO,EAAE,gBAAgB,CAAC,CAAC;QAC7D,aAAa,CAAC,mBAAmB,CAAC,OAAO,EAAE,gBAAgB,CAAC,CAAC;QAC7D,aAAa,CAAC,iBAAiB,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;QACzD,aAAa,CAAC,iBAAiB,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;QACzD,OAAO,aAAa,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;IACjD,CAAC;IAED,MAAM;QACJ,OAAO,IAAI,cAAc,CACvB,IAAI,CAAC,IAAI,EAAE,EACX,IAAI,CAAC,gBAAgB,EAAE,EACvB,IAAI,CAAC,gBAAgB,EAAE,EACvB,IAAI,CAAC,cAAc,EAAE,EACrB,IAAI,CAAC,cAAc,EAAE,CACtB,CAAC;IACJ,CAAC;IAGD,QAAQ,CAAC,EAAkB;QACzB,EAAE,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QACtB,EAAE,CAAC,gBAAgB,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAC9C,EAAE,CAAC,gBAAgB,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAC9C,EAAE,CAAC,cAAc,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;QAC1C,EAAE,CAAC,cAAc,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;IAC5C,CAAC;CACA;AAED,MAAM,OAAO,cAAc;IAElB;IACA;IACA;IACA;IACA;IALT,YACS,OAA+B,IAAI,EACnC,mBAA2B,MAAM,CAAC,GAAG,CAAC,EACtC,mBAA2B,MAAM,CAAC,GAAG,CAAC,EACtC,iBAAyB,MAAM,CAAC,GAAG,CAAC,EACpC,iBAAyB,MAAM,CAAC,GAAG,CAAC;QAJpC,SAAI,GAAJ,IAAI,CAA+B;QACnC,qBAAgB,GAAhB,gBAAgB,CAAsB;QACtC,qBAAgB,GAAhB,gBAAgB,CAAsB;QACtC,mBAAc,GAAd,cAAc,CAAsB;QACpC,mBAAc,GAAd,cAAc,CAAsB;IAC3C,CAAC;IAGH,IAAI,CAAC,OAA2B;QAC9B,MAAM,IAAI,GAAG,CAAC,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,IAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAEzE,OAAO,aAAa,CAAC,mBAAmB,CAAC,OAAO,EAC9C,IAAI,EACJ,IAAI,CAAC,gBAAgB,EACrB,IAAI,CAAC,gBAAgB,EACrB,IAAI,CAAC,cAAc,EACnB,IAAI,CAAC,cAAc,CACpB,CAAC;IACJ,CAAC;CACA"}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
export { OCRBlock, OCRBlockT } from './yom/ocrblock.js';
|
|
2
|
+
export { OCRLayout, OCRLayoutT } from './yom/ocrlayout.js';
|
|
3
|
+
export { OCRLine, OCRLineT } from './yom/ocrline.js';
|
|
4
|
+
export { OCRPage, OCRPageT } from './yom/ocrpage.js';
|
|
5
|
+
export { OCRToken, OCRTokenT } from './yom/ocrtoken.js';
|
|
6
|
+
//# sourceMappingURL=yom.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"yom.d.ts","sourceRoot":"","sources":["../../../../src/yom/generated/yomu/yom.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACxD,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAC3D,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC"}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
// automatically generated by the FlatBuffers compiler, do not modify
|
|
2
|
+
/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
|
|
3
|
+
export { OCRBlock, OCRBlockT } from './yom/ocrblock.js';
|
|
4
|
+
export { OCRLayout, OCRLayoutT } from './yom/ocrlayout.js';
|
|
5
|
+
export { OCRLine, OCRLineT } from './yom/ocrline.js';
|
|
6
|
+
export { OCRPage, OCRPageT } from './yom/ocrpage.js';
|
|
7
|
+
export { OCRToken, OCRTokenT } from './yom/ocrtoken.js';
|
|
8
|
+
//# sourceMappingURL=yom.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"yom.js","sourceRoot":"","sources":["../../../../src/yom/generated/yomu/yom.ts"],"names":[],"mappings":"AAAA,qEAAqE;AAErE,oIAAoI;AAEpI,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACxD,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAC3D,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC"}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
/** SHA-256 over `data`, returned as raw bytes (matches CryptoKit's `SHA256.hash` used natively). */
|
|
2
|
+
export declare function sha256(data: Uint8Array): Promise<Uint8Array>;
|
|
3
|
+
export declare function bytesEqual(a: Uint8Array, b: Uint8Array): boolean;
|
|
4
|
+
//# sourceMappingURL=hash.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"hash.d.ts","sourceRoot":"","sources":["../../src/yom/hash.ts"],"names":[],"mappings":"AAAA,oGAAoG;AACpG,wBAAsB,MAAM,CAAC,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC,CAGlE;AAED,wBAAgB,UAAU,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,UAAU,GAAG,OAAO,CAMhE"}
|
package/dist/yom/hash.js
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/** SHA-256 over `data`, returned as raw bytes (matches CryptoKit's `SHA256.hash` used natively). */
|
|
2
|
+
export async function sha256(data) {
|
|
3
|
+
const digest = await crypto.subtle.digest("SHA-256", toArrayBuffer(data));
|
|
4
|
+
return new Uint8Array(digest);
|
|
5
|
+
}
|
|
6
|
+
export function bytesEqual(a, b) {
|
|
7
|
+
if (a.length !== b.length)
|
|
8
|
+
return false;
|
|
9
|
+
for (let i = 0; i < a.length; i++) {
|
|
10
|
+
if (a[i] !== b[i])
|
|
11
|
+
return false;
|
|
12
|
+
}
|
|
13
|
+
return true;
|
|
14
|
+
}
|
|
15
|
+
// `crypto.subtle.digest` requires an ArrayBuffer-backed view; a Uint8Array
|
|
16
|
+
// sliced from a larger buffer (e.g. via `subarray`) may have a non-zero
|
|
17
|
+
// `byteOffset`, so copy defensively rather than pass `data.buffer` directly.
|
|
18
|
+
function toArrayBuffer(data) {
|
|
19
|
+
return data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength);
|
|
20
|
+
}
|
|
21
|
+
//# sourceMappingURL=hash.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"hash.js","sourceRoot":"","sources":["../../src/yom/hash.ts"],"names":[],"mappings":"AAAA,oGAAoG;AACpG,MAAM,CAAC,KAAK,UAAU,MAAM,CAAC,IAAgB;IAC3C,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC;IAC1E,OAAO,IAAI,UAAU,CAAC,MAAM,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,CAAa,EAAE,CAAa;IACrD,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM;QAAE,OAAO,KAAK,CAAC;IACxC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAClC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAAE,OAAO,KAAK,CAAC;IAClC,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,2EAA2E;AAC3E,wEAAwE;AACxE,6EAA6E;AAC7E,SAAS,aAAa,CAAC,IAAgB;IACrC,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAgB,CAAC;AAC9F,CAAC"}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Fixed binary layout shared by the YOM v1 header (4096 bytes, zero padded)
|
|
3
|
+
* and its footer mirror (the same fields, unpadded, appended at the end of
|
|
4
|
+
* the file). Mirrors the Swift implementation in
|
|
5
|
+
* `swift/Sources/YOMFormat/YOMHeader.swift` byte-for-byte:
|
|
6
|
+
* little-endian `magic(4) | formatVersion(4) | indexOffset(8) |
|
|
7
|
+
* indexCompressedLength(8) | indexUncompressedLength(8) | indexHash(32) |
|
|
8
|
+
* fileHash(32)`, 96 bytes total. Both `indexHash` and `fileHash` are zeroed
|
|
9
|
+
* out of the header/footer bytes fed into the whole-file SHA-256
|
|
10
|
+
* calculation (see `zeroingHashes`); the real digests are patched in only
|
|
11
|
+
* after that hash has been computed.
|
|
12
|
+
*/
|
|
13
|
+
export declare const HEADER_FIELDS_SIZE: number;
|
|
14
|
+
export interface YOMHeader {
|
|
15
|
+
magic: Uint8Array;
|
|
16
|
+
formatVersion: number;
|
|
17
|
+
indexOffset: bigint;
|
|
18
|
+
indexCompressedLength: bigint;
|
|
19
|
+
indexUncompressedLength: bigint;
|
|
20
|
+
indexHash: Uint8Array;
|
|
21
|
+
fileHash: Uint8Array;
|
|
22
|
+
}
|
|
23
|
+
export declare class YOMHeaderParseError extends Error {
|
|
24
|
+
constructor();
|
|
25
|
+
}
|
|
26
|
+
/** The fixed 96-byte field block, identical for header and footer. */
|
|
27
|
+
export declare function encodeHeaderFields(header: YOMHeader): Uint8Array;
|
|
28
|
+
/** The full 4096-byte, zero-padded header. */
|
|
29
|
+
export declare function encodeHeader(header: YOMHeader): Uint8Array;
|
|
30
|
+
/** The unpadded footer mirror (just the field block). */
|
|
31
|
+
export declare function encodeFooter(header: YOMHeader): Uint8Array;
|
|
32
|
+
/**
|
|
33
|
+
* Parses the fixed 96-byte field block from the front of `data`. Does NOT
|
|
34
|
+
* validate magic/version/semantic correctness -- callers (reader,
|
|
35
|
+
* validator) decide what to do with the parsed values.
|
|
36
|
+
*/
|
|
37
|
+
export declare function parseHeaderFields(data: Uint8Array): YOMHeader;
|
|
38
|
+
/**
|
|
39
|
+
* A copy with both hash fields zeroed, used as the input to the whole-file
|
|
40
|
+
* hash calculation (see module doc comment).
|
|
41
|
+
*/
|
|
42
|
+
export declare function zeroingHashes(header: YOMHeader): YOMHeader;
|
|
43
|
+
export declare function headersEqual(a: YOMHeader, b: YOMHeader): boolean;
|
|
44
|
+
//# sourceMappingURL=header.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"header.d.ts","sourceRoot":"","sources":["../../src/yom/header.ts"],"names":[],"mappings":"AAEA;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,kBAAkB,QAAwD,CAAC;AAExF,MAAM,WAAW,SAAS;IACxB,KAAK,EAAE,UAAU,CAAC;IAClB,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,uBAAuB,EAAE,MAAM,CAAC;IAChC,SAAS,EAAE,UAAU,CAAC;IACtB,QAAQ,EAAE,UAAU,CAAC;CACtB;AAED,qBAAa,mBAAoB,SAAQ,KAAK;;CAI7C;AAED,sEAAsE;AACtE,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,SAAS,GAAG,UAAU,CAmBhE;AAED,8CAA8C;AAC9C,wBAAgB,YAAY,CAAC,MAAM,EAAE,SAAS,GAAG,UAAU,CAK1D;AAED,yDAAyD;AACzD,wBAAgB,YAAY,CAAC,MAAM,EAAE,SAAS,GAAG,UAAU,CAE1D;AAED;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,UAAU,GAAG,SAAS,CA2B7D;AAED;;;GAGG;AACH,wBAAgB,aAAa,CAAC,MAAM,EAAE,SAAS,GAAG,SAAS,CAM1D;AAED,wBAAgB,YAAY,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,SAAS,GAAG,OAAO,CAUhE"}
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
import { YOM_HASH_LENGTH, YOM_HEADER_SIZE } from "./constants.js";
|
|
2
|
+
/**
|
|
3
|
+
* Fixed binary layout shared by the YOM v1 header (4096 bytes, zero padded)
|
|
4
|
+
* and its footer mirror (the same fields, unpadded, appended at the end of
|
|
5
|
+
* the file). Mirrors the Swift implementation in
|
|
6
|
+
* `swift/Sources/YOMFormat/YOMHeader.swift` byte-for-byte:
|
|
7
|
+
* little-endian `magic(4) | formatVersion(4) | indexOffset(8) |
|
|
8
|
+
* indexCompressedLength(8) | indexUncompressedLength(8) | indexHash(32) |
|
|
9
|
+
* fileHash(32)`, 96 bytes total. Both `indexHash` and `fileHash` are zeroed
|
|
10
|
+
* out of the header/footer bytes fed into the whole-file SHA-256
|
|
11
|
+
* calculation (see `zeroingHashes`); the real digests are patched in only
|
|
12
|
+
* after that hash has been computed.
|
|
13
|
+
*/
|
|
14
|
+
export const HEADER_FIELDS_SIZE = 4 + 4 + 8 + 8 + 8 + YOM_HASH_LENGTH + YOM_HASH_LENGTH;
|
|
15
|
+
export class YOMHeaderParseError extends Error {
|
|
16
|
+
constructor() {
|
|
17
|
+
super("YOM header truncated");
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
/** The fixed 96-byte field block, identical for header and footer. */
|
|
21
|
+
export function encodeHeaderFields(header) {
|
|
22
|
+
const out = new Uint8Array(HEADER_FIELDS_SIZE);
|
|
23
|
+
const view = new DataView(out.buffer);
|
|
24
|
+
let cursor = 0;
|
|
25
|
+
out.set(header.magic, cursor);
|
|
26
|
+
cursor += 4;
|
|
27
|
+
view.setUint32(cursor, header.formatVersion, true);
|
|
28
|
+
cursor += 4;
|
|
29
|
+
view.setBigUint64(cursor, header.indexOffset, true);
|
|
30
|
+
cursor += 8;
|
|
31
|
+
view.setBigUint64(cursor, header.indexCompressedLength, true);
|
|
32
|
+
cursor += 8;
|
|
33
|
+
view.setBigUint64(cursor, header.indexUncompressedLength, true);
|
|
34
|
+
cursor += 8;
|
|
35
|
+
out.set(header.indexHash, cursor);
|
|
36
|
+
cursor += YOM_HASH_LENGTH;
|
|
37
|
+
out.set(header.fileHash, cursor);
|
|
38
|
+
cursor += YOM_HASH_LENGTH;
|
|
39
|
+
return out;
|
|
40
|
+
}
|
|
41
|
+
/** The full 4096-byte, zero-padded header. */
|
|
42
|
+
export function encodeHeader(header) {
|
|
43
|
+
const fields = encodeHeaderFields(header);
|
|
44
|
+
const out = new Uint8Array(YOM_HEADER_SIZE);
|
|
45
|
+
out.set(fields, 0);
|
|
46
|
+
return out;
|
|
47
|
+
}
|
|
48
|
+
/** The unpadded footer mirror (just the field block). */
|
|
49
|
+
export function encodeFooter(header) {
|
|
50
|
+
return encodeHeaderFields(header);
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* Parses the fixed 96-byte field block from the front of `data`. Does NOT
|
|
54
|
+
* validate magic/version/semantic correctness -- callers (reader,
|
|
55
|
+
* validator) decide what to do with the parsed values.
|
|
56
|
+
*/
|
|
57
|
+
export function parseHeaderFields(data) {
|
|
58
|
+
if (data.length < HEADER_FIELDS_SIZE)
|
|
59
|
+
throw new YOMHeaderParseError();
|
|
60
|
+
const view = new DataView(data.buffer, data.byteOffset, data.byteLength);
|
|
61
|
+
let cursor = 0;
|
|
62
|
+
const magic = data.slice(cursor, cursor + 4);
|
|
63
|
+
cursor += 4;
|
|
64
|
+
const formatVersion = view.getUint32(cursor, true);
|
|
65
|
+
cursor += 4;
|
|
66
|
+
const indexOffset = view.getBigUint64(cursor, true);
|
|
67
|
+
cursor += 8;
|
|
68
|
+
const indexCompressedLength = view.getBigUint64(cursor, true);
|
|
69
|
+
cursor += 8;
|
|
70
|
+
const indexUncompressedLength = view.getBigUint64(cursor, true);
|
|
71
|
+
cursor += 8;
|
|
72
|
+
const indexHash = data.slice(cursor, cursor + YOM_HASH_LENGTH);
|
|
73
|
+
cursor += YOM_HASH_LENGTH;
|
|
74
|
+
const fileHash = data.slice(cursor, cursor + YOM_HASH_LENGTH);
|
|
75
|
+
cursor += YOM_HASH_LENGTH;
|
|
76
|
+
return {
|
|
77
|
+
magic,
|
|
78
|
+
formatVersion,
|
|
79
|
+
indexOffset,
|
|
80
|
+
indexCompressedLength,
|
|
81
|
+
indexUncompressedLength,
|
|
82
|
+
indexHash,
|
|
83
|
+
fileHash,
|
|
84
|
+
};
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
* A copy with both hash fields zeroed, used as the input to the whole-file
|
|
88
|
+
* hash calculation (see module doc comment).
|
|
89
|
+
*/
|
|
90
|
+
export function zeroingHashes(header) {
|
|
91
|
+
return {
|
|
92
|
+
...header,
|
|
93
|
+
indexHash: new Uint8Array(YOM_HASH_LENGTH),
|
|
94
|
+
fileHash: new Uint8Array(YOM_HASH_LENGTH),
|
|
95
|
+
};
|
|
96
|
+
}
|
|
97
|
+
export function headersEqual(a, b) {
|
|
98
|
+
return (bytesEqual(a.magic, b.magic) &&
|
|
99
|
+
a.formatVersion === b.formatVersion &&
|
|
100
|
+
a.indexOffset === b.indexOffset &&
|
|
101
|
+
a.indexCompressedLength === b.indexCompressedLength &&
|
|
102
|
+
a.indexUncompressedLength === b.indexUncompressedLength &&
|
|
103
|
+
bytesEqual(a.indexHash, b.indexHash) &&
|
|
104
|
+
bytesEqual(a.fileHash, b.fileHash));
|
|
105
|
+
}
|
|
106
|
+
function bytesEqual(a, b) {
|
|
107
|
+
if (a.length !== b.length)
|
|
108
|
+
return false;
|
|
109
|
+
for (let i = 0; i < a.length; i++) {
|
|
110
|
+
if (a[i] !== b[i])
|
|
111
|
+
return false;
|
|
112
|
+
}
|
|
113
|
+
return true;
|
|
114
|
+
}
|
|
115
|
+
//# sourceMappingURL=header.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"header.js","sourceRoot":"","sources":["../../src/yom/header.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAElE;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,eAAe,GAAG,eAAe,CAAC;AAYxF,MAAM,OAAO,mBAAoB,SAAQ,KAAK;IAC5C;QACE,KAAK,CAAC,sBAAsB,CAAC,CAAC;IAChC,CAAC;CACF;AAED,sEAAsE;AACtE,MAAM,UAAU,kBAAkB,CAAC,MAAiB;IAClD,MAAM,GAAG,GAAG,IAAI,UAAU,CAAC,kBAAkB,CAAC,CAAC;IAC/C,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACtC,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IAC9B,MAAM,IAAI,CAAC,CAAC;IACZ,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,aAAa,EAAE,IAAI,CAAC,CAAC;IACnD,MAAM,IAAI,CAAC,CAAC;IACZ,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,MAAM,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;IACpD,MAAM,IAAI,CAAC,CAAC;IACZ,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,MAAM,CAAC,qBAAqB,EAAE,IAAI,CAAC,CAAC;IAC9D,MAAM,IAAI,CAAC,CAAC;IACZ,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,MAAM,CAAC,uBAAuB,EAAE,IAAI,CAAC,CAAC;IAChE,MAAM,IAAI,CAAC,CAAC;IACZ,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAClC,MAAM,IAAI,eAAe,CAAC;IAC1B,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IACjC,MAAM,IAAI,eAAe,CAAC;IAC1B,OAAO,GAAG,CAAC;AACb,CAAC;AAED,8CAA8C;AAC9C,MAAM,UAAU,YAAY,CAAC,MAAiB;IAC5C,MAAM,MAAM,GAAG,kBAAkB,CAAC,MAAM,CAAC,CAAC;IAC1C,MAAM,GAAG,GAAG,IAAI,UAAU,CAAC,eAAe,CAAC,CAAC;IAC5C,GAAG,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACnB,OAAO,GAAG,CAAC;AACb,CAAC;AAED,yDAAyD;AACzD,MAAM,UAAU,YAAY,CAAC,MAAiB;IAC5C,OAAO,kBAAkB,CAAC,MAAM,CAAC,CAAC;AACpC,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,iBAAiB,CAAC,IAAgB;IAChD,IAAI,IAAI,CAAC,MAAM,GAAG,kBAAkB;QAAE,MAAM,IAAI,mBAAmB,EAAE,CAAC;IACtE,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;IACzE,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,GAAG,CAAC,CAAC,CAAC;IAC7C,MAAM,IAAI,CAAC,CAAC;IACZ,MAAM,aAAa,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IACnD,MAAM,IAAI,CAAC,CAAC;IACZ,MAAM,WAAW,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IACpD,MAAM,IAAI,CAAC,CAAC;IACZ,MAAM,qBAAqB,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAC9D,MAAM,IAAI,CAAC,CAAC;IACZ,MAAM,uBAAuB,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAChE,MAAM,IAAI,CAAC,CAAC;IACZ,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,GAAG,eAAe,CAAC,CAAC;IAC/D,MAAM,IAAI,eAAe,CAAC;IAC1B,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,GAAG,eAAe,CAAC,CAAC;IAC9D,MAAM,IAAI,eAAe,CAAC;IAC1B,OAAO;QACL,KAAK;QACL,aAAa;QACb,WAAW;QACX,qBAAqB;QACrB,uBAAuB;QACvB,SAAS;QACT,QAAQ;KACT,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,MAAiB;IAC7C,OAAO;QACL,GAAG,MAAM;QACT,SAAS,EAAE,IAAI,UAAU,CAAC,eAAe,CAAC;QAC1C,QAAQ,EAAE,IAAI,UAAU,CAAC,eAAe,CAAC;KAC1C,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,CAAY,EAAE,CAAY;IACrD,OAAO,CACL,UAAU,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC;QAC5B,CAAC,CAAC,aAAa,KAAK,CAAC,CAAC,aAAa;QACnC,CAAC,CAAC,WAAW,KAAK,CAAC,CAAC,WAAW;QAC/B,CAAC,CAAC,qBAAqB,KAAK,CAAC,CAAC,qBAAqB;QACnD,CAAC,CAAC,uBAAuB,KAAK,CAAC,CAAC,uBAAuB;QACvD,UAAU,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,SAAS,CAAC;QACpC,UAAU,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,QAAQ,CAAC,CACnC,CAAC;AACJ,CAAC;AAED,SAAS,UAAU,CAAC,CAAa,EAAE,CAAa;IAC9C,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM;QAAE,OAAO,KAAK,CAAC;IACxC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAClC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAAE,OAAO,KAAK,CAAC;IAClC,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC"}
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import type { MainIndex } from "./generated/yomu/yom/main-index.js";
|
|
2
|
+
import type { PageEntry } from "./generated/yomu/yom/page-entry.js";
|
|
3
|
+
import { type ByteSource, type HttpRangeOptions } from "./byteSource.js";
|
|
4
|
+
/**
|
|
5
|
+
* Opens a YOM v1 packfile for random-access reading over a {@link ByteSource}
|
|
6
|
+
* -- either an in-memory buffer ({@link MemoryByteSource}) or an HTTP server
|
|
7
|
+
* that honours `Range` requests ({@link HttpRangeByteSource}). Ports the Swift
|
|
8
|
+
* implementation in `swift/Sources/YOMFormat/YOMPackfileReader.swift`.
|
|
9
|
+
*
|
|
10
|
+
* `open`/`openSource`/`openURL` read only the fixed header and the main
|
|
11
|
+
* FlatBuffers index up front -- never the blob area. The index is
|
|
12
|
+
* bounds-verified (Gap 3) before traversal; each blob read enforces the
|
|
13
|
+
* reader-side size caps (Gap 2) before decompressing, then verifies the
|
|
14
|
+
* blob's stored SHA-256 as the final gate.
|
|
15
|
+
*
|
|
16
|
+
* Because reads may hit the network, all blob accessors are `Promise`-based.
|
|
17
|
+
* `pageImage`/`thumbnailImage` never invoke zstd: page and thumbnail blobs are
|
|
18
|
+
* always `.none` compression per `requiredCompression`.
|
|
19
|
+
*/
|
|
20
|
+
export declare class YOMReaderError extends Error {
|
|
21
|
+
}
|
|
22
|
+
export declare class YOMPackfileReader {
|
|
23
|
+
readonly mainIndex: MainIndex;
|
|
24
|
+
private readonly source;
|
|
25
|
+
private readonly indexBytes;
|
|
26
|
+
private constructor();
|
|
27
|
+
/** Opens from an in-memory buffer (wraps a {@link MemoryByteSource}). */
|
|
28
|
+
static open(bytes: Uint8Array): Promise<YOMPackfileReader>;
|
|
29
|
+
/** Opens over any {@link ByteSource}. Reads only the header + index. */
|
|
30
|
+
static openSource(source: ByteSource): Promise<YOMPackfileReader>;
|
|
31
|
+
/**
|
|
32
|
+
* Opens a `.yom` over HTTP using `Range` requests -- fetching only the
|
|
33
|
+
* header + index up front, then each blob on demand. When the server does
|
|
34
|
+
* not support ranges, throws `RangeNotSupportedError` unless
|
|
35
|
+
* `fallbackToFullFetch` is set, in which case it does a single full `GET`
|
|
36
|
+
* and reads from memory.
|
|
37
|
+
*/
|
|
38
|
+
static openURL(url: string, opts?: HttpRangeOptions & {
|
|
39
|
+
fallbackToFullFetch?: boolean;
|
|
40
|
+
}): Promise<YOMPackfileReader>;
|
|
41
|
+
get pageCount(): number;
|
|
42
|
+
/** The book title stored in the package's metadata, or `""` if absent. */
|
|
43
|
+
get title(): string;
|
|
44
|
+
page(index: number): PageEntry;
|
|
45
|
+
/**
|
|
46
|
+
* Reads exactly one blob's byte range, enforcing the reader-side size caps
|
|
47
|
+
* (Gap 2) before decompressing, then verifying its stored hash against the
|
|
48
|
+
* decompressed (original) bytes -- the same convention the writer uses when
|
|
49
|
+
* hashing.
|
|
50
|
+
*/
|
|
51
|
+
readBlob(blobIndex: number): Promise<Uint8Array>;
|
|
52
|
+
/** Never invokes zstd -- page image blobs are always `.none` by policy. */
|
|
53
|
+
pageImage(pageIndex: number): Promise<Uint8Array>;
|
|
54
|
+
/** Never invokes zstd -- thumbnail blobs are always `.none` by policy. */
|
|
55
|
+
thumbnailImage(pageIndex: number): Promise<Uint8Array>;
|
|
56
|
+
}
|
|
57
|
+
//# sourceMappingURL=reader.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"reader.d.ts","sourceRoot":"","sources":["../../src/yom/reader.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,oCAAoC,CAAC;AACpE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,oCAAoC,CAAC;AAQpE,OAAO,EAGL,KAAK,UAAU,EACf,KAAK,gBAAgB,EACtB,MAAM,iBAAiB,CAAC;AAGzB;;;;;;;;;;;;;;;GAeG;AACH,qBAAa,cAAe,SAAQ,KAAK;CAAG;AAE5C,qBAAa,iBAAiB;IAC5B,QAAQ,CAAC,SAAS,EAAE,SAAS,CAAC;IAC9B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAa;IAEpC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAa;IAExC,OAAO;IAMP,yEAAyE;WAC5D,IAAI,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,iBAAiB,CAAC;IAIhE,wEAAwE;WAC3D,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,OAAO,CAAC,iBAAiB,CAAC;IAuBvE;;;;;;OAMG;WACU,OAAO,CAClB,GAAG,EAAE,MAAM,EACX,IAAI,GAAE,gBAAgB,GAAG;QAAE,mBAAmB,CAAC,EAAE,OAAO,CAAA;KAAO,GAC9D,OAAO,CAAC,iBAAiB,CAAC;IAiB7B,IAAI,SAAS,IAAI,MAAM,CAEtB;IAED,0EAA0E;IAC1E,IAAI,KAAK,IAAI,MAAM,CAIlB;IAED,IAAI,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS;IAM9B;;;;;OAKG;IACG,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC;IA2BtD,2EAA2E;IACrE,SAAS,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC;IAIvD,0EAA0E;IACpE,cAAc,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC;CAG7D"}
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
import { Compression } from "./generated/yomu/yom/compression.js";
|
|
2
|
+
import { YOM_FORMAT_VERSION, YOM_HEADER_SIZE, YOM_MAGIC } from "./constants.js";
|
|
3
|
+
import { parseHeaderFields } from "./header.js";
|
|
4
|
+
import { bytesEqual, sha256 } from "./hash.js";
|
|
5
|
+
import { zstdDecompress } from "./zstd.js";
|
|
6
|
+
import { getVerifiedRootAsMainIndex } from "./flatbufferVerify.js";
|
|
7
|
+
import { enforceBlobSizePolicy } from "./blobLimits.js";
|
|
8
|
+
import { HttpRangeByteSource, MemoryByteSource, } from "./byteSource.js";
|
|
9
|
+
import { RangeNotSupportedError } from "./errors.js";
|
|
10
|
+
/**
|
|
11
|
+
* Opens a YOM v1 packfile for random-access reading over a {@link ByteSource}
|
|
12
|
+
* -- either an in-memory buffer ({@link MemoryByteSource}) or an HTTP server
|
|
13
|
+
* that honours `Range` requests ({@link HttpRangeByteSource}). Ports the Swift
|
|
14
|
+
* implementation in `swift/Sources/YOMFormat/YOMPackfileReader.swift`.
|
|
15
|
+
*
|
|
16
|
+
* `open`/`openSource`/`openURL` read only the fixed header and the main
|
|
17
|
+
* FlatBuffers index up front -- never the blob area. The index is
|
|
18
|
+
* bounds-verified (Gap 3) before traversal; each blob read enforces the
|
|
19
|
+
* reader-side size caps (Gap 2) before decompressing, then verifies the
|
|
20
|
+
* blob's stored SHA-256 as the final gate.
|
|
21
|
+
*
|
|
22
|
+
* Because reads may hit the network, all blob accessors are `Promise`-based.
|
|
23
|
+
* `pageImage`/`thumbnailImage` never invoke zstd: page and thumbnail blobs are
|
|
24
|
+
* always `.none` compression per `requiredCompression`.
|
|
25
|
+
*/
|
|
26
|
+
export class YOMReaderError extends Error {
|
|
27
|
+
}
|
|
28
|
+
export class YOMPackfileReader {
|
|
29
|
+
mainIndex;
|
|
30
|
+
source;
|
|
31
|
+
// Retained so the index ByteBuffer that `mainIndex` reads from stays alive.
|
|
32
|
+
indexBytes;
|
|
33
|
+
constructor(source, indexBytes, mainIndex) {
|
|
34
|
+
this.source = source;
|
|
35
|
+
this.indexBytes = indexBytes;
|
|
36
|
+
this.mainIndex = mainIndex;
|
|
37
|
+
}
|
|
38
|
+
/** Opens from an in-memory buffer (wraps a {@link MemoryByteSource}). */
|
|
39
|
+
static async open(bytes) {
|
|
40
|
+
return YOMPackfileReader.openSource(new MemoryByteSource(bytes));
|
|
41
|
+
}
|
|
42
|
+
/** Opens over any {@link ByteSource}. Reads only the header + index. */
|
|
43
|
+
static async openSource(source) {
|
|
44
|
+
if (source.size < YOM_HEADER_SIZE)
|
|
45
|
+
throw new YOMReaderError("file too small");
|
|
46
|
+
const headerBytes = await source.read(0, YOM_HEADER_SIZE);
|
|
47
|
+
const header = parseHeaderFields(headerBytes);
|
|
48
|
+
if (!bytesEqual(header.magic, YOM_MAGIC))
|
|
49
|
+
throw new YOMReaderError("bad magic");
|
|
50
|
+
if (header.formatVersion !== YOM_FORMAT_VERSION) {
|
|
51
|
+
throw new YOMReaderError(`unsupported format version ${header.formatVersion}`);
|
|
52
|
+
}
|
|
53
|
+
const indexOffset = Number(header.indexOffset);
|
|
54
|
+
const indexLength = Number(header.indexUncompressedLength);
|
|
55
|
+
if (indexOffset < YOM_HEADER_SIZE || indexOffset + indexLength > source.size) {
|
|
56
|
+
throw new YOMReaderError("index out of bounds");
|
|
57
|
+
}
|
|
58
|
+
const indexBytes = await source.read(indexOffset, indexLength);
|
|
59
|
+
// Cheap root + vector-length bounds verify on the hot open path (Gap 3).
|
|
60
|
+
const mainIndex = getVerifiedRootAsMainIndex(indexBytes, false);
|
|
61
|
+
return new YOMPackfileReader(source, indexBytes, mainIndex);
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* Opens a `.yom` over HTTP using `Range` requests -- fetching only the
|
|
65
|
+
* header + index up front, then each blob on demand. When the server does
|
|
66
|
+
* not support ranges, throws `RangeNotSupportedError` unless
|
|
67
|
+
* `fallbackToFullFetch` is set, in which case it does a single full `GET`
|
|
68
|
+
* and reads from memory.
|
|
69
|
+
*/
|
|
70
|
+
static async openURL(url, opts = {}) {
|
|
71
|
+
const { fallbackToFullFetch, ...rangeOpts } = opts;
|
|
72
|
+
try {
|
|
73
|
+
const source = await HttpRangeByteSource.create(url, rangeOpts);
|
|
74
|
+
return await YOMPackfileReader.openSource(source);
|
|
75
|
+
}
|
|
76
|
+
catch (err) {
|
|
77
|
+
if (fallbackToFullFetch && err instanceof RangeNotSupportedError) {
|
|
78
|
+
const fetchFn = rangeOpts.fetch ?? ((input, init) => fetch(input, init));
|
|
79
|
+
const res = await fetchFn(url, { headers: rangeOpts.headers, signal: rangeOpts.signal });
|
|
80
|
+
if (!res.ok)
|
|
81
|
+
throw new YOMReaderError(`full fetch failed with status ${res.status} for ${url}`);
|
|
82
|
+
const bytes = new Uint8Array(await res.arrayBuffer());
|
|
83
|
+
return YOMPackfileReader.openSource(new MemoryByteSource(bytes));
|
|
84
|
+
}
|
|
85
|
+
throw err;
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
get pageCount() {
|
|
89
|
+
return this.mainIndex.pagesLength();
|
|
90
|
+
}
|
|
91
|
+
/** The book title stored in the package's metadata, or `""` if absent. */
|
|
92
|
+
get title() {
|
|
93
|
+
const book = this.mainIndex.book();
|
|
94
|
+
const title = book?.title();
|
|
95
|
+
return typeof title === "string" ? title : "";
|
|
96
|
+
}
|
|
97
|
+
page(index) {
|
|
98
|
+
const page = this.mainIndex.pages(index);
|
|
99
|
+
if (!page)
|
|
100
|
+
throw new YOMReaderError("page index out of range");
|
|
101
|
+
return page;
|
|
102
|
+
}
|
|
103
|
+
/**
|
|
104
|
+
* Reads exactly one blob's byte range, enforcing the reader-side size caps
|
|
105
|
+
* (Gap 2) before decompressing, then verifying its stored hash against the
|
|
106
|
+
* decompressed (original) bytes -- the same convention the writer uses when
|
|
107
|
+
* hashing.
|
|
108
|
+
*/
|
|
109
|
+
async readBlob(blobIndex) {
|
|
110
|
+
if (blobIndex < 0 || blobIndex >= this.mainIndex.blobsLength()) {
|
|
111
|
+
throw new YOMReaderError("blob index out of range");
|
|
112
|
+
}
|
|
113
|
+
const blob = this.mainIndex.blobs(blobIndex);
|
|
114
|
+
if (!blob)
|
|
115
|
+
throw new YOMReaderError("blob index out of range");
|
|
116
|
+
const start = Number(blob.offset());
|
|
117
|
+
const compressedLength = Number(blob.compressedLength());
|
|
118
|
+
const uncompressedLength = Number(blob.uncompressedLength());
|
|
119
|
+
if (start < 0 || compressedLength < 0 || start + compressedLength > this.source.size) {
|
|
120
|
+
throw new YOMReaderError("blob out of bounds");
|
|
121
|
+
}
|
|
122
|
+
enforceBlobSizePolicy(blob.kind(), compressedLength, uncompressedLength);
|
|
123
|
+
const rawBytes = await this.source.read(start, compressedLength);
|
|
124
|
+
const original = blob.compression() === Compression.None ? rawBytes : await zstdDecompress(rawBytes, uncompressedLength);
|
|
125
|
+
const actualHash = await sha256(original);
|
|
126
|
+
const expectedHash = blob.hashArray();
|
|
127
|
+
if (!expectedHash || !bytesEqual(actualHash, expectedHash)) {
|
|
128
|
+
throw new YOMReaderError("blob hash mismatch");
|
|
129
|
+
}
|
|
130
|
+
return original;
|
|
131
|
+
}
|
|
132
|
+
/** Never invokes zstd -- page image blobs are always `.none` by policy. */
|
|
133
|
+
async pageImage(pageIndex) {
|
|
134
|
+
return this.readBlob(this.page(pageIndex).pageImageBlobIndex());
|
|
135
|
+
}
|
|
136
|
+
/** Never invokes zstd -- thumbnail blobs are always `.none` by policy. */
|
|
137
|
+
async thumbnailImage(pageIndex) {
|
|
138
|
+
return this.readBlob(this.page(pageIndex).thumbnailBlobIndex());
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
//# sourceMappingURL=reader.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"reader.js","sourceRoot":"","sources":["../../src/yom/reader.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,WAAW,EAAE,MAAM,qCAAqC,CAAC;AAClE,OAAO,EAAE,kBAAkB,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAChF,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAChD,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AAC/C,OAAO,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAC3C,OAAO,EAAE,0BAA0B,EAAE,MAAM,uBAAuB,CAAC;AACnE,OAAO,EAAE,qBAAqB,EAAE,MAAM,iBAAiB,CAAC;AACxD,OAAO,EACL,mBAAmB,EACnB,gBAAgB,GAGjB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,sBAAsB,EAAE,MAAM,aAAa,CAAC;AAErD;;;;;;;;;;;;;;;GAeG;AACH,MAAM,OAAO,cAAe,SAAQ,KAAK;CAAG;AAE5C,MAAM,OAAO,iBAAiB;IACnB,SAAS,CAAY;IACb,MAAM,CAAa;IACpC,4EAA4E;IAC3D,UAAU,CAAa;IAExC,YAAoB,MAAkB,EAAE,UAAsB,EAAE,SAAoB;QAClF,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;IAC7B,CAAC;IAED,yEAAyE;IACzE,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,KAAiB;QACjC,OAAO,iBAAiB,CAAC,UAAU,CAAC,IAAI,gBAAgB,CAAC,KAAK,CAAC,CAAC,CAAC;IACnE,CAAC;IAED,wEAAwE;IACxE,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,MAAkB;QACxC,IAAI,MAAM,CAAC,IAAI,GAAG,eAAe;YAAE,MAAM,IAAI,cAAc,CAAC,gBAAgB,CAAC,CAAC;QAE9E,MAAM,WAAW,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC,EAAE,eAAe,CAAC,CAAC;QAC1D,MAAM,MAAM,GAAG,iBAAiB,CAAC,WAAW,CAAC,CAAC;QAC9C,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,KAAK,EAAE,SAAS,CAAC;YAAE,MAAM,IAAI,cAAc,CAAC,WAAW,CAAC,CAAC;QAChF,IAAI,MAAM,CAAC,aAAa,KAAK,kBAAkB,EAAE,CAAC;YAChD,MAAM,IAAI,cAAc,CAAC,8BAA8B,MAAM,CAAC,aAAa,EAAE,CAAC,CAAC;QACjF,CAAC;QAED,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;QAC/C,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,uBAAuB,CAAC,CAAC;QAC3D,IAAI,WAAW,GAAG,eAAe,IAAI,WAAW,GAAG,WAAW,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC;YAC7E,MAAM,IAAI,cAAc,CAAC,qBAAqB,CAAC,CAAC;QAClD,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC;QAC/D,yEAAyE;QACzE,MAAM,SAAS,GAAG,0BAA0B,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;QAEhE,OAAO,IAAI,iBAAiB,CAAC,MAAM,EAAE,UAAU,EAAE,SAAS,CAAC,CAAC;IAC9D,CAAC;IAED;;;;;;OAMG;IACH,MAAM,CAAC,KAAK,CAAC,OAAO,CAClB,GAAW,EACX,OAA6D,EAAE;QAE/D,MAAM,EAAE,mBAAmB,EAAE,GAAG,SAAS,EAAE,GAAG,IAAI,CAAC;QACnD,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,mBAAmB,CAAC,MAAM,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;YAChE,OAAO,MAAM,iBAAiB,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;QACpD,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,mBAAmB,IAAI,GAAG,YAAY,sBAAsB,EAAE,CAAC;gBACjE,MAAM,OAAO,GAAG,SAAS,CAAC,KAAK,IAAI,CAAC,CAAC,KAAa,EAAE,IAAkB,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC;gBAC/F,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,SAAS,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC;gBACzF,IAAI,CAAC,GAAG,CAAC,EAAE;oBAAE,MAAM,IAAI,cAAc,CAAC,iCAAiC,GAAG,CAAC,MAAM,QAAQ,GAAG,EAAE,CAAC,CAAC;gBAChG,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,WAAW,EAAE,CAAC,CAAC;gBACtD,OAAO,iBAAiB,CAAC,UAAU,CAAC,IAAI,gBAAgB,CAAC,KAAK,CAAC,CAAC,CAAC;YACnE,CAAC;YACD,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;IAED,IAAI,SAAS;QACX,OAAO,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,CAAC;IACtC,CAAC;IAED,0EAA0E;IAC1E,IAAI,KAAK;QACP,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;QACnC,MAAM,KAAK,GAAG,IAAI,EAAE,KAAK,EAAE,CAAC;QAC5B,OAAO,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;IAChD,CAAC;IAED,IAAI,CAAC,KAAa;QAChB,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACzC,IAAI,CAAC,IAAI;YAAE,MAAM,IAAI,cAAc,CAAC,yBAAyB,CAAC,CAAC;QAC/D,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,QAAQ,CAAC,SAAiB;QAC9B,IAAI,SAAS,GAAG,CAAC,IAAI,SAAS,IAAI,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,EAAE,CAAC;YAC/D,MAAM,IAAI,cAAc,CAAC,yBAAyB,CAAC,CAAC;QACtD,CAAC;QACD,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QAC7C,IAAI,CAAC,IAAI;YAAE,MAAM,IAAI,cAAc,CAAC,yBAAyB,CAAC,CAAC;QAE/D,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QACpC,MAAM,gBAAgB,GAAG,MAAM,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC,CAAC;QACzD,MAAM,kBAAkB,GAAG,MAAM,CAAC,IAAI,CAAC,kBAAkB,EAAE,CAAC,CAAC;QAC7D,IAAI,KAAK,GAAG,CAAC,IAAI,gBAAgB,GAAG,CAAC,IAAI,KAAK,GAAG,gBAAgB,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACrF,MAAM,IAAI,cAAc,CAAC,oBAAoB,CAAC,CAAC;QACjD,CAAC;QACD,qBAAqB,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,gBAAgB,EAAE,kBAAkB,CAAC,CAAC;QAEzE,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,gBAAgB,CAAC,CAAC;QACjE,MAAM,QAAQ,GACZ,IAAI,CAAC,WAAW,EAAE,KAAK,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,cAAc,CAAC,QAAQ,EAAE,kBAAkB,CAAC,CAAC;QAE1G,MAAM,UAAU,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC1C,MAAM,YAAY,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;QACtC,IAAI,CAAC,YAAY,IAAI,CAAC,UAAU,CAAC,UAAU,EAAE,YAAY,CAAC,EAAE,CAAC;YAC3D,MAAM,IAAI,cAAc,CAAC,oBAAoB,CAAC,CAAC;QACjD,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,2EAA2E;IAC3E,KAAK,CAAC,SAAS,CAAC,SAAiB;QAC/B,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,kBAAkB,EAAE,CAAC,CAAC;IAClE,CAAC;IAED,0EAA0E;IAC1E,KAAK,CAAC,cAAc,CAAC,SAAiB;QACpC,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,kBAAkB,EAAE,CAAC,CAAC;IAClE,CAAC;CACF"}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Builds YOM's search dictionary + postings blobs over normalized text
|
|
3
|
+
* (`textNormalizer.ts` output), using character bigrams. Ports the Swift
|
|
4
|
+
* implementation in `swift/Sources/YOMFormat/YOMSearchIndexBuilder.swift`.
|
|
5
|
+
*
|
|
6
|
+
* **Dictionary blob**: the concatenated UTF-8 bytes of every term, in sorted
|
|
7
|
+
* order. Redundant with `SearchTermRef.term` in the main index but is a
|
|
8
|
+
* deliberately simple, schema-valid choice (see the Swift source's doc
|
|
9
|
+
* comment) -- the format only requires the byte range exist, not that it
|
|
10
|
+
* hold any particular content.
|
|
11
|
+
*
|
|
12
|
+
* **Postings blob**: for each term, a delta+varint-encoded sequence of
|
|
13
|
+
* `(page, position)` hits sorted by page then position, where `position` is
|
|
14
|
+
* the *code-point* offset within that page's normalized text. Never
|
|
15
|
+
* compressed (`.none`) per the fixed policy.
|
|
16
|
+
*
|
|
17
|
+
* Iterates by Unicode code point (`Array.from`), the closest JS analogue of
|
|
18
|
+
* Swift's extended-grapheme-cluster `Character` iteration -- the two only
|
|
19
|
+
* diverge for combining marks/emoji sequences, which NFKC-normalized
|
|
20
|
+
* Japanese book text does not produce in practice.
|
|
21
|
+
*/
|
|
22
|
+
export interface SearchTerm {
|
|
23
|
+
term: string;
|
|
24
|
+
dictionaryOffset: number;
|
|
25
|
+
dictionaryLength: number;
|
|
26
|
+
postingsOffset: number;
|
|
27
|
+
postingsLength: number;
|
|
28
|
+
}
|
|
29
|
+
export interface SearchIndexResult {
|
|
30
|
+
dictionaryBlob: Uint8Array;
|
|
31
|
+
postingsBlob: Uint8Array;
|
|
32
|
+
searchTerms: SearchTerm[];
|
|
33
|
+
}
|
|
34
|
+
export interface Hit {
|
|
35
|
+
page: number;
|
|
36
|
+
position: number;
|
|
37
|
+
}
|
|
38
|
+
export declare function buildSearchIndex(normalizedPageTexts: string[]): SearchIndexResult;
|
|
39
|
+
/** Decodes one term's postings byte range back into `(page, position)` hits. */
|
|
40
|
+
export declare function decodePostings(blob: Uint8Array): Hit[];
|
|
41
|
+
//# sourceMappingURL=searchIndex.d.ts.map
|