zstdify 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +95 -0
- package/dist/bitstream/bitReader.d.ts +26 -0
- package/dist/bitstream/bitReader.js +86 -0
- package/dist/bitstream/bitReader.js.map +1 -0
- package/dist/bitstream/bitReader.test.d.ts +1 -0
- package/dist/bitstream/bitReader.test.js +47 -0
- package/dist/bitstream/bitReader.test.js.map +1 -0
- package/dist/bitstream/bitReaderReverse.d.ts +23 -0
- package/dist/bitstream/bitReaderReverse.js +84 -0
- package/dist/bitstream/bitReaderReverse.js.map +1 -0
- package/dist/bitstream/bitReaderReverse.test.d.ts +1 -0
- package/dist/bitstream/bitReaderReverse.test.js +49 -0
- package/dist/bitstream/bitReaderReverse.test.js.map +1 -0
- package/dist/bitstream/bitWriter.d.ts +15 -0
- package/dist/bitstream/bitWriter.js +47 -0
- package/dist/bitstream/bitWriter.js.map +1 -0
- package/dist/bitstream/index.d.ts +4 -0
- package/dist/bitstream/index.js +5 -0
- package/dist/bitstream/index.js.map +1 -0
- package/dist/bitstream/littleEndian.d.ts +7 -0
- package/dist/bitstream/littleEndian.js +45 -0
- package/dist/bitstream/littleEndian.js.map +1 -0
- package/dist/bitstream/littleEndian.test.d.ts +1 -0
- package/dist/bitstream/littleEndian.test.js +21 -0
- package/dist/bitstream/littleEndian.test.js.map +1 -0
- package/dist/bitstream/varint.d.ts +9 -0
- package/dist/bitstream/varint.js +40 -0
- package/dist/bitstream/varint.js.map +1 -0
- package/dist/bitstream/varint.test.d.ts +1 -0
- package/dist/bitstream/varint.test.js +25 -0
- package/dist/bitstream/varint.test.js.map +1 -0
- package/dist/compress.d.ts +9 -0
- package/dist/compress.js +74 -0
- package/dist/compress.js.map +1 -0
- package/dist/decode/block.d.ts +11 -0
- package/dist/decode/block.js +28 -0
- package/dist/decode/block.js.map +1 -0
- package/dist/decode/block.test.d.ts +1 -0
- package/dist/decode/block.test.js +26 -0
- package/dist/decode/block.test.js.map +1 -0
- package/dist/decode/decompressFrame.d.ts +9 -0
- package/dist/decode/decompressFrame.js +147 -0
- package/dist/decode/decompressFrame.js.map +1 -0
- package/dist/decode/literals.corruption.test.d.ts +1 -0
- package/dist/decode/literals.corruption.test.js +26 -0
- package/dist/decode/literals.corruption.test.js.map +1 -0
- package/dist/decode/literals.d.ts +49 -0
- package/dist/decode/literals.js +300 -0
- package/dist/decode/literals.js.map +1 -0
- package/dist/decode/literals.test.d.ts +1 -0
- package/dist/decode/literals.test.js +52 -0
- package/dist/decode/literals.test.js.map +1 -0
- package/dist/decode/reconstruct.d.ts +13 -0
- package/dist/decode/reconstruct.js +80 -0
- package/dist/decode/reconstruct.js.map +1 -0
- package/dist/decode/reconstruct.test.d.ts +1 -0
- package/dist/decode/reconstruct.test.js +42 -0
- package/dist/decode/reconstruct.test.js.map +1 -0
- package/dist/decode/sequences.corruption.test.d.ts +1 -0
- package/dist/decode/sequences.corruption.test.js +32 -0
- package/dist/decode/sequences.corruption.test.js.map +1 -0
- package/dist/decode/sequences.d.ts +21 -0
- package/dist/decode/sequences.js +222 -0
- package/dist/decode/sequences.js.map +1 -0
- package/dist/decode/sequences.level1.test.d.ts +1 -0
- package/dist/decode/sequences.level1.test.js +35 -0
- package/dist/decode/sequences.level1.test.js.map +1 -0
- package/dist/decompress.d.ts +11 -0
- package/dist/decompress.js +59 -0
- package/dist/decompress.js.map +1 -0
- package/dist/dictionary/decoderDictionary.d.ts +15 -0
- package/dist/dictionary/decoderDictionary.js +116 -0
- package/dist/dictionary/decoderDictionary.js.map +1 -0
- package/dist/dictionary/decoderDictionary.test.d.ts +1 -0
- package/dist/dictionary/decoderDictionary.test.js +87 -0
- package/dist/dictionary/decoderDictionary.test.js.map +1 -0
- package/dist/encode/blockWriter.d.ts +5 -0
- package/dist/encode/blockWriter.js +27 -0
- package/dist/encode/blockWriter.js.map +1 -0
- package/dist/encode/blockWriter.test.d.ts +1 -0
- package/dist/encode/blockWriter.test.js +31 -0
- package/dist/encode/blockWriter.test.js.map +1 -0
- package/dist/encode/compressedBlock.d.ts +3 -0
- package/dist/encode/compressedBlock.js +449 -0
- package/dist/encode/compressedBlock.js.map +1 -0
- package/dist/encode/compressedBlock.test.d.ts +1 -0
- package/dist/encode/compressedBlock.test.js +63 -0
- package/dist/encode/compressedBlock.test.js.map +1 -0
- package/dist/encode/frameWriter.d.ts +4 -0
- package/dist/encode/frameWriter.js +34 -0
- package/dist/encode/frameWriter.js.map +1 -0
- package/dist/encode/frameWriter.test.d.ts +1 -0
- package/dist/encode/frameWriter.test.js +38 -0
- package/dist/encode/frameWriter.test.js.map +1 -0
- package/dist/encode/greedySequences.d.ts +7 -0
- package/dist/encode/greedySequences.js +82 -0
- package/dist/encode/greedySequences.js.map +1 -0
- package/dist/encode/greedySequences.test.d.ts +1 -0
- package/dist/encode/greedySequences.test.js +33 -0
- package/dist/encode/greedySequences.test.js.map +1 -0
- package/dist/entropy/fse.d.ts +33 -0
- package/dist/entropy/fse.js +217 -0
- package/dist/entropy/fse.js.map +1 -0
- package/dist/entropy/fse.test.d.ts +1 -0
- package/dist/entropy/fse.test.js +41 -0
- package/dist/entropy/fse.test.js.map +1 -0
- package/dist/entropy/huffman.d.ts +24 -0
- package/dist/entropy/huffman.js +70 -0
- package/dist/entropy/huffman.js.map +1 -0
- package/dist/entropy/huffman.test.d.ts +1 -0
- package/dist/entropy/huffman.test.js +22 -0
- package/dist/entropy/huffman.test.js.map +1 -0
- package/dist/entropy/index.d.ts +6 -0
- package/dist/entropy/index.js +5 -0
- package/dist/entropy/index.js.map +1 -0
- package/dist/entropy/predefined.d.ts +10 -0
- package/dist/entropy/predefined.js +18 -0
- package/dist/entropy/predefined.js.map +1 -0
- package/dist/entropy/weights.d.ts +20 -0
- package/dist/entropy/weights.js +108 -0
- package/dist/entropy/weights.js.map +1 -0
- package/dist/entropy/weights.test.d.ts +1 -0
- package/dist/entropy/weights.test.js +38 -0
- package/dist/entropy/weights.test.js.map +1 -0
- package/dist/errors.d.ts +7 -0
- package/dist/errors.js +13 -0
- package/dist/errors.js.map +1 -0
- package/dist/errors.test.d.ts +1 -0
- package/dist/errors.test.js +16 -0
- package/dist/errors.test.js.map +1 -0
- package/dist/frame/checksum.d.ts +17 -0
- package/dist/frame/checksum.js +94 -0
- package/dist/frame/checksum.js.map +1 -0
- package/dist/frame/checksum.test.d.ts +1 -0
- package/dist/frame/checksum.test.js +28 -0
- package/dist/frame/checksum.test.js.map +1 -0
- package/dist/frame/frameHeader.d.ts +26 -0
- package/dist/frame/frameHeader.js +127 -0
- package/dist/frame/frameHeader.js.map +1 -0
- package/dist/frame/frameHeader.test.d.ts +1 -0
- package/dist/frame/frameHeader.test.js +83 -0
- package/dist/frame/frameHeader.test.js.map +1 -0
- package/dist/frame/index.d.ts +4 -0
- package/dist/frame/index.js +4 -0
- package/dist/frame/index.js.map +1 -0
- package/dist/frame/skippable.d.ts +12 -0
- package/dist/frame/skippable.js +27 -0
- package/dist/frame/skippable.js.map +1 -0
- package/dist/frame/skippable.test.d.ts +1 -0
- package/dist/frame/skippable.test.js +35 -0
- package/dist/frame/skippable.test.js.map +1 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.js +3 -0
- package/dist/index.js.map +1 -0
- package/dist/tsconfig.tsbuildinfo +1 -0
- package/package.json +38 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"frameWriter.js","sourceRoot":"","sources":["../../src/encode/frameWriter.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,MAAM,UAAU,GAAG,UAAU,CAAC;AAE9B,MAAM,UAAU,gBAAgB,CAAC,WAAmB,EAAE,WAAoB,EAAc;IACtF,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,MAAM,CAAC,IAAI,CAAC,UAAU,GAAG,IAAI,EAAE,CAAC,UAAU,IAAI,CAAC,CAAC,GAAG,IAAI,EAAE,CAAC,UAAU,IAAI,EAAE,CAAC,GAAG,IAAI,EAAE,CAAC,UAAU,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;IAE/G,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,WAAW,IAAI,GAAG,EAAE,CAAC;QACvB,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC;QACd,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC;IAChB,CAAC;SAAM,IAAI,WAAW,IAAI,GAAG,GAAG,KAAK,GAAG,CAAC,EAAE,CAAC;QAC1C,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC;QACd,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC;IAChB,CAAC;SAAM,CAAC;QACN,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC;QACd,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC;IAChB,CAAC;IACD,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAClC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAEjB,IAAI,WAAW,IAAI,GAAG,EAAE,CAAC;QACvB,MAAM,CAAC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,CAAC;IAClC,CAAC;SAAM,IAAI,WAAW,IAAI,GAAG,GAAG,KAAK,GAAG,CAAC,EAAE,CAAC;QAC1C,MAAM,CAAC,IAAI,CAAC,CAAC,WAAW,GAAG,GAAG,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC,WAAW,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;IAC7E,CAAC;SAAM,CAAC;QACN,MAAM,CAAC,IAAI,CAAC,WAAW,GAAG,IAAI,EAAE,CAAC,WAAW,IAAI,CAAC,CAAC,GAAG,IAAI,EAAE,CAAC,WAAW,IAAI,EAAE,CAAC,GAAG,IAAI,EAAE,CAAC,WAAW,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;IACrH,CAAC;IAED,OAAO,IAAI,UAAU,CAAC,MAAM,CAAC,CAAC;AAAA,CAC/B"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import { describe, expect, it } from 'vitest';
|
|
2
|
+
import { writeFrameHeader } from './frameWriter.js';
|
|
3
|
+
const ZSTD_MAGIC_LE = [0x28, 0xb5, 0x2f, 0xfd];
|
|
4
|
+
describe('frameWriter', () => {
|
|
5
|
+
it('writeFrameHeader writes magic and 1-byte content size for small content', () => {
|
|
6
|
+
const h = writeFrameHeader(100, false);
|
|
7
|
+
expect(h.length).toBeGreaterThanOrEqual(6);
|
|
8
|
+
expect(h[0]).toBe(ZSTD_MAGIC_LE[0]);
|
|
9
|
+
expect(h[1]).toBe(ZSTD_MAGIC_LE[1]);
|
|
10
|
+
expect(h[2]).toBe(ZSTD_MAGIC_LE[2]);
|
|
11
|
+
expect(h[3]).toBe(ZSTD_MAGIC_LE[3]);
|
|
12
|
+
// FHD: single segment (1<<5), content size flag 0 (1 byte), no checksum
|
|
13
|
+
expect(h[4]).toBe(1 << 5);
|
|
14
|
+
expect(h[5]).toBe(100);
|
|
15
|
+
});
|
|
16
|
+
it('writeFrameHeader uses 2-byte content size for medium content', () => {
|
|
17
|
+
const h = writeFrameHeader(256 + 100, false);
|
|
18
|
+
expect(h.length).toBeGreaterThanOrEqual(7);
|
|
19
|
+
expect(h[4]).toBe((1 << 6) | (1 << 5));
|
|
20
|
+
// 256+100-256 = 100, LE
|
|
21
|
+
expect(h[5]).toBe(100);
|
|
22
|
+
expect(h[6]).toBe(0);
|
|
23
|
+
});
|
|
24
|
+
it('writeFrameHeader uses 4-byte content size for large content', () => {
|
|
25
|
+
const h = writeFrameHeader(300_000, false);
|
|
26
|
+
expect(h.length).toBeGreaterThanOrEqual(9);
|
|
27
|
+
expect(h[4]).toBe((2 << 6) | (1 << 5));
|
|
28
|
+
expect(h[5]).toBe(300_000 & 0xff);
|
|
29
|
+
expect(h[6]).toBe((300_000 >> 8) & 0xff);
|
|
30
|
+
expect(h[7]).toBe((300_000 >> 16) & 0xff);
|
|
31
|
+
expect(h[8]).toBe((300_000 >> 24) & 0xff);
|
|
32
|
+
});
|
|
33
|
+
it('writeFrameHeader sets checksum flag when hasChecksum true', () => {
|
|
34
|
+
const h = writeFrameHeader(5, true);
|
|
35
|
+
expect(h[4]).toBe((1 << 5) | (1 << 2));
|
|
36
|
+
});
|
|
37
|
+
});
|
|
38
|
+
//# sourceMappingURL=frameWriter.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"frameWriter.test.js","sourceRoot":"","sources":["../../src/encode/frameWriter.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAEpD,MAAM,aAAa,GAAG,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;AAE/C,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE,CAAC;IAC5B,EAAE,CAAC,yEAAyE,EAAE,GAAG,EAAE,CAAC;QAClF,MAAM,CAAC,GAAG,gBAAgB,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QACvC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC;QAC3C,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;QACpC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;QACpC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;QACpC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;QACpC,wEAAwE;QACxE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QAC1B,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAAA,CACxB,CAAC,CAAC;IAEH,EAAE,CAAC,8DAA8D,EAAE,GAAG,EAAE,CAAC;QACvE,MAAM,CAAC,GAAG,gBAAgB,CAAC,GAAG,GAAG,GAAG,EAAE,KAAK,CAAC,CAAC;QAC7C,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC;QAC3C,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACvC,wBAAwB;QACxB,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACvB,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAAA,CACtB,CAAC,CAAC;IAEH,EAAE,CAAC,6DAA6D,EAAE,GAAG,EAAE,CAAC;QACtE,MAAM,CAAC,GAAG,gBAAgB,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QAC3C,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC;QAC3C,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACvC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,CAAC;QAClC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;QACzC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;QAC1C,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;IAAA,CAC3C,CAAC,CAAC;IAEH,EAAE,CAAC,2DAA2D,EAAE,GAAG,EAAE,CAAC;QACpE,MAAM,CAAC,GAAG,gBAAgB,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;QACpC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAAA,CACxC,CAAC,CAAC;AAAA,CACJ,CAAC,CAAC"}
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
const WINDOW_SIZE = 128 * 1024;
|
|
2
|
+
const MIN_MATCH = 3;
|
|
3
|
+
const HASH_BITS = 16;
|
|
4
|
+
const HASH_SIZE = 1 << HASH_BITS;
|
|
5
|
+
function hash3(data, pos) {
|
|
6
|
+
const a = data[pos] ?? 0;
|
|
7
|
+
const b = data[pos + 1] ?? 0;
|
|
8
|
+
const c = data[pos + 2] ?? 0;
|
|
9
|
+
return ((a * 2654435761 + b * 2246822519 + c * 3266489917) >>> 0) >>> (32 - HASH_BITS);
|
|
10
|
+
}
|
|
11
|
+
function matchLength(data, a, b) {
|
|
12
|
+
const max = data.length - a;
|
|
13
|
+
let n = 0;
|
|
14
|
+
while (n < max && (data[a + n] ?? 0) === (data[b + n] ?? 0)) {
|
|
15
|
+
n++;
|
|
16
|
+
}
|
|
17
|
+
return n;
|
|
18
|
+
}
|
|
19
|
+
export function buildGreedySequences(input) {
|
|
20
|
+
if (input.length < MIN_MATCH) {
|
|
21
|
+
return { literals: input.slice(), sequences: [], trailingLiterals: input.length };
|
|
22
|
+
}
|
|
23
|
+
const lastPos = new Int32Array(HASH_SIZE);
|
|
24
|
+
lastPos.fill(-1);
|
|
25
|
+
const sequences = [];
|
|
26
|
+
const literalSpans = [];
|
|
27
|
+
let anchor = 0;
|
|
28
|
+
let pos = 0;
|
|
29
|
+
while (pos + MIN_MATCH <= input.length) {
|
|
30
|
+
const h = hash3(input, pos);
|
|
31
|
+
const candidate = lastPos[h] ?? -1;
|
|
32
|
+
lastPos[h] = pos;
|
|
33
|
+
if (candidate < 0) {
|
|
34
|
+
pos++;
|
|
35
|
+
continue;
|
|
36
|
+
}
|
|
37
|
+
const offset = pos - candidate;
|
|
38
|
+
if (offset <= 0 || offset > WINDOW_SIZE) {
|
|
39
|
+
pos++;
|
|
40
|
+
continue;
|
|
41
|
+
}
|
|
42
|
+
if ((input[pos] ?? 0) !== (input[candidate] ?? 0)) {
|
|
43
|
+
pos++;
|
|
44
|
+
continue;
|
|
45
|
+
}
|
|
46
|
+
const len = matchLength(input, pos, candidate);
|
|
47
|
+
if (len < MIN_MATCH) {
|
|
48
|
+
pos++;
|
|
49
|
+
continue;
|
|
50
|
+
}
|
|
51
|
+
const literalsLength = pos - anchor;
|
|
52
|
+
literalSpans.push([anchor, pos]);
|
|
53
|
+
sequences.push({
|
|
54
|
+
literalsLength,
|
|
55
|
+
// We currently emit non-repeat offsets only; repeated-offset coding can be layered later.
|
|
56
|
+
offset: offset + 3,
|
|
57
|
+
matchLength: len,
|
|
58
|
+
});
|
|
59
|
+
const matchEnd = pos + len;
|
|
60
|
+
for (let p = pos + 1; p + MIN_MATCH <= matchEnd; p++) {
|
|
61
|
+
const hp = hash3(input, p);
|
|
62
|
+
lastPos[hp] = p;
|
|
63
|
+
}
|
|
64
|
+
pos = matchEnd;
|
|
65
|
+
anchor = pos;
|
|
66
|
+
}
|
|
67
|
+
const trailingLiterals = input.length - anchor;
|
|
68
|
+
const totalLiteralBytes = input.length - sequences.reduce((sum, s) => sum + s.matchLength, 0);
|
|
69
|
+
const literals = new Uint8Array(totalLiteralBytes);
|
|
70
|
+
let out = 0;
|
|
71
|
+
for (const [start, end] of literalSpans) {
|
|
72
|
+
const chunk = input.subarray(start, end);
|
|
73
|
+
literals.set(chunk, out);
|
|
74
|
+
out += chunk.length;
|
|
75
|
+
}
|
|
76
|
+
if (trailingLiterals > 0) {
|
|
77
|
+
const tail = input.subarray(anchor);
|
|
78
|
+
literals.set(tail, out);
|
|
79
|
+
}
|
|
80
|
+
return { literals, sequences, trailingLiterals };
|
|
81
|
+
}
|
|
82
|
+
//# sourceMappingURL=greedySequences.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"greedySequences.js","sourceRoot":"","sources":["../../src/encode/greedySequences.ts"],"names":[],"mappings":"AAQA,MAAM,WAAW,GAAG,GAAG,GAAG,IAAI,CAAC;AAC/B,MAAM,SAAS,GAAG,CAAC,CAAC;AACpB,MAAM,SAAS,GAAG,EAAE,CAAC;AACrB,MAAM,SAAS,GAAG,CAAC,IAAI,SAAS,CAAC;AAEjC,SAAS,KAAK,CAAC,IAAgB,EAAE,GAAW,EAAU;IACpD,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACzB,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;IAC7B,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;IAC7B,OAAO,CAAC,CAAC,CAAC,GAAG,UAAU,GAAG,CAAC,GAAG,UAAU,GAAG,CAAC,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,EAAE,GAAG,SAAS,CAAC,CAAC;AAAA,CACxF;AAED,SAAS,WAAW,CAAC,IAAgB,EAAE,CAAS,EAAE,CAAS,EAAU;IACnE,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;IAC5B,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;QAC5D,CAAC,EAAE,CAAC;IACN,CAAC;IACD,OAAO,CAAC,CAAC;AAAA,CACV;AAED,MAAM,UAAU,oBAAoB,CAAC,KAAiB,EAAsB;IAC1E,IAAI,KAAK,CAAC,MAAM,GAAG,SAAS,EAAE,CAAC;QAC7B,OAAO,EAAE,QAAQ,EAAE,KAAK,CAAC,KAAK,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,gBAAgB,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC;IACpF,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,UAAU,CAAC,SAAS,CAAC,CAAC;IAC1C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAEjB,MAAM,SAAS,GAAe,EAAE,CAAC;IACjC,MAAM,YAAY,GAA4B,EAAE,CAAC;IACjD,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,GAAG,GAAG,CAAC,CAAC;IAEZ,OAAO,GAAG,GAAG,SAAS,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QACvC,MAAM,CAAC,GAAG,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QAC5B,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QACnC,OAAO,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC;QAEjB,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;YAClB,GAAG,EAAE,CAAC;YACN,SAAS;QACX,CAAC;QACD,MAAM,MAAM,GAAG,GAAG,GAAG,SAAS,CAAC;QAC/B,IAAI,MAAM,IAAI,CAAC,IAAI,MAAM,GAAG,WAAW,EAAE,CAAC;YACxC,GAAG,EAAE,CAAC;YACN,SAAS;QACX,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;YAClD,GAAG,EAAE,CAAC;YACN,SAAS;QACX,CAAC;QACD,MAAM,GAAG,GAAG,WAAW,CAAC,KAAK,EAAE,GAAG,EAAE,SAAS,CAAC,CAAC;QAC/C,IAAI,GAAG,GAAG,SAAS,EAAE,CAAC;YACpB,GAAG,EAAE,CAAC;YACN,SAAS;QACX,CAAC;QAED,MAAM,cAAc,GAAG,GAAG,GAAG,MAAM,CAAC;QACpC,YAAY,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC;QACjC,SAAS,CAAC,IAAI,CAAC;YACb,cAAc;YACd,0FAA0F;YAC1F,MAAM,EAAE,MAAM,GAAG,CAAC;YAClB,WAAW,EAAE,GAAG;SACjB,CAAC,CAAC;QAEH,MAAM,QAAQ,GAAG,GAAG,GAAG,GAAG,CAAC;QAC3B,KAAK,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,IAAI,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC;YACrD,MAAM,EAAE,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;YAC3B,OAAO,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC;QAClB,CAAC;QACD,GAAG,GAAG,QAAQ,CAAC;QACf,MAAM,GAAG,GAAG,CAAC;IACf,CAAC;IAED,MAAM,gBAAgB,GAAG,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC;IAC/C,MAAM,iBAAiB,GAAG,KAAK,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;IAC9F,MAAM,QAAQ,GAAG,IAAI,UAAU,CAAC,iBAAiB,CAAC,CAAC;IACnD,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,KAAK,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,IAAI,YAAY,EAAE,CAAC;QACxC,MAAM,KAAK,GAAG,KAAK,CAAC,QAAQ,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QACzC,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QACzB,GAAG,IAAI,KAAK,CAAC,MAAM,CAAC;IACtB,CAAC;IACD,IAAI,gBAAgB,GAAG,CAAC,EAAE,CAAC;QACzB,MAAM,IAAI,GAAG,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACpC,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IAC1B,CAAC;IAED,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,gBAAgB,EAAE,CAAC;AAAA,CAClD"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import { describe, expect, it } from 'vitest';
|
|
2
|
+
import { executeSequences } from '../decode/reconstruct.js';
|
|
3
|
+
import { buildGreedySequences } from './greedySequences.js';
|
|
4
|
+
describe('buildGreedySequences', () => {
|
|
5
|
+
it('reconstructs repetitive text via generated sequences', () => {
|
|
6
|
+
const input = new TextEncoder().encode('hello world hello world hello world hello world hello world ');
|
|
7
|
+
const plan = buildGreedySequences(input);
|
|
8
|
+
const output = executeSequences(plan.literals, plan.sequences, 128 * 1024);
|
|
9
|
+
expect(output).toEqual(input);
|
|
10
|
+
expect(plan.sequences.length).toBeGreaterThan(0);
|
|
11
|
+
});
|
|
12
|
+
it('reconstructs binary payload via generated sequences', () => {
|
|
13
|
+
const input = new Uint8Array(4096);
|
|
14
|
+
for (let i = 0; i < input.length; i++) {
|
|
15
|
+
input[i] = i & 0xff;
|
|
16
|
+
}
|
|
17
|
+
const doubled = new Uint8Array(input.length * 2);
|
|
18
|
+
doubled.set(input, 0);
|
|
19
|
+
doubled.set(input, input.length);
|
|
20
|
+
const plan = buildGreedySequences(doubled);
|
|
21
|
+
const output = executeSequences(plan.literals, plan.sequences, 128 * 1024);
|
|
22
|
+
expect(output).toEqual(doubled);
|
|
23
|
+
expect(plan.sequences.length).toBeGreaterThan(0);
|
|
24
|
+
});
|
|
25
|
+
it('returns literals-only result when no matches exist', () => {
|
|
26
|
+
const input = new TextEncoder().encode('abcdefghijklmnopqrstuvwxyz');
|
|
27
|
+
const plan = buildGreedySequences(input);
|
|
28
|
+
expect(plan.sequences).toEqual([]);
|
|
29
|
+
expect(plan.literals).toEqual(input);
|
|
30
|
+
expect(plan.trailingLiterals).toBe(input.length);
|
|
31
|
+
});
|
|
32
|
+
});
|
|
33
|
+
//# sourceMappingURL=greedySequences.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"greedySequences.test.js","sourceRoot":"","sources":["../../src/encode/greedySequences.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAC5D,OAAO,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC;AAE5D,QAAQ,CAAC,sBAAsB,EAAE,GAAG,EAAE,CAAC;IACrC,EAAE,CAAC,sDAAsD,EAAE,GAAG,EAAE,CAAC;QAC/D,MAAM,KAAK,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,8DAA8D,CAAC,CAAC;QACvG,MAAM,IAAI,GAAG,oBAAoB,CAAC,KAAK,CAAC,CAAC;QACzC,MAAM,MAAM,GAAG,gBAAgB,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,EAAE,GAAG,GAAG,IAAI,CAAC,CAAC;QAC3E,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAC9B,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;IAAA,CAClD,CAAC,CAAC;IAEH,EAAE,CAAC,qDAAqD,EAAE,GAAG,EAAE,CAAC;QAC9D,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,CAAC;QACnC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC;QACtB,CAAC;QACD,MAAM,OAAO,GAAG,IAAI,UAAU,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACjD,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QACtB,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;QACjC,MAAM,IAAI,GAAG,oBAAoB,CAAC,OAAO,CAAC,CAAC;QAC3C,MAAM,MAAM,GAAG,gBAAgB,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,EAAE,GAAG,GAAG,IAAI,CAAC,CAAC;QAC3E,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAChC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;IAAA,CAClD,CAAC,CAAC;IAEH,EAAE,CAAC,oDAAoD,EAAE,GAAG,EAAE,CAAC;QAC7D,MAAM,KAAK,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,4BAA4B,CAAC,CAAC;QACrE,MAAM,IAAI,GAAG,oBAAoB,CAAC,KAAK,CAAC,CAAC;QACzC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QACnC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QACrC,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAAA,CAClD,CAAC,CAAC;AAAA,CACJ,CAAC,CAAC"}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* FSE (Finite State Entropy) decode: table build and symbol decode.
|
|
3
|
+
* Zstd FSE streams are read backward.
|
|
4
|
+
*/
|
|
5
|
+
import type { BitReaderReverse } from '../bitstream/bitReaderReverse.js';
|
|
6
|
+
export interface FSEDecodeRow {
|
|
7
|
+
symbol: number;
|
|
8
|
+
numBits: number;
|
|
9
|
+
baseline: number;
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* Build FSE decode table from normalized counts.
|
|
13
|
+
* Counts are -1 for "less than 1" (full state reset) symbols.
|
|
14
|
+
* Based on zstd FSE_buildDTable logic.
|
|
15
|
+
*/
|
|
16
|
+
export declare function buildFSEDecodeTable(normalizedCounter: readonly number[], tableLog: number): FSEDecodeRow[];
|
|
17
|
+
/**
|
|
18
|
+
* Decode one FSE symbol. Updates state in place.
|
|
19
|
+
*/
|
|
20
|
+
export declare function decodeFSESymbol(table: readonly FSEDecodeRow[], _tableLog: number, reader: BitReaderReverse, state: {
|
|
21
|
+
value: number;
|
|
22
|
+
}): number;
|
|
23
|
+
/**
|
|
24
|
+
* Read FSE normalized counts from compressed header (readNCount).
|
|
25
|
+
* Used when symbol type uses RLE or Compressed mode (not Predefined).
|
|
26
|
+
* Bitstream is read forward, little-endian.
|
|
27
|
+
*/
|
|
28
|
+
export declare function readNCount(data: Uint8Array, offset: number, maxSymbolValue: number, maxTableLog: number): {
|
|
29
|
+
normalizedCounter: number[];
|
|
30
|
+
tableLog: number;
|
|
31
|
+
maxSymbolValue: number;
|
|
32
|
+
bytesRead: number;
|
|
33
|
+
};
|
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* FSE (Finite State Entropy) decode: table build and symbol decode.
|
|
3
|
+
* Zstd FSE streams are read backward.
|
|
4
|
+
*/
|
|
5
|
+
import { ZstdError } from '../errors.js';
|
|
6
|
+
const FSE_TABLESTEP = (tableSize) => (tableSize >> 1) + (tableSize >> 3) + 3;
|
|
7
|
+
/**
|
|
8
|
+
* Build FSE decode table from normalized counts.
|
|
9
|
+
* Counts are -1 for "less than 1" (full state reset) symbols.
|
|
10
|
+
* Based on zstd FSE_buildDTable logic.
|
|
11
|
+
*/
|
|
12
|
+
export function buildFSEDecodeTable(normalizedCounter, tableLog) {
|
|
13
|
+
if (!normalizedCounter || normalizedCounter.length === 0) {
|
|
14
|
+
throw new ZstdError('FSE: invalid normalized counter', 'corruption_detected');
|
|
15
|
+
}
|
|
16
|
+
const tableSize = 1 << tableLog;
|
|
17
|
+
const tableSymbol = new Array(tableSize);
|
|
18
|
+
const maxSymbolValue = normalizedCounter.length - 1;
|
|
19
|
+
const symbolNext = new Array(maxSymbolValue + 1);
|
|
20
|
+
let highThreshold = tableSize - 1;
|
|
21
|
+
for (let s = 0; s <= maxSymbolValue; s++) {
|
|
22
|
+
const n = normalizedCounter[s] ?? 0;
|
|
23
|
+
if (n === -1) {
|
|
24
|
+
tableSymbol[highThreshold] = s;
|
|
25
|
+
highThreshold--;
|
|
26
|
+
symbolNext[s] = 1;
|
|
27
|
+
}
|
|
28
|
+
else {
|
|
29
|
+
symbolNext[s] = n;
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
const step = FSE_TABLESTEP(tableSize);
|
|
33
|
+
const tableMask = tableSize - 1;
|
|
34
|
+
let position = 0;
|
|
35
|
+
for (let s = 0; s <= maxSymbolValue; s++) {
|
|
36
|
+
const n = normalizedCounter[s] ?? 0;
|
|
37
|
+
if (n <= 0)
|
|
38
|
+
continue;
|
|
39
|
+
for (let i = 0; i < n; i++) {
|
|
40
|
+
tableSymbol[position] = s;
|
|
41
|
+
do {
|
|
42
|
+
position = (position + step) & tableMask;
|
|
43
|
+
} while (position > highThreshold);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
const table = new Array(tableSize);
|
|
47
|
+
for (let u = 0; u < tableSize; u++) {
|
|
48
|
+
const symbol = tableSymbol[u];
|
|
49
|
+
if (symbol === undefined) {
|
|
50
|
+
throw new ZstdError('FSE invalid decode table', 'corruption_detected');
|
|
51
|
+
}
|
|
52
|
+
const nextState = symbolNext[symbol];
|
|
53
|
+
if (nextState === undefined)
|
|
54
|
+
throw new ZstdError('FSE invalid symbol', 'corruption_detected');
|
|
55
|
+
symbolNext[symbol] = nextState + 1;
|
|
56
|
+
const nbBits = tableLog - 31 + Math.clz32(nextState);
|
|
57
|
+
const baseline = (nextState << nbBits) - tableSize;
|
|
58
|
+
table[u] = { symbol, numBits: nbBits, baseline };
|
|
59
|
+
}
|
|
60
|
+
return table;
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Decode one FSE symbol. Updates state in place.
|
|
64
|
+
*/
|
|
65
|
+
export function decodeFSESymbol(table, _tableLog, reader, state) {
|
|
66
|
+
const row = table[state.value];
|
|
67
|
+
if (!row)
|
|
68
|
+
throw new ZstdError('FSE invalid state', 'corruption_detected');
|
|
69
|
+
const symbol = row.symbol;
|
|
70
|
+
const nbBits = row.numBits;
|
|
71
|
+
const baseline = row.baseline;
|
|
72
|
+
const bits = nbBits > 0 ? reader.readBits(nbBits) : 0;
|
|
73
|
+
state.value = baseline + bits;
|
|
74
|
+
return symbol;
|
|
75
|
+
}
|
|
76
|
+
function readU32LESafe(data, offset) {
|
|
77
|
+
return (((data[offset] ?? 0) |
|
|
78
|
+
((data[offset + 1] ?? 0) << 8) |
|
|
79
|
+
((data[offset + 2] ?? 0) << 16) |
|
|
80
|
+
((data[offset + 3] ?? 0) << 24)) >>>
|
|
81
|
+
0);
|
|
82
|
+
}
|
|
83
|
+
function highbit32(v) {
|
|
84
|
+
return 31 - Math.clz32(v >>> 0);
|
|
85
|
+
}
|
|
86
|
+
function ctz32(v) {
|
|
87
|
+
const x = v >>> 0;
|
|
88
|
+
if (x === 0)
|
|
89
|
+
return 32;
|
|
90
|
+
return 31 - Math.clz32((x & -x) >>> 0);
|
|
91
|
+
}
|
|
92
|
+
/**
|
|
93
|
+
* Read FSE normalized counts from compressed header (readNCount).
|
|
94
|
+
* Used when symbol type uses RLE or Compressed mode (not Predefined).
|
|
95
|
+
* Bitstream is read forward, little-endian.
|
|
96
|
+
*/
|
|
97
|
+
export function readNCount(data, offset, maxSymbolValue, maxTableLog) {
|
|
98
|
+
const remainingInput = data.length - offset;
|
|
99
|
+
if (remainingInput <= 0) {
|
|
100
|
+
throw new ZstdError('FSE readNCount: truncated input', 'corruption_detected');
|
|
101
|
+
}
|
|
102
|
+
const parseBody = (buf, hbSize) => {
|
|
103
|
+
const normalizedCounter = new Array(maxSymbolValue + 1).fill(0);
|
|
104
|
+
let ip = 0;
|
|
105
|
+
const iend = hbSize;
|
|
106
|
+
const maxSV1 = maxSymbolValue + 1;
|
|
107
|
+
let previous0 = false;
|
|
108
|
+
let charnum = 0;
|
|
109
|
+
let bitStream = readU32LESafe(buf, ip);
|
|
110
|
+
let nbBits = (bitStream & 0x0f) + 5;
|
|
111
|
+
if (nbBits > maxTableLog) {
|
|
112
|
+
throw new ZstdError('FSE readNCount: tableLog too large', 'corruption_detected');
|
|
113
|
+
}
|
|
114
|
+
const tableLog = nbBits;
|
|
115
|
+
bitStream >>>= 4;
|
|
116
|
+
let bitCount = 4;
|
|
117
|
+
let remaining = (1 << nbBits) + 1;
|
|
118
|
+
let threshold = 1 << nbBits;
|
|
119
|
+
nbBits += 1;
|
|
120
|
+
const reload = () => {
|
|
121
|
+
if (ip <= iend - 7 || ip + (bitCount >> 3) <= iend - 4) {
|
|
122
|
+
ip += bitCount >> 3;
|
|
123
|
+
bitCount &= 7;
|
|
124
|
+
}
|
|
125
|
+
else {
|
|
126
|
+
bitCount -= 8 * (iend - 4 - ip);
|
|
127
|
+
bitCount &= 31;
|
|
128
|
+
ip = iend - 4;
|
|
129
|
+
}
|
|
130
|
+
bitStream = readU32LESafe(buf, ip) >>> bitCount;
|
|
131
|
+
};
|
|
132
|
+
while (true) {
|
|
133
|
+
if (previous0) {
|
|
134
|
+
let repeats = ctz32((~bitStream | 0x80000000) >>> 0) >> 1;
|
|
135
|
+
while (repeats >= 12) {
|
|
136
|
+
charnum += 3 * 12;
|
|
137
|
+
if (ip <= iend - 7) {
|
|
138
|
+
ip += 3;
|
|
139
|
+
}
|
|
140
|
+
else {
|
|
141
|
+
bitCount -= 8 * (iend - 7 - ip);
|
|
142
|
+
bitCount &= 31;
|
|
143
|
+
ip = iend - 4;
|
|
144
|
+
}
|
|
145
|
+
bitStream = readU32LESafe(buf, ip) >>> bitCount;
|
|
146
|
+
repeats = ctz32((~bitStream | 0x80000000) >>> 0) >> 1;
|
|
147
|
+
}
|
|
148
|
+
charnum += 3 * repeats;
|
|
149
|
+
bitStream >>>= 2 * repeats;
|
|
150
|
+
bitCount += 2 * repeats;
|
|
151
|
+
const lastRepeat = bitStream & 3;
|
|
152
|
+
if (lastRepeat >= 3) {
|
|
153
|
+
throw new ZstdError('FSE readNCount: invalid zero repeat', 'corruption_detected');
|
|
154
|
+
}
|
|
155
|
+
charnum += lastRepeat;
|
|
156
|
+
bitCount += 2;
|
|
157
|
+
if (charnum >= maxSV1)
|
|
158
|
+
break;
|
|
159
|
+
reload();
|
|
160
|
+
}
|
|
161
|
+
const max = 2 * threshold - 1 - remaining;
|
|
162
|
+
let count;
|
|
163
|
+
if ((bitStream & (threshold - 1)) < max) {
|
|
164
|
+
count = bitStream & (threshold - 1);
|
|
165
|
+
bitCount += nbBits - 1;
|
|
166
|
+
}
|
|
167
|
+
else {
|
|
168
|
+
count = bitStream & (2 * threshold - 1);
|
|
169
|
+
if (count >= threshold)
|
|
170
|
+
count -= max;
|
|
171
|
+
bitCount += nbBits;
|
|
172
|
+
}
|
|
173
|
+
count -= 1;
|
|
174
|
+
if (count >= 0) {
|
|
175
|
+
remaining -= count;
|
|
176
|
+
}
|
|
177
|
+
else {
|
|
178
|
+
remaining += count;
|
|
179
|
+
}
|
|
180
|
+
normalizedCounter[charnum] = count;
|
|
181
|
+
charnum += 1;
|
|
182
|
+
previous0 = count === 0;
|
|
183
|
+
if (remaining < threshold) {
|
|
184
|
+
if (remaining <= 1)
|
|
185
|
+
break;
|
|
186
|
+
nbBits = highbit32(remaining) + 1;
|
|
187
|
+
threshold = 1 << (nbBits - 1);
|
|
188
|
+
}
|
|
189
|
+
if (charnum >= maxSV1)
|
|
190
|
+
break;
|
|
191
|
+
reload();
|
|
192
|
+
}
|
|
193
|
+
if (remaining !== 1) {
|
|
194
|
+
throw new ZstdError('FSE readNCount: invalid probability sum', 'corruption_detected');
|
|
195
|
+
}
|
|
196
|
+
if (charnum > maxSV1 || bitCount > 32) {
|
|
197
|
+
throw new ZstdError('FSE readNCount: corrupted header', 'corruption_detected');
|
|
198
|
+
}
|
|
199
|
+
ip += (bitCount + 7) >> 3;
|
|
200
|
+
const outMaxSymbol = charnum - 1;
|
|
201
|
+
for (let i = charnum; i <= maxSymbolValue; i++) {
|
|
202
|
+
normalizedCounter[i] = 0;
|
|
203
|
+
}
|
|
204
|
+
return { normalizedCounter, tableLog, maxSymbolValue: outMaxSymbol, bytesRead: ip };
|
|
205
|
+
};
|
|
206
|
+
if (remainingInput < 8) {
|
|
207
|
+
const scratch = new Uint8Array(8);
|
|
208
|
+
scratch.set(data.subarray(offset));
|
|
209
|
+
const parsed = parseBody(scratch, 8);
|
|
210
|
+
if (parsed.bytesRead > remainingInput) {
|
|
211
|
+
throw new ZstdError('FSE readNCount: truncated input', 'corruption_detected');
|
|
212
|
+
}
|
|
213
|
+
return parsed;
|
|
214
|
+
}
|
|
215
|
+
return parseBody(data.subarray(offset), remainingInput);
|
|
216
|
+
}
|
|
217
|
+
//# sourceMappingURL=fse.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fse.js","sourceRoot":"","sources":["../../src/entropy/fse.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAQzC,MAAM,aAAa,GAAG,CAAC,SAAiB,EAAE,EAAE,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC,GAAG,CAAC,SAAS,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;AAErF;;;;GAIG;AACH,MAAM,UAAU,mBAAmB,CAAC,iBAAoC,EAAE,QAAgB,EAAkB;IAC1G,IAAI,CAAC,iBAAiB,IAAI,iBAAiB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzD,MAAM,IAAI,SAAS,CAAC,iCAAiC,EAAE,qBAAqB,CAAC,CAAC;IAChF,CAAC;IACD,MAAM,SAAS,GAAG,CAAC,IAAI,QAAQ,CAAC;IAChC,MAAM,WAAW,GAAa,IAAI,KAAK,CAAC,SAAS,CAAC,CAAC;IACnD,MAAM,cAAc,GAAG,iBAAiB,CAAC,MAAM,GAAG,CAAC,CAAC;IAEpD,MAAM,UAAU,GAAa,IAAI,KAAK,CAAC,cAAc,GAAG,CAAC,CAAC,CAAC;IAC3D,IAAI,aAAa,GAAG,SAAS,GAAG,CAAC,CAAC;IAElC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,cAAc,EAAE,CAAC,EAAE,EAAE,CAAC;QACzC,MAAM,CAAC,GAAG,iBAAiB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACpC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;YACb,WAAW,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;YAC/B,aAAa,EAAE,CAAC;YAChB,UAAU,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QACpB,CAAC;aAAM,CAAC;YACN,UAAU,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QACpB,CAAC;IACH,CAAC;IAED,MAAM,IAAI,GAAG,aAAa,CAAC,SAAS,CAAC,CAAC;IACtC,MAAM,SAAS,GAAG,SAAS,GAAG,CAAC,CAAC;IAChC,IAAI,QAAQ,GAAG,CAAC,CAAC;IAEjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,cAAc,EAAE,CAAC,EAAE,EAAE,CAAC;QACzC,MAAM,CAAC,GAAG,iBAAiB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACpC,IAAI,CAAC,IAAI,CAAC;YAAE,SAAS;QACrB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,WAAW,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;YAC1B,GAAG,CAAC;gBACF,QAAQ,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,SAAS,CAAC;YAC3C,CAAC,QAAQ,QAAQ,GAAG,aAAa,EAAE;QACrC,CAAC;IACH,CAAC;IAED,MAAM,KAAK,GAAmB,IAAI,KAAK,CAAC,SAAS,CAAC,CAAC;IACnD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC;QACnC,MAAM,MAAM,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;QAC9B,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACzB,MAAM,IAAI,SAAS,CAAC,0BAA0B,EAAE,qBAAqB,CAAC,CAAC;QACzE,CAAC;QACD,MAAM,SAAS,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;QACrC,IAAI,SAAS,KAAK,SAAS;YAAE,MAAM,IAAI,SAAS,CAAC,oBAAoB,EAAE,qBAAqB,CAAC,CAAC;QAC9F,UAAU,CAAC,MAAM,CAAC,GAAG,SAAS,GAAG,CAAC,CAAC;QAEnC,MAAM,MAAM,GAAG,QAAQ,GAAG,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QACrD,MAAM,QAAQ,GAAG,CAAC,SAAS,IAAI,MAAM,CAAC,GAAG,SAAS,CAAC;QACnD,KAAK,CAAC,CAAC,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;IACnD,CAAC;IAED,OAAO,KAAK,CAAC;AAAA,CACd;AAED;;GAEG;AACH,MAAM,UAAU,eAAe,CAC7B,KAA8B,EAC9B,SAAiB,EACjB,MAAwB,EACxB,KAAwB,EAChB;IACR,MAAM,GAAG,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAC/B,IAAI,CAAC,GAAG;QAAE,MAAM,IAAI,SAAS,CAAC,mBAAmB,EAAE,qBAAqB,CAAC,CAAC;IAC1E,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC;IAC1B,MAAM,MAAM,GAAG,GAAG,CAAC,OAAO,CAAC;IAC3B,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC;IAC9B,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACtD,KAAK,CAAC,KAAK,GAAG,QAAQ,GAAG,IAAI,CAAC;IAC9B,OAAO,MAAM,CAAC;AAAA,CACf;AAED,SAAS,aAAa,CAAC,IAAgB,EAAE,MAAc,EAAU;IAC/D,OAAO,CACL,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAClB,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC;QAC9B,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;QAC/B,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAClC,CAAC,CACF,CAAC;AAAA,CACH;AAED,SAAS,SAAS,CAAC,CAAS,EAAU;IACpC,OAAO,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;AAAA,CACjC;AAED,SAAS,KAAK,CAAC,CAAS,EAAU;IAChC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAClB,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IACvB,OAAO,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;AAAA,CACxC;AAED;;;;GAIG;AACH,MAAM,UAAU,UAAU,CACxB,IAAgB,EAChB,MAAc,EACd,cAAsB,EACtB,WAAmB,EAC2E;IAC9F,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IAC5C,IAAI,cAAc,IAAI,CAAC,EAAE,CAAC;QACxB,MAAM,IAAI,SAAS,CAAC,iCAAiC,EAAE,qBAAqB,CAAC,CAAC;IAChF,CAAC;IAED,MAAM,SAAS,GAAG,CAAC,GAAe,EAAE,MAAc,EAAE,EAAE,CAAC;QACrD,MAAM,iBAAiB,GAAG,IAAI,KAAK,CAAS,cAAc,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACxE,IAAI,EAAE,GAAG,CAAC,CAAC;QACX,MAAM,IAAI,GAAG,MAAM,CAAC;QACpB,MAAM,MAAM,GAAG,cAAc,GAAG,CAAC,CAAC;QAClC,IAAI,SAAS,GAAG,KAAK,CAAC;QACtB,IAAI,OAAO,GAAG,CAAC,CAAC;QAEhB,IAAI,SAAS,GAAG,aAAa,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QACvC,IAAI,MAAM,GAAG,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;QACpC,IAAI,MAAM,GAAG,WAAW,EAAE,CAAC;YACzB,MAAM,IAAI,SAAS,CAAC,oCAAoC,EAAE,qBAAqB,CAAC,CAAC;QACnF,CAAC;QACD,MAAM,QAAQ,GAAG,MAAM,CAAC;QACxB,SAAS,MAAM,CAAC,CAAC;QACjB,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,IAAI,SAAS,GAAG,CAAC,CAAC,IAAI,MAAM,CAAC,GAAG,CAAC,CAAC;QAClC,IAAI,SAAS,GAAG,CAAC,IAAI,MAAM,CAAC;QAC5B,MAAM,IAAI,CAAC,CAAC;QAEZ,MAAM,MAAM,GAAG,GAAG,EAAE,CAAC;YACnB,IAAI,EAAE,IAAI,IAAI,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,QAAQ,IAAI,CAAC,CAAC,IAAI,IAAI,GAAG,CAAC,EAAE,CAAC;gBACvD,EAAE,IAAI,QAAQ,IAAI,CAAC,CAAC;gBACpB,QAAQ,IAAI,CAAC,CAAC;YAChB,CAAC;iBAAM,CAAC;gBACN,QAAQ,IAAI,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;gBAChC,QAAQ,IAAI,EAAE,CAAC;gBACf,EAAE,GAAG,IAAI,GAAG,CAAC,CAAC;YAChB,CAAC;YACD,SAAS,GAAG,aAAa,CAAC,GAAG,EAAE,EAAE,CAAC,KAAK,QAAQ,CAAC;QAAA,CACjD,CAAC;QAEF,OAAO,IAAI,EAAE,CAAC;YACZ,IAAI,SAAS,EAAE,CAAC;gBACd,IAAI,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,SAAS,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC;gBAC1D,OAAO,OAAO,IAAI,EAAE,EAAE,CAAC;oBACrB,OAAO,IAAI,CAAC,GAAG,EAAE,CAAC;oBAClB,IAAI,EAAE,IAAI,IAAI,GAAG,CAAC,EAAE,CAAC;wBACnB,EAAE,IAAI,CAAC,CAAC;oBACV,CAAC;yBAAM,CAAC;wBACN,QAAQ,IAAI,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;wBAChC,QAAQ,IAAI,EAAE,CAAC;wBACf,EAAE,GAAG,IAAI,GAAG,CAAC,CAAC;oBAChB,CAAC;oBACD,SAAS,GAAG,aAAa,CAAC,GAAG,EAAE,EAAE,CAAC,KAAK,QAAQ,CAAC;oBAChD,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,SAAS,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC;gBACxD,CAAC;gBACD,OAAO,IAAI,CAAC,GAAG,OAAO,CAAC;gBACvB,SAAS,MAAM,CAAC,GAAG,OAAO,CAAC;gBAC3B,QAAQ,IAAI,CAAC,GAAG,OAAO,CAAC;gBAExB,MAAM,UAAU,GAAG,SAAS,GAAG,CAAC,CAAC;gBACjC,IAAI,UAAU,IAAI,CAAC,EAAE,CAAC;oBACpB,MAAM,IAAI,SAAS,CAAC,qCAAqC,EAAE,qBAAqB,CAAC,CAAC;gBACpF,CAAC;gBACD,OAAO,IAAI,UAAU,CAAC;gBACtB,QAAQ,IAAI,CAAC,CAAC;gBAEd,IAAI,OAAO,IAAI,MAAM;oBAAE,MAAM;gBAC7B,MAAM,EAAE,CAAC;YACX,CAAC;YAED,MAAM,GAAG,GAAG,CAAC,GAAG,SAAS,GAAG,CAAC,GAAG,SAAS,CAAC;YAC1C,IAAI,KAAa,CAAC;YAClB,IAAI,CAAC,SAAS,GAAG,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,EAAE,CAAC;gBACxC,KAAK,GAAG,SAAS,GAAG,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC;gBACpC,QAAQ,IAAI,MAAM,GAAG,CAAC,CAAC;YACzB,CAAC;iBAAM,CAAC;gBACN,KAAK,GAAG,SAAS,GAAG,CAAC,CAAC,GAAG,SAAS,GAAG,CAAC,CAAC,CAAC;gBACxC,IAAI,KAAK,IAAI,SAAS;oBAAE,KAAK,IAAI,GAAG,CAAC;gBACrC,QAAQ,IAAI,MAAM,CAAC;YACrB,CAAC;YAED,KAAK,IAAI,CAAC,CAAC;YACX,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;gBACf,SAAS,IAAI,KAAK,CAAC;YACrB,CAAC;iBAAM,CAAC;gBACN,SAAS,IAAI,KAAK,CAAC;YACrB,CAAC;YAED,iBAAiB,CAAC,OAAO,CAAC,GAAG,KAAK,CAAC;YACnC,OAAO,IAAI,CAAC,CAAC;YACb,SAAS,GAAG,KAAK,KAAK,CAAC,CAAC;YAExB,IAAI,SAAS,GAAG,SAAS,EAAE,CAAC;gBAC1B,IAAI,SAAS,IAAI,CAAC;oBAAE,MAAM;gBAC1B,MAAM,GAAG,SAAS,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;gBAClC,SAAS,GAAG,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAChC,CAAC;YAED,IAAI,OAAO,IAAI,MAAM;gBAAE,MAAM;YAC7B,MAAM,EAAE,CAAC;QACX,CAAC;QAED,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;YACpB,MAAM,IAAI,SAAS,CAAC,yCAAyC,EAAE,qBAAqB,CAAC,CAAC;QACxF,CAAC;QACD,IAAI,OAAO,GAAG,MAAM,IAAI,QAAQ,GAAG,EAAE,EAAE,CAAC;YACtC,MAAM,IAAI,SAAS,CAAC,kCAAkC,EAAE,qBAAqB,CAAC,CAAC;QACjF,CAAC;QAED,EAAE,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;QAC1B,MAAM,YAAY,GAAG,OAAO,GAAG,CAAC,CAAC;QACjC,KAAK,IAAI,CAAC,GAAG,OAAO,EAAE,CAAC,IAAI,cAAc,EAAE,CAAC,EAAE,EAAE,CAAC;YAC/C,iBAAiB,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAC3B,CAAC;QAED,OAAO,EAAE,iBAAiB,EAAE,QAAQ,EAAE,cAAc,EAAE,YAAY,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC;IAAA,CACrF,CAAC;IAEF,IAAI,cAAc,GAAG,CAAC,EAAE,CAAC;QACvB,MAAM,OAAO,GAAG,IAAI,UAAU,CAAC,CAAC,CAAC,CAAC;QAClC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;QACnC,MAAM,MAAM,GAAG,SAAS,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QACrC,IAAI,MAAM,CAAC,SAAS,GAAG,cAAc,EAAE,CAAC;YACtC,MAAM,IAAI,SAAS,CAAC,iCAAiC,EAAE,qBAAqB,CAAC,CAAC;QAChF,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,OAAO,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,cAAc,CAAC,CAAC;AAAA,CACzD"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import { describe, expect, it } from 'vitest';
|
|
2
|
+
import { BitReaderReverse } from '../bitstream/bitReaderReverse.js';
|
|
3
|
+
import { buildFSEDecodeTable, decodeFSESymbol, readNCount } from './fse.js';
|
|
4
|
+
import { LITERALS_LENGTH_DEFAULT_DISTRIBUTION, LITERALS_LENGTH_TABLE_LOG } from './predefined.js';
|
|
5
|
+
describe('FSE', () => {
|
|
6
|
+
it('readNCount decodes 2-symbol distribution', () => {
|
|
7
|
+
const result = readNCount(new Uint8Array([0x10, 0x3f, 0x01]), 0, 255, 12);
|
|
8
|
+
expect(result.tableLog).toBe(5);
|
|
9
|
+
expect(result.maxSymbolValue).toBe(1);
|
|
10
|
+
expect(result.normalizedCounter[0]).toBe(16);
|
|
11
|
+
expect(result.normalizedCounter[1]).toBe(16);
|
|
12
|
+
expect(result.bytesRead).toBe(2);
|
|
13
|
+
});
|
|
14
|
+
it('readNCount matches short-buffer and padded-buffer parsing', () => {
|
|
15
|
+
const short = new Uint8Array([0x10, 0x3f, 0x01]);
|
|
16
|
+
const padded = new Uint8Array([0x10, 0x3f, 0x01, 0, 0, 0, 0, 0]);
|
|
17
|
+
const a = readNCount(short, 0, 255, 12);
|
|
18
|
+
const b = readNCount(padded, 0, 255, 12);
|
|
19
|
+
expect(a.tableLog).toBe(b.tableLog);
|
|
20
|
+
expect(a.maxSymbolValue).toBe(b.maxSymbolValue);
|
|
21
|
+
expect(a.bytesRead).toBe(2);
|
|
22
|
+
expect(b.bytesRead).toBe(2);
|
|
23
|
+
expect(a.normalizedCounter.slice(0, 8)).toEqual(b.normalizedCounter.slice(0, 8));
|
|
24
|
+
});
|
|
25
|
+
it('buildFSEDecodeTable from predefined literals length', () => {
|
|
26
|
+
const table = buildFSEDecodeTable(LITERALS_LENGTH_DEFAULT_DISTRIBUTION, LITERALS_LENGTH_TABLE_LOG);
|
|
27
|
+
expect(table.length).toBe(1 << LITERALS_LENGTH_TABLE_LOG);
|
|
28
|
+
expect(table.every((r) => r.symbol >= 0 && r.numBits >= 0)).toBe(true);
|
|
29
|
+
});
|
|
30
|
+
it('decodeFSESymbol updates state', () => {
|
|
31
|
+
const table = buildFSEDecodeTable(LITERALS_LENGTH_DEFAULT_DISTRIBUTION, LITERALS_LENGTH_TABLE_LOG);
|
|
32
|
+
const data = new Uint8Array([0x55, 0xaa, 0x01]);
|
|
33
|
+
const reader = new BitReaderReverse(data, 0, 3);
|
|
34
|
+
reader.skipPadding();
|
|
35
|
+
const state = { value: 0 };
|
|
36
|
+
const symbol = decodeFSESymbol(table, LITERALS_LENGTH_TABLE_LOG, reader, state);
|
|
37
|
+
expect(typeof symbol).toBe('number');
|
|
38
|
+
expect(state.value).toBeGreaterThanOrEqual(0);
|
|
39
|
+
});
|
|
40
|
+
});
|
|
41
|
+
//# sourceMappingURL=fse.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fse.test.js","sourceRoot":"","sources":["../../src/entropy/fse.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,gBAAgB,EAAE,MAAM,kCAAkC,CAAC;AACpE,OAAO,EAAE,mBAAmB,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AAC5E,OAAO,EAAE,oCAAoC,EAAE,yBAAyB,EAAE,MAAM,iBAAiB,CAAC;AAElG,QAAQ,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC;IACpB,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE,CAAC;QACnD,MAAM,MAAM,GAAG,UAAU,CAAC,IAAI,UAAU,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;QAC1E,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAChC,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACtC,MAAM,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC7C,MAAM,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC7C,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAAA,CAClC,CAAC,CAAC;IAEH,EAAE,CAAC,2DAA2D,EAAE,GAAG,EAAE,CAAC;QACpE,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;QACjD,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QACjE,MAAM,CAAC,GAAG,UAAU,CAAC,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;QACxC,MAAM,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;QACzC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;QACpC,MAAM,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC;QAChD,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC5B,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC5B,MAAM,CAAC,CAAC,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAAA,CAClF,CAAC,CAAC;IACH,EAAE,CAAC,qDAAqD,EAAE,GAAG,EAAE,CAAC;QAC9D,MAAM,KAAK,GAAG,mBAAmB,CAAC,oCAAoC,EAAE,yBAAyB,CAAC,CAAC;QACnG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,yBAAyB,CAAC,CAAC;QAC1D,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAAA,CACxE,CAAC,CAAC;IAEH,EAAE,CAAC,+BAA+B,EAAE,GAAG,EAAE,CAAC;QACxC,MAAM,KAAK,GAAG,mBAAmB,CAAC,oCAAoC,EAAE,yBAAyB,CAAC,CAAC;QACnG,MAAM,IAAI,GAAG,IAAI,UAAU,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;QAChD,MAAM,MAAM,GAAG,IAAI,gBAAgB,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;QAChD,MAAM,CAAC,WAAW,EAAE,CAAC;QACrB,MAAM,KAAK,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;QAC3B,MAAM,MAAM,GAAG,eAAe,CAAC,KAAK,EAAE,yBAAyB,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;QAChF,MAAM,CAAC,OAAO,MAAM,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACrC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC;IAAA,CAC/C,CAAC,CAAC;AAAA,CACJ,CAAC,CAAC"}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Huffman decode: build decode table from weights, decode symbols.
|
|
3
|
+
* Zstd Huffman streams are read backward.
|
|
4
|
+
*/
|
|
5
|
+
import type { BitReaderReverse } from '../bitstream/bitReaderReverse.js';
|
|
6
|
+
export interface HuffmanDecodeRow {
|
|
7
|
+
symbol: number;
|
|
8
|
+
numBits: number;
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* Convert weights to number of bits per symbol.
|
|
12
|
+
* Weight = 0 means symbol not present. Weight 1 = least frequent, max weight = most frequent.
|
|
13
|
+
* Number_of_Bits = Weight ? (Max_Number_of_Bits + 1 - Weight) : 0
|
|
14
|
+
*/
|
|
15
|
+
export declare function weightsToNumBits(weights: readonly number[], maxNumBits: number): number[];
|
|
16
|
+
/**
|
|
17
|
+
* Build Huffman decode table from symbol bit lengths.
|
|
18
|
+
* Returns table indexed by prefix code (first maxNumBits bits).
|
|
19
|
+
*/
|
|
20
|
+
export declare function buildHuffmanDecodeTable(numBits: readonly number[], maxNumBits: number): HuffmanDecodeRow[];
|
|
21
|
+
/**
|
|
22
|
+
* Decode one Huffman symbol. Reader must be positioned at start of code.
|
|
23
|
+
*/
|
|
24
|
+
export declare function decodeHuffmanSymbol(table: readonly HuffmanDecodeRow[], maxNumBits: number, reader: BitReaderReverse): number;
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Huffman decode: build decode table from weights, decode symbols.
|
|
3
|
+
* Zstd Huffman streams are read backward.
|
|
4
|
+
*/
|
|
5
|
+
import { ZstdError } from '../errors.js';
|
|
6
|
+
/**
|
|
7
|
+
* Convert weights to number of bits per symbol.
|
|
8
|
+
* Weight = 0 means symbol not present. Weight 1 = least frequent, max weight = most frequent.
|
|
9
|
+
* Number_of_Bits = Weight ? (Max_Number_of_Bits + 1 - Weight) : 0
|
|
10
|
+
*/
|
|
11
|
+
export function weightsToNumBits(weights, maxNumBits) {
|
|
12
|
+
const result = [];
|
|
13
|
+
for (let i = 0; i < weights.length; i++) {
|
|
14
|
+
const w = weights[i] ?? 0;
|
|
15
|
+
result.push(w ? maxNumBits + 1 - w : 0);
|
|
16
|
+
}
|
|
17
|
+
return result;
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Build Huffman decode table from symbol bit lengths.
|
|
21
|
+
* Returns table indexed by prefix code (first maxNumBits bits).
|
|
22
|
+
*/
|
|
23
|
+
export function buildHuffmanDecodeTable(numBits, maxNumBits) {
|
|
24
|
+
const tableSize = 1 << maxNumBits;
|
|
25
|
+
const table = new Array(tableSize);
|
|
26
|
+
const rankCount = new Array(maxNumBits + 1).fill(0);
|
|
27
|
+
for (let s = 0; s < numBits.length; s++) {
|
|
28
|
+
const len = numBits[s] ?? 0;
|
|
29
|
+
if (len < 0 || len > maxNumBits) {
|
|
30
|
+
throw new ZstdError('Huffman invalid bit length', 'corruption_detected');
|
|
31
|
+
}
|
|
32
|
+
rankCount[len] = (rankCount[len] ?? 0) + 1;
|
|
33
|
+
}
|
|
34
|
+
const rankIdx = new Array(maxNumBits + 1).fill(0);
|
|
35
|
+
rankIdx[maxNumBits] = 0;
|
|
36
|
+
for (let len = maxNumBits; len >= 1; len--) {
|
|
37
|
+
const current = rankIdx[len] ?? 0;
|
|
38
|
+
rankIdx[len - 1] = current + (rankCount[len] ?? 0) * (1 << (maxNumBits - len));
|
|
39
|
+
}
|
|
40
|
+
if (rankIdx[0] !== tableSize) {
|
|
41
|
+
throw new ZstdError('Huffman invalid tree', 'corruption_detected');
|
|
42
|
+
}
|
|
43
|
+
for (let symbol = 0; symbol < numBits.length; symbol++) {
|
|
44
|
+
const len = numBits[symbol] ?? 0;
|
|
45
|
+
if (len === 0)
|
|
46
|
+
continue;
|
|
47
|
+
const code = rankIdx[len] ?? 0;
|
|
48
|
+
const span = 1 << (maxNumBits - len);
|
|
49
|
+
for (let i = 0; i < span; i++) {
|
|
50
|
+
table[code + i] = { symbol, numBits: len };
|
|
51
|
+
}
|
|
52
|
+
rankIdx[len] = code + span;
|
|
53
|
+
}
|
|
54
|
+
return table;
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* Decode one Huffman symbol. Reader must be positioned at start of code.
|
|
58
|
+
*/
|
|
59
|
+
export function decodeHuffmanSymbol(table, maxNumBits, reader) {
|
|
60
|
+
const peek = reader.readBits(maxNumBits);
|
|
61
|
+
const row = table[peek];
|
|
62
|
+
if (!row)
|
|
63
|
+
throw new ZstdError('Huffman invalid code', 'corruption_detected');
|
|
64
|
+
const unread = maxNumBits - row.numBits;
|
|
65
|
+
if (unread > 0) {
|
|
66
|
+
reader.unreadBits(unread);
|
|
67
|
+
}
|
|
68
|
+
return row.symbol;
|
|
69
|
+
}
|
|
70
|
+
//# sourceMappingURL=huffman.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"huffman.js","sourceRoot":"","sources":["../../src/entropy/huffman.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAOzC;;;;GAIG;AACH,MAAM,UAAU,gBAAgB,CAAC,OAA0B,EAAE,UAAkB,EAAY;IACzF,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,MAAM,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAC1B,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1C,CAAC;IACD,OAAO,MAAM,CAAC;AAAA,CACf;AAED;;;GAGG;AACH,MAAM,UAAU,uBAAuB,CAAC,OAA0B,EAAE,UAAkB,EAAsB;IAC1G,MAAM,SAAS,GAAG,CAAC,IAAI,UAAU,CAAC;IAClC,MAAM,KAAK,GAAuB,IAAI,KAAK,CAAC,SAAS,CAAC,CAAC;IACvD,MAAM,SAAS,GAAG,IAAI,KAAK,CAAS,UAAU,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC5D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,MAAM,GAAG,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAC5B,IAAI,GAAG,GAAG,CAAC,IAAI,GAAG,GAAG,UAAU,EAAE,CAAC;YAChC,MAAM,IAAI,SAAS,CAAC,4BAA4B,EAAE,qBAAqB,CAAC,CAAC;QAC3E,CAAC;QACD,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;IAC7C,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,KAAK,CAAS,UAAU,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC1D,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;IACxB,KAAK,IAAI,GAAG,GAAG,UAAU,EAAE,GAAG,IAAI,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC;QAC3C,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAClC,OAAO,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,OAAO,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,UAAU,GAAG,GAAG,CAAC,CAAC,CAAC;IACjF,CAAC;IACD,IAAI,OAAO,CAAC,CAAC,CAAC,KAAK,SAAS,EAAE,CAAC;QAC7B,MAAM,IAAI,SAAS,CAAC,sBAAsB,EAAE,qBAAqB,CAAC,CAAC;IACrE,CAAC;IAED,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,CAAC;QACvD,MAAM,GAAG,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACjC,IAAI,GAAG,KAAK,CAAC;YAAE,SAAS;QACxB,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC/B,MAAM,IAAI,GAAG,CAAC,IAAI,CAAC,UAAU,GAAG,GAAG,CAAC,CAAC;QACrC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC;YAC9B,KAAK,CAAC,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC;QAC7C,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;IAC7B,CAAC;IAED,OAAO,KAAK,CAAC;AAAA,CACd;AAED;;GAEG;AACH,MAAM,UAAU,mBAAmB,CACjC,KAAkC,EAClC,UAAkB,EAClB,MAAwB,EAChB;IACR,MAAM,IAAI,GAAG,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;IACzC,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC;IACxB,IAAI,CAAC,GAAG;QAAE,MAAM,IAAI,SAAS,CAAC,sBAAsB,EAAE,qBAAqB,CAAC,CAAC;IAC7E,MAAM,MAAM,GAAG,UAAU,GAAG,GAAG,CAAC,OAAO,CAAC;IACxC,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;QACf,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;IAC5B,CAAC;IACD,OAAO,GAAG,CAAC,MAAM,CAAC;AAAA,CACnB"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { describe, expect, it } from 'vitest';
|
|
2
|
+
import { BitReaderReverse } from '../bitstream/bitReaderReverse.js';
|
|
3
|
+
import { buildHuffmanDecodeTable, decodeHuffmanSymbol } from './huffman.js';
|
|
4
|
+
describe('Huffman', () => {
|
|
5
|
+
it('builds canonical decode table in rank order', () => {
|
|
6
|
+
const table = buildHuffmanDecodeTable([1, 2, 2], 2);
|
|
7
|
+
expect(table.map((r) => r?.symbol)).toEqual([1, 2, 0, 0]);
|
|
8
|
+
expect(table.map((r) => r?.numBits)).toEqual([2, 2, 1, 1]);
|
|
9
|
+
});
|
|
10
|
+
it('decode consumes symbol bit length (not max bits)', () => {
|
|
11
|
+
const table = buildHuffmanDecodeTable([1, 2, 2], 2);
|
|
12
|
+
// Reverse bitstream payload is first byte; last byte is only the end-mark.
|
|
13
|
+
// For first byte 0b10100000, the first 2-bit peek is binary 10 (index 2),
|
|
14
|
+
// whose row has numBits=1, so one bit must remain unread.
|
|
15
|
+
const reader = new BitReaderReverse(new Uint8Array([0xa0, 0x01]), 0, 2);
|
|
16
|
+
reader.skipPadding();
|
|
17
|
+
const symbol = decodeHuffmanSymbol(table, 2, reader);
|
|
18
|
+
expect(symbol).toBe(0);
|
|
19
|
+
expect(reader.readBits(1)).toBe(0);
|
|
20
|
+
});
|
|
21
|
+
});
|
|
22
|
+
//# sourceMappingURL=huffman.test.js.map
|