zstdify 1.1.0 → 1.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md
CHANGED
|
@@ -5,6 +5,8 @@
|
|
|
5
5
|
[![Tests][tests-badge]][tests-url]
|
|
6
6
|
[![Coverage][coverage-badge]][coverage-url]
|
|
7
7
|
|
|
8
|
+
|
|
9
|
+
|
|
8
10
|
Pure JavaScript/TypeScript zstd compression/decompression library. No native dependencies, works in Node.js and browsers.
|
|
9
11
|
|
|
10
12
|
## Features
|
|
@@ -23,7 +25,7 @@ Pure JavaScript/TypeScript zstd compression/decompression library. No native dep
|
|
|
23
25
|
- **Dictionary generation**:
|
|
24
26
|
- Pure TypeScript dictionary training from sample payloads.
|
|
25
27
|
- Zstd-inspired training options (`fastcover`/`cover`/`legacy` style knobs).
|
|
26
|
-
- **Interop-focused**: `zstdify` output is decoded by the official `zstd` CLI
|
|
28
|
+
- **Interop-focused**: `zstdify` output is decoded by the official `zstd` CLI and by the [zstddec](https://www.npmjs.com/package/zstddec) npm package; `zstd` CLI output is decoded by `zstdify`.
|
|
27
29
|
- **Extensively tested**:
|
|
28
30
|
- Round-trip and property-based tests.
|
|
29
31
|
- Conformance fixtures from known-good archives generated by the official `zstd` tool.
|
|
@@ -41,6 +43,22 @@ const restored = decompress(compressed);
|
|
|
41
43
|
// restored equals data
|
|
42
44
|
```
|
|
43
45
|
|
|
46
|
+
### Tree-shaking friendly imports
|
|
47
|
+
|
|
48
|
+
The default import path remains unchanged:
|
|
49
|
+
|
|
50
|
+
```ts
|
|
51
|
+
import { compress, decompress, generateDictionary } from 'zstdify';
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
For stricter bundling control, you can import feature-specific subpaths:
|
|
55
|
+
|
|
56
|
+
```ts
|
|
57
|
+
import { compress } from 'zstdify/compress';
|
|
58
|
+
import { decompress } from 'zstdify/decompress';
|
|
59
|
+
import { generateDictionary } from 'zstdify/dictionary';
|
|
60
|
+
```
|
|
61
|
+
|
|
44
62
|
## API
|
|
45
63
|
|
|
46
64
|
- `compress(input: Uint8Array, options?: { level?: number; checksum?: boolean; dictionary?: Uint8Array | { bytes: Uint8Array; id?: number }; noDictId?: boolean }): Uint8Array`
|
|
@@ -98,6 +116,8 @@ All of the following run as part of the test suite (`pnpm test` / `pnpm vitest`)
|
|
|
98
116
|
- **Round-trip**: `decompress(compress(x)) === x` for a variety of payloads and levels, plus property-based tests with [fast-check](https://fast-check.dev/).
|
|
99
117
|
- **Conformance fixtures**: Pre-generated `.zst` files from the official zstd CLI (legacy fixtures and a committed decodecorpus-style **corpus** with manifest); we decompress and compare. See [packages/zstdify-tests/fixtures/README.md](packages/zstdify-tests/fixtures/README.md).
|
|
100
118
|
- **Differential (zstd ↔ zstdify)**: We test zstd compress → zstdify decompress and zstdify compress → zstd decompress across payloads and levels.
|
|
119
|
+
- **zstddec**: zstdify compress → [zstddec](https://www.npmjs.com/package/zstddec) decode at all compression levels (0–9) and with content checksum.
|
|
120
|
+
- **Node zstd ratio**: Compare compression ratio of Node's built-in zstd vs zstdify on representative payloads and levels; assert zstdify is within ~10% of Node's compressed size.
|
|
101
121
|
- **Corruption**: Truncation, checksum mismatch, invalid header bits, and related error paths.
|
|
102
122
|
- **Compression regression**: Compressed sizes for fixed payloads are checked against golden values (ratio stability).
|
|
103
123
|
- **Decompress robustness**: Each corpus fixture is decompressed in its own test (one test per file), so the suite tracks decompress behavior per input. See [upstream zstd TESTING.md](https://github.com/facebook/zstd/blob/dev/TESTING.md) for comparison.
|
|
@@ -118,6 +138,11 @@ pnpm make-release:cli
|
|
|
118
138
|
- `packages/cli` - CLI tool (`zstdify-cli` on npm)
|
|
119
139
|
- `packages/cli-tests` - Tests of the CLI tool
|
|
120
140
|
|
|
141
|
+
## Acknowledgements
|
|
142
|
+
|
|
143
|
+
This project is made possible by the original [zstd](https://github.com/facebook/zstd) project by Meta and its contributors.
|
|
144
|
+
The monorepo, project, and CLI structure were bootstrapped from [hdrify](https://github.com/bhouston/hdrify), which made this project much easier to build.
|
|
145
|
+
|
|
121
146
|
# License
|
|
122
147
|
|
|
123
148
|
MIT
|
|
@@ -132,5 +157,5 @@ MIT
|
|
|
132
157
|
[npmtrends-url]: https://www.npmtrends.com/zstdify
|
|
133
158
|
[tests-badge]: https://github.com/bhouston/zstdify/workflows/Tests/badge.svg
|
|
134
159
|
[tests-url]: https://github.com/bhouston/zstdify/actions/workflows/test.yml
|
|
135
|
-
[coverage-badge]: https://codecov.io/gh/bhouston/zstdify/
|
|
136
|
-
[coverage-url]: https://codecov.io/gh/bhouston/zstdify
|
|
160
|
+
[coverage-badge]: https://codecov.io/gh/bhouston/zstdify/graph/badge.svg?token=K9NKF7F3T0
|
|
161
|
+
[coverage-url]: https://codecov.io/gh/bhouston/zstdify
|
package/dist/compress.js
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* Compress input data using zstd.
|
|
3
3
|
* Level 0: raw blocks only (no compression, fast).
|
|
4
4
|
*/
|
|
5
|
-
import {
|
|
5
|
+
import { resolveDictionaryIdForCompression } from './dictionary/compressorDictionary.js';
|
|
6
6
|
import { writeRawBlock, writeRLEBlock } from './encode/blockWriter.js';
|
|
7
7
|
import { buildCompressedBlockPayload, writeCompressedBlock } from './encode/compressedBlock.js';
|
|
8
8
|
import { writeFrameHeader } from './encode/frameWriter.js';
|
|
@@ -16,10 +16,11 @@ export function compress(input, options) {
|
|
|
16
16
|
const dictionary = options?.dictionary;
|
|
17
17
|
const dictionaryBytes = dictionary instanceof Uint8Array ? dictionary : dictionary?.bytes;
|
|
18
18
|
const providedDictionaryId = dictionary instanceof Uint8Array ? null : (dictionary?.id ?? null);
|
|
19
|
-
const
|
|
20
|
-
?
|
|
21
|
-
:
|
|
22
|
-
|
|
19
|
+
const dictionaryId = options?.noDictId
|
|
20
|
+
? null
|
|
21
|
+
: dictionaryBytes && dictionaryBytes.length > 0
|
|
22
|
+
? resolveDictionaryIdForCompression(dictionaryBytes, providedDictionaryId)
|
|
23
|
+
: providedDictionaryId;
|
|
23
24
|
if (dictionaryId !== null && (!Number.isInteger(dictionaryId) || dictionaryId <= 0 || dictionaryId > 0xffff_ffff)) {
|
|
24
25
|
throw new ZstdError('dictionary.id must be a 32-bit positive integer', 'parameter_unsupported');
|
|
25
26
|
}
|
package/dist/compress.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"compress.js","sourceRoot":"","sources":["../src/compress.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"compress.js","sourceRoot":"","sources":["../src/compress.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,iCAAiC,EAAE,MAAM,sCAAsC,CAAC;AACzF,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAC;AACvE,OAAO,EAAE,2BAA2B,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AAChG,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC3D,OAAO,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AACnE,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAE,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AAS/D,MAAM,SAAS,GAAG,GAAG,GAAG,IAAI,CAAC;AAE7B,MAAM,UAAU,QAAQ,CAAC,KAAiB,EAAE,OAAyB,EAAc;IACjF,MAAM,KAAK,GAAG,OAAO,EAAE,KAAK,IAAI,CAAC,CAAC;IAClC,MAAM,WAAW,GAAG,OAAO,EAAE,QAAQ,IAAI,KAAK,CAAC;IAC/C,MAAM,UAAU,GAAG,OAAO,EAAE,UAAU,CAAC;IACvC,MAAM,eAAe,GAAG,UAAU,YAAY,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,UAAU,EAAE,KAAK,CAAC;IAC1F,MAAM,oBAAoB,GAAG,UAAU,YAAY,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,UAAU,EAAE,EAAE,IAAI,IAAI,CAAC,CAAC;IAChG,MAAM,YAAY,GAAG,OAAO,EAAE,QAAQ;QACpC,CAAC,CAAC,IAAI;QACN,CAAC,CAAC,eAAe,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC;YAC7C,CAAC,CAAC,iCAAiC,CAAC,eAAe,EAAE,oBAAoB,CAAC;YAC1E,CAAC,CAAC,oBAAoB,CAAC;IAC3B,IAAI,YAAY,KAAK,IAAI,IAAI,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,YAAY,CAAC,IAAI,YAAY,IAAI,CAAC,IAAI,YAAY,GAAG,WAAW,CAAC,EAAE,CAAC;QAClH,MAAM,IAAI,SAAS,CAAC,iDAAiD,EAAE,uBAAuB,CAAC,CAAC;IAClG,CAAC;IACD,MAAM,MAAM,GAAiB,EAAE,CAAC;IAEhC,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,MAAM,EAAE,WAAW,EAAE,YAAY,CAAC,CAAC,CAAC;IAEvE,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,SAAS,CAAC,CAAC;IAChF,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,OAAO,MAAM,GAAG,KAAK,CAAC,MAAM,IAAI,UAAU,GAAG,UAAU,EAAE,CAAC;QACxD,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC;QACxD,MAAM,IAAI,GAAG,UAAU,KAAK,UAAU,GAAG,CAAC,CAAC;QAC3C,MAAM,KAAK,GAAG,KAAK,CAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC,CAAC;QACpD,IAAI,KAAK,GAAG,CAAC,IAAI,IAAI,GAAG,CAAC,EAAE,CAAC;YAC1B,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;gBACd,MAAM,IAAI,GAAG,oBAAoB,CAAC,KAAK,CAAC,CAAC;gBACzC,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC9B,MAAM,OAAO,GAAG,2BAA2B,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;oBAC3E,IAAI,OAAO,EAAE,CAAC;wBACZ,MAAM,UAAU,GAAG,oBAAoB,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;wBACvD,MAAM,GAAG,GAAG,aAAa,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;wBACrD,IAAI,UAAU,CAAC,MAAM,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC;4BACnC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;4BACxB,MAAM,IAAI,IAAI,CAAC;4BACf,UAAU,EAAE,CAAC;4BACb,SAAS;wBACX,CAAC;oBACH,CAAC;gBACH,CAAC;YACH,CAAC;YACD,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YACjC,IAAI,KAAK,GAAG,IAAI,CAAC;YACjB,KAAK,IAAI,CAAC,GAAG,MAAM,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,GAAG,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC;gBAChD,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,KAAK,EAAE,CAAC;oBAC9B,KAAK,GAAG,KAAK,CAAC;oBACd,MAAM;gBACR,CAAC;YACH,CAAC;YACD,IAAI,KAAK,EAAE,CAAC;gBACV,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;YAChD,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;YACxD,CAAC;QACH,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;QACxD,CAAC;QACD,MAAM,IAAI,IAAI,CAAC;QACf,UAAU,EAAE,CAAC;IACf,CAAC;IAED,IAAI,WAAW,EAAE,CAAC;QAChB,MAAM,QAAQ,GAAG,wBAAwB,CAAC,KAAK,CAAC,CAAC;QACjD,MAAM,CAAC,IAAI,CACT,IAAI,UAAU,CAAC,CAAC,QAAQ,GAAG,IAAI,EAAE,CAAC,QAAQ,KAAK,CAAC,CAAC,GAAG,IAAI,EAAE,CAAC,QAAQ,KAAK,EAAE,CAAC,GAAG,IAAI,EAAE,CAAC,QAAQ,KAAK,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC,CAC/G,CAAC;IACJ,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACvD,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,KAAK,CAAC,CAAC;IACrC,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QACvB,GAAG,IAAI,KAAK,CAAC,MAAM,CAAC;IACtB,CAAC;IACD,OAAO,MAAM,CAAC;AAAA,CACf"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function resolveDictionaryIdForCompression(dictionaryBytes: Uint8Array, providedDictionaryId?: number | null): number | null;
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { readU32LE } from '../bitstream/littleEndian.js';
|
|
2
|
+
import { ZstdError } from '../errors.js';
|
|
3
|
+
const ZSTD_DICTIONARY_MAGIC = 0xec30a437;
|
|
4
|
+
export function resolveDictionaryIdForCompression(dictionaryBytes, providedDictionaryId = null) {
|
|
5
|
+
if (dictionaryBytes.length < 8 || readU32LE(dictionaryBytes, 0) !== ZSTD_DICTIONARY_MAGIC) {
|
|
6
|
+
return providedDictionaryId;
|
|
7
|
+
}
|
|
8
|
+
if (dictionaryBytes.length <= 8) {
|
|
9
|
+
throw new ZstdError('Dictionary too small', 'corruption_detected');
|
|
10
|
+
}
|
|
11
|
+
const parsedDictionaryId = readU32LE(dictionaryBytes, 4);
|
|
12
|
+
if (parsedDictionaryId === 0) {
|
|
13
|
+
throw new ZstdError('Dictionary ID must be non-zero', 'corruption_detected');
|
|
14
|
+
}
|
|
15
|
+
if (providedDictionaryId !== null && providedDictionaryId !== parsedDictionaryId) {
|
|
16
|
+
throw new ZstdError('Provided dictionary ID does not match dictionary content', 'corruption_detected');
|
|
17
|
+
}
|
|
18
|
+
return parsedDictionaryId;
|
|
19
|
+
}
|
|
20
|
+
//# sourceMappingURL=compressorDictionary.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"compressorDictionary.js","sourceRoot":"","sources":["../../src/dictionary/compressorDictionary.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,8BAA8B,CAAC;AACzD,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAEzC,MAAM,qBAAqB,GAAG,UAAU,CAAC;AAEzC,MAAM,UAAU,iCAAiC,CAC/C,eAA2B,EAC3B,oBAAoB,GAAkB,IAAI,EAC3B;IACf,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,IAAI,SAAS,CAAC,eAAe,EAAE,CAAC,CAAC,KAAK,qBAAqB,EAAE,CAAC;QAC1F,OAAO,oBAAoB,CAAC;IAC9B,CAAC;IACD,IAAI,eAAe,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QAChC,MAAM,IAAI,SAAS,CAAC,sBAAsB,EAAE,qBAAqB,CAAC,CAAC;IACrE,CAAC;IAED,MAAM,kBAAkB,GAAG,SAAS,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC;IACzD,IAAI,kBAAkB,KAAK,CAAC,EAAE,CAAC;QAC7B,MAAM,IAAI,SAAS,CAAC,gCAAgC,EAAE,qBAAqB,CAAC,CAAC;IAC/E,CAAC;IACD,IAAI,oBAAoB,KAAK,IAAI,IAAI,oBAAoB,KAAK,kBAAkB,EAAE,CAAC;QACjF,MAAM,IAAI,SAAS,CAAC,0DAA0D,EAAE,qBAAqB,CAAC,CAAC;IACzG,CAAC;IAED,OAAO,kBAAkB,CAAC;AAAA,CAC3B"}
|
package/package.json
CHANGED
|
@@ -1,10 +1,34 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "zstdify",
|
|
3
3
|
"description": "Pure TypeScript zstd compression library",
|
|
4
|
-
"version": "1.1.
|
|
4
|
+
"version": "1.1.2",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|
|
7
7
|
"types": "dist/index.d.ts",
|
|
8
|
+
"sideEffects": false,
|
|
9
|
+
"exports": {
|
|
10
|
+
".": {
|
|
11
|
+
"types": "./dist/index.d.ts",
|
|
12
|
+
"import": "./dist/index.js",
|
|
13
|
+
"default": "./dist/index.js"
|
|
14
|
+
},
|
|
15
|
+
"./compress": {
|
|
16
|
+
"types": "./dist/compress.d.ts",
|
|
17
|
+
"import": "./dist/compress.js",
|
|
18
|
+
"default": "./dist/compress.js"
|
|
19
|
+
},
|
|
20
|
+
"./decompress": {
|
|
21
|
+
"types": "./dist/decompress.d.ts",
|
|
22
|
+
"import": "./dist/decompress.js",
|
|
23
|
+
"default": "./dist/decompress.js"
|
|
24
|
+
},
|
|
25
|
+
"./dictionary": {
|
|
26
|
+
"types": "./dist/dictionary/generateDictionary.d.ts",
|
|
27
|
+
"import": "./dist/dictionary/generateDictionary.js",
|
|
28
|
+
"default": "./dist/dictionary/generateDictionary.js"
|
|
29
|
+
},
|
|
30
|
+
"./package.json": "./package.json"
|
|
31
|
+
},
|
|
8
32
|
"license": "MIT",
|
|
9
33
|
"keywords": [
|
|
10
34
|
"zstd",
|