typenative 0.3.1 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/ci.yml +2 -0
- package/CHANGELOG.md +258 -0
- package/CONTRIBUTION.md +1 -1
- package/README.md +17 -1
- package/bin/cli.js +26 -11
- package/bin/go/bridge.js +74 -14
- package/bin/go/convert.js +59 -1
- package/bin/go/metadata.js +13 -4
- package/bin/go/overrides.js +4 -6
- package/bin/go/stdlib.generated.js +2 -0
- package/bin/index.js +338 -47
- package/bin/main.js +36 -14
- package/bin/node/assert.js +2 -2
- package/bin/node/buffer.js +9 -0
- package/bin/node/child_process.js +51 -3
- package/bin/node/cluster.js +41 -3
- package/bin/node/core.js +1 -1
- package/bin/node/crypto.js +205 -4
- package/bin/node/dgram.js +230 -6
- package/bin/node/dns.js +471 -8
- package/bin/node/errno_tables.js +445 -0
- package/bin/node/events.js +1 -1
- package/bin/node/fs.js +22 -15
- package/bin/node/fs_watch.js +407 -0
- package/bin/node/http.js +128 -30
- package/bin/node/http2.js +37 -7
- package/bin/node/module_v8.js +142 -0
- package/bin/node/net.js +9 -6
- package/bin/node/process.js +242 -0
- package/bin/node/readline.js +272 -1
- package/bin/node/registry.js +3 -1
- package/bin/node/sqlite.js +60 -12
- package/bin/node/stream.js +145 -10
- package/bin/node/test.js +311 -19
- package/bin/node/timers.js +1 -1
- package/bin/node/typed_arrays.js +608 -0
- package/bin/node/url.js +23 -5
- package/bin/node/util.js +21 -1
- package/bin/node/util_extra.js +6 -30
- package/bin/node/worker_threads.js +79 -19
- package/bin/node/zlib.js +131 -16
- package/bin/transpiler.js +4729 -616
- package/bin/typecheck-node.js +1 -0
- package/bin/version.js +1 -1
- package/conformance.md +1030 -877
- package/examples/README.md +54 -0
- package/examples/fibonacci/README.md +89 -0
- package/examples/fibonacci/bench.mjs +87 -0
- package/examples/fibonacci/main.ts +21 -0
- package/examples/hello-world/README.md +62 -0
- package/examples/hello-world/bench.mjs +66 -0
- package/examples/hello-world/main.ts +7 -0
- package/examples/json-server/README.md +127 -0
- package/examples/json-server/bench.mjs +263 -0
- package/examples/json-server/main.ts +67 -0
- package/govendor/github.com/andybalholm/brotli/LICENSE +19 -0
- package/govendor/github.com/andybalholm/brotli/README.md +19 -0
- package/govendor/github.com/andybalholm/brotli/backward_references.go +185 -0
- package/govendor/github.com/andybalholm/brotli/backward_references_hq.go +796 -0
- package/govendor/github.com/andybalholm/brotli/bit_cost.go +436 -0
- package/govendor/github.com/andybalholm/brotli/bit_reader.go +266 -0
- package/govendor/github.com/andybalholm/brotli/bitwriter.go +56 -0
- package/govendor/github.com/andybalholm/brotli/block_splitter.go +144 -0
- package/govendor/github.com/andybalholm/brotli/block_splitter_command.go +434 -0
- package/govendor/github.com/andybalholm/brotli/block_splitter_distance.go +433 -0
- package/govendor/github.com/andybalholm/brotli/block_splitter_literal.go +433 -0
- package/govendor/github.com/andybalholm/brotli/brotli_bit_stream.go +1435 -0
- package/govendor/github.com/andybalholm/brotli/cluster.go +30 -0
- package/govendor/github.com/andybalholm/brotli/cluster_command.go +164 -0
- package/govendor/github.com/andybalholm/brotli/cluster_distance.go +326 -0
- package/govendor/github.com/andybalholm/brotli/cluster_literal.go +326 -0
- package/govendor/github.com/andybalholm/brotli/command.go +254 -0
- package/govendor/github.com/andybalholm/brotli/compress_fragment.go +834 -0
- package/govendor/github.com/andybalholm/brotli/compress_fragment_two_pass.go +775 -0
- package/govendor/github.com/andybalholm/brotli/constants.go +77 -0
- package/govendor/github.com/andybalholm/brotli/context.go +2176 -0
- package/govendor/github.com/andybalholm/brotli/decode.go +2570 -0
- package/govendor/github.com/andybalholm/brotli/dictionary.go +122890 -0
- package/govendor/github.com/andybalholm/brotli/dictionary_hash.go +32779 -0
- package/govendor/github.com/andybalholm/brotli/encode.go +1220 -0
- package/govendor/github.com/andybalholm/brotli/encoder.go +177 -0
- package/govendor/github.com/andybalholm/brotli/encoder_dict.go +22 -0
- package/govendor/github.com/andybalholm/brotli/encoder_fast.go +201 -0
- package/govendor/github.com/andybalholm/brotli/entropy_encode.go +592 -0
- package/govendor/github.com/andybalholm/brotli/entropy_encode_static.go +4399 -0
- package/govendor/github.com/andybalholm/brotli/fast_log.go +290 -0
- package/govendor/github.com/andybalholm/brotli/find_match_length.go +45 -0
- package/govendor/github.com/andybalholm/brotli/go.mod +3 -0
- package/govendor/github.com/andybalholm/brotli/h10.go +287 -0
- package/govendor/github.com/andybalholm/brotli/h5.go +214 -0
- package/govendor/github.com/andybalholm/brotli/h6.go +216 -0
- package/govendor/github.com/andybalholm/brotli/hash.go +342 -0
- package/govendor/github.com/andybalholm/brotli/hash_composite.go +93 -0
- package/govendor/github.com/andybalholm/brotli/hash_forgetful_chain.go +252 -0
- package/govendor/github.com/andybalholm/brotli/hash_longest_match_quickly.go +214 -0
- package/govendor/github.com/andybalholm/brotli/hash_rolling.go +168 -0
- package/govendor/github.com/andybalholm/brotli/histogram.go +226 -0
- package/govendor/github.com/andybalholm/brotli/http.go +201 -0
- package/govendor/github.com/andybalholm/brotli/huffman.go +653 -0
- package/govendor/github.com/andybalholm/brotli/literal_cost.go +182 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/bargain1.go +212 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/bargain2.go +248 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/bargain3.go +278 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/emitter.go +34 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/m0.go +175 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/m4.go +339 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/matchfinder.go +103 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/pathfinder.go +328 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/textencoder.go +53 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/trio.go +327 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/zdfast.go +258 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/zfast.go +206 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/zm.go +302 -0
- package/govendor/github.com/andybalholm/brotli/memory.go +66 -0
- package/govendor/github.com/andybalholm/brotli/metablock.go +574 -0
- package/govendor/github.com/andybalholm/brotli/metablock_command.go +165 -0
- package/govendor/github.com/andybalholm/brotli/metablock_distance.go +165 -0
- package/govendor/github.com/andybalholm/brotli/metablock_literal.go +165 -0
- package/govendor/github.com/andybalholm/brotli/params.go +37 -0
- package/govendor/github.com/andybalholm/brotli/platform.go +103 -0
- package/govendor/github.com/andybalholm/brotli/prefix.go +30 -0
- package/govendor/github.com/andybalholm/brotli/prefix_dec.go +723 -0
- package/govendor/github.com/andybalholm/brotli/quality.go +196 -0
- package/govendor/github.com/andybalholm/brotli/reader.go +127 -0
- package/govendor/github.com/andybalholm/brotli/ringbuffer.go +134 -0
- package/govendor/github.com/andybalholm/brotli/state.go +313 -0
- package/govendor/github.com/andybalholm/brotli/static_dict.go +662 -0
- package/govendor/github.com/andybalholm/brotli/static_dict_lut.go +75094 -0
- package/govendor/github.com/andybalholm/brotli/symbol_list.go +22 -0
- package/govendor/github.com/andybalholm/brotli/transform.go +641 -0
- package/govendor/github.com/andybalholm/brotli/utf8_util.go +70 -0
- package/govendor/github.com/andybalholm/brotli/util.go +7 -0
- package/govendor/github.com/andybalholm/brotli/write_bits.go +52 -0
- package/govendor/github.com/andybalholm/brotli/writer.go +162 -0
- package/govendor/github.com/fsnotify/fsnotify/.mailmap +2 -0
- package/govendor/github.com/fsnotify/fsnotify/CHANGELOG.md +651 -0
- package/govendor/github.com/fsnotify/fsnotify/CONTRIBUTING.md +147 -0
- package/govendor/github.com/fsnotify/fsnotify/LICENSE +25 -0
- package/govendor/github.com/fsnotify/fsnotify/README.md +214 -0
- package/govendor/github.com/fsnotify/fsnotify/backend_fen.go +469 -0
- package/govendor/github.com/fsnotify/fsnotify/backend_inotify.go +597 -0
- package/govendor/github.com/fsnotify/fsnotify/backend_kqueue.go +735 -0
- package/govendor/github.com/fsnotify/fsnotify/backend_other.go +22 -0
- package/govendor/github.com/fsnotify/fsnotify/backend_windows.go +702 -0
- package/govendor/github.com/fsnotify/fsnotify/fsnotify.go +526 -0
- package/govendor/github.com/fsnotify/fsnotify/go.mod +3 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/darwin.go +20 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_darwin.go +15 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_dragonfly.go +15 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_freebsd.go +20 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_kqueue.go +32 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_linux.go +56 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_netbsd.go +15 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_openbsd.go +16 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_solaris.go +45 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_windows.go +40 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/freebsd.go +20 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/internal.go +2 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/unix.go +20 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/unix2.go +25 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/windows.go +40 -0
- package/govendor/github.com/fsnotify/fsnotify/shared.go +64 -0
- package/govendor/github.com/fsnotify/fsnotify/staticcheck.conf +3 -0
- package/govendor/github.com/fsnotify/fsnotify/system_bsd.go +7 -0
- package/govendor/github.com/fsnotify/fsnotify/system_darwin.go +8 -0
- package/govendor/manifest.txt +3 -1
- package/package.json +1 -1
- package/types/typenative-go.d.ts +23 -0
- package/types/typenative-npm.d.ts +241 -5
- package/types/typenative.d.ts +302 -12
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
package brotli
|
|
2
|
+
|
|
3
|
+
import "github.com/andybalholm/brotli/matchfinder"
|
|
4
|
+
|
|
5
|
+
// An Encoder implements the matchfinder.Encoder interface, writing in Brotli format.
|
|
6
|
+
type Encoder struct {
|
|
7
|
+
wroteHeader bool
|
|
8
|
+
bw bitWriter
|
|
9
|
+
distCache []distanceCode
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
func (e *Encoder) Reset() {
|
|
13
|
+
e.wroteHeader = false
|
|
14
|
+
e.bw = bitWriter{}
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
func (e *Encoder) Encode(dst []byte, src []byte, matches []matchfinder.Match, lastBlock bool) []byte {
|
|
18
|
+
e.bw.dst = dst
|
|
19
|
+
if !e.wroteHeader {
|
|
20
|
+
e.bw.writeBits(4, 15)
|
|
21
|
+
e.wroteHeader = true
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
if len(src) == 0 {
|
|
25
|
+
if lastBlock {
|
|
26
|
+
e.bw.writeBits(2, 3) // islast + isempty
|
|
27
|
+
e.bw.jumpToByteBoundary()
|
|
28
|
+
return e.bw.dst
|
|
29
|
+
}
|
|
30
|
+
return dst
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
var literalHisto [256]uint32
|
|
34
|
+
var commandHisto [704]uint32
|
|
35
|
+
var distanceHisto [64]uint32
|
|
36
|
+
literalCount := 0
|
|
37
|
+
commandCount := 0
|
|
38
|
+
distanceCount := 0
|
|
39
|
+
|
|
40
|
+
if len(e.distCache) < len(matches) {
|
|
41
|
+
e.distCache = make([]distanceCode, len(matches))
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
// first pass: build the histograms
|
|
45
|
+
pos := 0
|
|
46
|
+
|
|
47
|
+
// d is the ring buffer of the last 4 distances.
|
|
48
|
+
d := [4]int{-10, -10, -10, -10}
|
|
49
|
+
for i, m := range matches {
|
|
50
|
+
if m.Unmatched > 0 {
|
|
51
|
+
for _, c := range src[pos : pos+m.Unmatched] {
|
|
52
|
+
literalHisto[c]++
|
|
53
|
+
}
|
|
54
|
+
literalCount += m.Unmatched
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
insertCode := getInsertLengthCode(uint(m.Unmatched))
|
|
58
|
+
copyCode := getCopyLengthCode(uint(m.Length))
|
|
59
|
+
if m.Length == 0 {
|
|
60
|
+
// If the stream ends with unmatched bytes, we need a dummy copy length.
|
|
61
|
+
copyCode = 2
|
|
62
|
+
}
|
|
63
|
+
command := combineLengthCodes(insertCode, copyCode, i > 0 && m.Distance == matches[i-1].Distance)
|
|
64
|
+
commandHisto[command]++
|
|
65
|
+
commandCount++
|
|
66
|
+
|
|
67
|
+
if command >= 128 && m.Length != 0 {
|
|
68
|
+
var distCode distanceCode
|
|
69
|
+
switch m.Distance {
|
|
70
|
+
case d[3]:
|
|
71
|
+
distCode.code = 0
|
|
72
|
+
case d[2]:
|
|
73
|
+
distCode.code = 1
|
|
74
|
+
case d[1]:
|
|
75
|
+
distCode.code = 2
|
|
76
|
+
case d[0]:
|
|
77
|
+
distCode.code = 3
|
|
78
|
+
case d[3] - 1:
|
|
79
|
+
distCode.code = 4
|
|
80
|
+
case d[3] + 1:
|
|
81
|
+
distCode.code = 5
|
|
82
|
+
case d[3] - 2:
|
|
83
|
+
distCode.code = 6
|
|
84
|
+
case d[3] + 2:
|
|
85
|
+
distCode.code = 7
|
|
86
|
+
case d[3] - 3:
|
|
87
|
+
distCode.code = 8
|
|
88
|
+
case d[3] + 3:
|
|
89
|
+
distCode.code = 9
|
|
90
|
+
|
|
91
|
+
// In my testing, codes 10–15 actually reduced the compression ratio.
|
|
92
|
+
|
|
93
|
+
default:
|
|
94
|
+
distCode = getDistanceCode(m.Distance)
|
|
95
|
+
}
|
|
96
|
+
e.distCache[i] = distCode
|
|
97
|
+
distanceHisto[distCode.code]++
|
|
98
|
+
distanceCount++
|
|
99
|
+
if distCode.code != 0 {
|
|
100
|
+
d[0], d[1], d[2], d[3] = d[1], d[2], d[3], m.Distance
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
pos += m.Unmatched + m.Length
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
storeMetaBlockHeaderBW(uint(len(src)), false, &e.bw)
|
|
108
|
+
e.bw.writeBits(13, 0)
|
|
109
|
+
|
|
110
|
+
var literalDepths [256]byte
|
|
111
|
+
var literalBits [256]uint16
|
|
112
|
+
buildAndStoreHuffmanTreeFastBW(literalHisto[:], uint(literalCount), 8, literalDepths[:], literalBits[:], &e.bw)
|
|
113
|
+
|
|
114
|
+
var commandDepths [704]byte
|
|
115
|
+
var commandBits [704]uint16
|
|
116
|
+
buildAndStoreHuffmanTreeFastBW(commandHisto[:], uint(commandCount), 10, commandDepths[:], commandBits[:], &e.bw)
|
|
117
|
+
|
|
118
|
+
var distanceDepths [64]byte
|
|
119
|
+
var distanceBits [64]uint16
|
|
120
|
+
buildAndStoreHuffmanTreeFastBW(distanceHisto[:], uint(distanceCount), 6, distanceDepths[:], distanceBits[:], &e.bw)
|
|
121
|
+
|
|
122
|
+
pos = 0
|
|
123
|
+
for i, m := range matches {
|
|
124
|
+
insertCode := getInsertLengthCode(uint(m.Unmatched))
|
|
125
|
+
copyCode := getCopyLengthCode(uint(m.Length))
|
|
126
|
+
if m.Length == 0 {
|
|
127
|
+
// If the stream ends with unmatched bytes, we need a dummy copy length.
|
|
128
|
+
copyCode = 2
|
|
129
|
+
}
|
|
130
|
+
command := combineLengthCodes(insertCode, copyCode, i > 0 && m.Distance == matches[i-1].Distance)
|
|
131
|
+
e.bw.writeBits(uint(commandDepths[command]), uint64(commandBits[command]))
|
|
132
|
+
if kInsExtra[insertCode] > 0 {
|
|
133
|
+
e.bw.writeBits(uint(kInsExtra[insertCode]), uint64(m.Unmatched)-uint64(kInsBase[insertCode]))
|
|
134
|
+
}
|
|
135
|
+
if kCopyExtra[copyCode] > 0 {
|
|
136
|
+
e.bw.writeBits(uint(kCopyExtra[copyCode]), uint64(m.Length)-uint64(kCopyBase[copyCode]))
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
if m.Unmatched > 0 {
|
|
140
|
+
for _, c := range src[pos : pos+m.Unmatched] {
|
|
141
|
+
e.bw.writeBits(uint(literalDepths[c]), uint64(literalBits[c]))
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
if command >= 128 && m.Length != 0 {
|
|
146
|
+
distCode := e.distCache[i]
|
|
147
|
+
e.bw.writeBits(uint(distanceDepths[distCode.code]), uint64(distanceBits[distCode.code]))
|
|
148
|
+
if distCode.nExtra > 0 {
|
|
149
|
+
e.bw.writeBits(distCode.nExtra, distCode.extraBits)
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
pos += m.Unmatched + m.Length
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
if lastBlock {
|
|
157
|
+
e.bw.writeBits(2, 3) // islast + isempty
|
|
158
|
+
e.bw.jumpToByteBoundary()
|
|
159
|
+
}
|
|
160
|
+
return e.bw.dst
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
type distanceCode struct {
|
|
164
|
+
code int
|
|
165
|
+
nExtra uint
|
|
166
|
+
extraBits uint64
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
func getDistanceCode(distance int) distanceCode {
|
|
170
|
+
d := distance + 3
|
|
171
|
+
nbits := log2FloorNonZero(uint(d)) - 1
|
|
172
|
+
prefix := (d >> nbits) & 1
|
|
173
|
+
offset := (2 + prefix) << nbits
|
|
174
|
+
distcode := int(2*(nbits-1)) + prefix + 16
|
|
175
|
+
extra := d - offset
|
|
176
|
+
return distanceCode{distcode, uint(nbits), uint64(extra)}
|
|
177
|
+
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
package brotli
|
|
2
|
+
|
|
3
|
+
/* Dictionary data (words and transforms) for 1 possible context */
|
|
4
|
+
type encoderDictionary struct {
|
|
5
|
+
words *dictionary
|
|
6
|
+
cutoffTransformsCount uint32
|
|
7
|
+
cutoffTransforms uint64
|
|
8
|
+
hash_table []uint16
|
|
9
|
+
buckets []uint16
|
|
10
|
+
dict_words []dictWord
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
func initEncoderDictionary(dict *encoderDictionary) {
|
|
14
|
+
dict.words = getDictionary()
|
|
15
|
+
|
|
16
|
+
dict.hash_table = kStaticDictionaryHash[:]
|
|
17
|
+
dict.buckets = kStaticDictionaryBuckets[:]
|
|
18
|
+
dict.dict_words = kStaticDictionaryWords[:]
|
|
19
|
+
|
|
20
|
+
dict.cutoffTransformsCount = kCutoffTransformsCount
|
|
21
|
+
dict.cutoffTransforms = kCutoffTransforms
|
|
22
|
+
}
|
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
package brotli
|
|
2
|
+
|
|
3
|
+
import (
|
|
4
|
+
"math"
|
|
5
|
+
|
|
6
|
+
"github.com/andybalholm/brotli/matchfinder"
|
|
7
|
+
)
|
|
8
|
+
|
|
9
|
+
func gaussianProbability(x, mean, stdDev float64) float64 {
|
|
10
|
+
return math.Exp(-(x-mean)*(x-mean)/(2*stdDev*stdDev)) / math.Sqrt(2*math.Pi*stdDev*stdDev)
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
// A FastEncoder implements the matchfinder.Encoder interface, writing in Brotli
|
|
14
|
+
// format. It uses a simplified encoding (like level 0 in the reference
|
|
15
|
+
// implementation) to save time. It does not support matches shorter than four
|
|
16
|
+
// bytes.
|
|
17
|
+
type FastEncoder struct {
|
|
18
|
+
wroteHeader bool
|
|
19
|
+
bw bitWriter
|
|
20
|
+
commandHisto [704]uint32
|
|
21
|
+
distanceHisto [64]uint32
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
func (e *FastEncoder) Reset() {
|
|
25
|
+
e.wroteHeader = false
|
|
26
|
+
e.bw = bitWriter{}
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
func (e *FastEncoder) Encode(dst []byte, src []byte, matches []matchfinder.Match, lastBlock bool) []byte {
|
|
30
|
+
e.bw.dst = dst
|
|
31
|
+
if !e.wroteHeader {
|
|
32
|
+
e.bw.writeBits(4, 15)
|
|
33
|
+
e.wroteHeader = true
|
|
34
|
+
|
|
35
|
+
// Fill the histograms with default statistics.
|
|
36
|
+
|
|
37
|
+
// For the command codes we're using for insert lengths (insert + 2-byte copy),
|
|
38
|
+
// fill the histogram with a Zipf-squared distribution.
|
|
39
|
+
for i := range 24 {
|
|
40
|
+
e.commandHisto[combineLengthCodes(uint16(i), 0, false)] = uint32(2000 / (i + 1) / (i + 1))
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
// For the command codes we're using for copy lengths (0 insert + copy
|
|
44
|
+
// (length - 2), with repeat distance),
|
|
45
|
+
// fill the histogram with Zipf distribution starting at code 1 (match length 5),
|
|
46
|
+
// but a smaller frequency for code 0.
|
|
47
|
+
e.commandHisto[combineLengthCodes(0, 0, true)] = 50
|
|
48
|
+
for i := 1; i < 24; i++ {
|
|
49
|
+
e.commandHisto[combineLengthCodes(0, uint16(i), i < 16)] = uint32(800 / i)
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
// Fill in the combined codes for short insert and copy lengths.
|
|
53
|
+
for insertCode := range 6 {
|
|
54
|
+
copyCode := 2
|
|
55
|
+
e.commandHisto[128+insertCode<<3+copyCode] = uint32(100 / (insertCode + 1) / (insertCode + 1) / copyCode)
|
|
56
|
+
for copyCode := 3; copyCode < 8; copyCode++ {
|
|
57
|
+
e.commandHisto[128+insertCode<<3+copyCode] = uint32(343 / (insertCode + 1) / (insertCode + 1) / copyCode)
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
// Fill e.distanceHisto with a normal distribution.
|
|
62
|
+
e.distanceHisto[0] = 100
|
|
63
|
+
for i := 16; i < 64; i++ {
|
|
64
|
+
e.distanceHisto[i] = max(uint32(gaussianProbability(float64(i), 32, 8)*10000), 1)
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
if len(src) == 0 {
|
|
69
|
+
if lastBlock {
|
|
70
|
+
e.bw.writeBits(2, 3) // islast + isempty
|
|
71
|
+
e.bw.jumpToByteBoundary()
|
|
72
|
+
return e.bw.dst
|
|
73
|
+
}
|
|
74
|
+
return dst
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
var literalHisto [256]uint32
|
|
78
|
+
for _, c := range src {
|
|
79
|
+
literalHisto[c]++
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
storeMetaBlockHeaderBW(uint(len(src)), false, &e.bw)
|
|
83
|
+
e.bw.writeBits(13, 0)
|
|
84
|
+
|
|
85
|
+
var literalDepths [256]byte
|
|
86
|
+
var literalBits [256]uint16
|
|
87
|
+
buildAndStoreHuffmanTreeFastBW(literalHisto[:], uint(len(src)), 8, literalDepths[:], literalBits[:], &e.bw)
|
|
88
|
+
|
|
89
|
+
var commandDepths [704]byte
|
|
90
|
+
var commandBits [704]uint16
|
|
91
|
+
commandCount := 0
|
|
92
|
+
for _, n := range e.commandHisto {
|
|
93
|
+
commandCount += int(n)
|
|
94
|
+
}
|
|
95
|
+
buildAndStoreHuffmanTreeFastBW(e.commandHisto[:], uint(commandCount), 10, commandDepths[:], commandBits[:], &e.bw)
|
|
96
|
+
|
|
97
|
+
var distanceDepths [64]byte
|
|
98
|
+
var distanceBits [64]uint16
|
|
99
|
+
distanceCount := 0
|
|
100
|
+
for _, n := range e.distanceHisto {
|
|
101
|
+
distanceCount += int(n)
|
|
102
|
+
}
|
|
103
|
+
buildAndStoreHuffmanTreeFastBW(e.distanceHisto[:], uint(distanceCount), 6, distanceDepths[:], distanceBits[:], &e.bw)
|
|
104
|
+
|
|
105
|
+
// Reset the statistics, starting with a count of 1 for each symbol we might use.
|
|
106
|
+
for i := range 24 {
|
|
107
|
+
e.commandHisto[combineLengthCodes(uint16(i), 0, false)] = 1
|
|
108
|
+
}
|
|
109
|
+
for i := range 24 {
|
|
110
|
+
e.commandHisto[combineLengthCodes(0, uint16(i), i < 16)] = 1
|
|
111
|
+
}
|
|
112
|
+
for insertCode := range 6 {
|
|
113
|
+
for copyCode := 2; copyCode < 8; copyCode++ {
|
|
114
|
+
e.commandHisto[128+insertCode<<3+copyCode] = 1
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
e.distanceHisto[0] = 1
|
|
118
|
+
for i := 16; i < 64; i++ {
|
|
119
|
+
e.distanceHisto[i] = 1
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
pos := 0
|
|
123
|
+
for i, m := range matches {
|
|
124
|
+
if m.Length < 4 && m.Length != 0 {
|
|
125
|
+
panic("match too short (minimum 4 bytes)")
|
|
126
|
+
}
|
|
127
|
+
lengthFinished := false
|
|
128
|
+
// Write a command with the appropriate insert length, and a copy length of 2.
|
|
129
|
+
if m.Unmatched < 6 {
|
|
130
|
+
var command int
|
|
131
|
+
if m.Length < 10 && m.Length != 0 {
|
|
132
|
+
// We can use a combined insert/copy code with no extra bits.
|
|
133
|
+
command = m.Unmatched<<3 + m.Length - 2 + 128
|
|
134
|
+
lengthFinished = true
|
|
135
|
+
} else {
|
|
136
|
+
command = m.Unmatched<<3 + 128
|
|
137
|
+
}
|
|
138
|
+
e.bw.writeBits(uint(commandDepths[command]), uint64(commandBits[command]))
|
|
139
|
+
e.commandHisto[command]++
|
|
140
|
+
} else {
|
|
141
|
+
insertCode := getInsertLengthCode(uint(m.Unmatched))
|
|
142
|
+
command := combineLengthCodes(insertCode, 0, false)
|
|
143
|
+
e.bw.writeBits(uint(commandDepths[command]), uint64(commandBits[command]))
|
|
144
|
+
e.bw.writeBits(uint(kInsExtra[insertCode]), uint64(m.Unmatched)-uint64(kInsBase[insertCode]))
|
|
145
|
+
e.commandHisto[command]++
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
// Write the literals, if any.
|
|
149
|
+
if m.Unmatched > 0 {
|
|
150
|
+
for _, c := range src[pos : pos+m.Unmatched] {
|
|
151
|
+
e.bw.writeBits(uint(literalDepths[c]), uint64(literalBits[c]))
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
if m.Length != 0 {
|
|
156
|
+
// Write the distance code.
|
|
157
|
+
var distCode distanceCode
|
|
158
|
+
if i == 0 || m.Distance != matches[i-1].Distance {
|
|
159
|
+
distCode = getDistanceCode(m.Distance)
|
|
160
|
+
}
|
|
161
|
+
e.bw.writeBits(uint(distanceDepths[distCode.code]), uint64(distanceBits[distCode.code]))
|
|
162
|
+
if distCode.nExtra > 0 {
|
|
163
|
+
e.bw.writeBits(distCode.nExtra, distCode.extraBits)
|
|
164
|
+
}
|
|
165
|
+
e.distanceHisto[distCode.code]++
|
|
166
|
+
|
|
167
|
+
// Write a command for the remainder of the match (after the first two bytes
|
|
168
|
+
// from before), using the previous distance.
|
|
169
|
+
switch {
|
|
170
|
+
case lengthFinished:
|
|
171
|
+
// We don't need to finish the length.
|
|
172
|
+
case m.Length < 12:
|
|
173
|
+
command := m.Length - 4
|
|
174
|
+
e.bw.writeBits(uint(commandDepths[command]), uint64(commandBits[command]))
|
|
175
|
+
e.commandHisto[command]++
|
|
176
|
+
case m.Length < 72:
|
|
177
|
+
copyCode := getCopyLengthCode(uint(m.Length - 2))
|
|
178
|
+
command := combineLengthCodes(0, copyCode, true)
|
|
179
|
+
e.bw.writeBits(uint(commandDepths[command]), uint64(commandBits[command]))
|
|
180
|
+
e.bw.writeBits(uint(kCopyExtra[copyCode]), uint64(m.Length-2)-uint64(kCopyBase[copyCode]))
|
|
181
|
+
e.commandHisto[command]++
|
|
182
|
+
default:
|
|
183
|
+
copyCode := getCopyLengthCode(uint(m.Length - 2))
|
|
184
|
+
command := combineLengthCodes(0, copyCode, false)
|
|
185
|
+
e.bw.writeBits(uint(commandDepths[command]), uint64(commandBits[command]))
|
|
186
|
+
e.bw.writeBits(uint(kCopyExtra[copyCode]), uint64(m.Length-2)-uint64(kCopyBase[copyCode]))
|
|
187
|
+
e.bw.writeBits(uint(distanceDepths[0]), uint64(distanceBits[0]))
|
|
188
|
+
e.commandHisto[command]++
|
|
189
|
+
e.distanceHisto[0]++
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
pos += m.Unmatched + m.Length
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
if lastBlock {
|
|
197
|
+
e.bw.writeBits(2, 3) // islast + isempty
|
|
198
|
+
e.bw.jumpToByteBoundary()
|
|
199
|
+
}
|
|
200
|
+
return e.bw.dst
|
|
201
|
+
}
|