typenative 0.3.1 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/ci.yml +2 -0
- package/CHANGELOG.md +258 -0
- package/CONTRIBUTION.md +1 -1
- package/README.md +17 -1
- package/bin/cli.js +26 -11
- package/bin/go/bridge.js +74 -14
- package/bin/go/convert.js +59 -1
- package/bin/go/metadata.js +13 -4
- package/bin/go/overrides.js +4 -6
- package/bin/go/stdlib.generated.js +2 -0
- package/bin/index.js +338 -47
- package/bin/main.js +36 -14
- package/bin/node/assert.js +2 -2
- package/bin/node/buffer.js +9 -0
- package/bin/node/child_process.js +51 -3
- package/bin/node/cluster.js +41 -3
- package/bin/node/core.js +1 -1
- package/bin/node/crypto.js +205 -4
- package/bin/node/dgram.js +230 -6
- package/bin/node/dns.js +471 -8
- package/bin/node/errno_tables.js +445 -0
- package/bin/node/events.js +1 -1
- package/bin/node/fs.js +22 -15
- package/bin/node/fs_watch.js +407 -0
- package/bin/node/http.js +128 -30
- package/bin/node/http2.js +37 -7
- package/bin/node/module_v8.js +142 -0
- package/bin/node/net.js +9 -6
- package/bin/node/process.js +242 -0
- package/bin/node/readline.js +272 -1
- package/bin/node/registry.js +3 -1
- package/bin/node/sqlite.js +60 -12
- package/bin/node/stream.js +145 -10
- package/bin/node/test.js +311 -19
- package/bin/node/timers.js +1 -1
- package/bin/node/typed_arrays.js +608 -0
- package/bin/node/url.js +23 -5
- package/bin/node/util.js +21 -1
- package/bin/node/util_extra.js +6 -30
- package/bin/node/worker_threads.js +79 -19
- package/bin/node/zlib.js +131 -16
- package/bin/transpiler.js +4729 -616
- package/bin/typecheck-node.js +1 -0
- package/bin/version.js +1 -1
- package/conformance.md +1030 -877
- package/examples/README.md +54 -0
- package/examples/fibonacci/README.md +89 -0
- package/examples/fibonacci/bench.mjs +87 -0
- package/examples/fibonacci/main.ts +21 -0
- package/examples/hello-world/README.md +62 -0
- package/examples/hello-world/bench.mjs +66 -0
- package/examples/hello-world/main.ts +7 -0
- package/examples/json-server/README.md +127 -0
- package/examples/json-server/bench.mjs +263 -0
- package/examples/json-server/main.ts +67 -0
- package/govendor/github.com/andybalholm/brotli/LICENSE +19 -0
- package/govendor/github.com/andybalholm/brotli/README.md +19 -0
- package/govendor/github.com/andybalholm/brotli/backward_references.go +185 -0
- package/govendor/github.com/andybalholm/brotli/backward_references_hq.go +796 -0
- package/govendor/github.com/andybalholm/brotli/bit_cost.go +436 -0
- package/govendor/github.com/andybalholm/brotli/bit_reader.go +266 -0
- package/govendor/github.com/andybalholm/brotli/bitwriter.go +56 -0
- package/govendor/github.com/andybalholm/brotli/block_splitter.go +144 -0
- package/govendor/github.com/andybalholm/brotli/block_splitter_command.go +434 -0
- package/govendor/github.com/andybalholm/brotli/block_splitter_distance.go +433 -0
- package/govendor/github.com/andybalholm/brotli/block_splitter_literal.go +433 -0
- package/govendor/github.com/andybalholm/brotli/brotli_bit_stream.go +1435 -0
- package/govendor/github.com/andybalholm/brotli/cluster.go +30 -0
- package/govendor/github.com/andybalholm/brotli/cluster_command.go +164 -0
- package/govendor/github.com/andybalholm/brotli/cluster_distance.go +326 -0
- package/govendor/github.com/andybalholm/brotli/cluster_literal.go +326 -0
- package/govendor/github.com/andybalholm/brotli/command.go +254 -0
- package/govendor/github.com/andybalholm/brotli/compress_fragment.go +834 -0
- package/govendor/github.com/andybalholm/brotli/compress_fragment_two_pass.go +775 -0
- package/govendor/github.com/andybalholm/brotli/constants.go +77 -0
- package/govendor/github.com/andybalholm/brotli/context.go +2176 -0
- package/govendor/github.com/andybalholm/brotli/decode.go +2570 -0
- package/govendor/github.com/andybalholm/brotli/dictionary.go +122890 -0
- package/govendor/github.com/andybalholm/brotli/dictionary_hash.go +32779 -0
- package/govendor/github.com/andybalholm/brotli/encode.go +1220 -0
- package/govendor/github.com/andybalholm/brotli/encoder.go +177 -0
- package/govendor/github.com/andybalholm/brotli/encoder_dict.go +22 -0
- package/govendor/github.com/andybalholm/brotli/encoder_fast.go +201 -0
- package/govendor/github.com/andybalholm/brotli/entropy_encode.go +592 -0
- package/govendor/github.com/andybalholm/brotli/entropy_encode_static.go +4399 -0
- package/govendor/github.com/andybalholm/brotli/fast_log.go +290 -0
- package/govendor/github.com/andybalholm/brotli/find_match_length.go +45 -0
- package/govendor/github.com/andybalholm/brotli/go.mod +3 -0
- package/govendor/github.com/andybalholm/brotli/h10.go +287 -0
- package/govendor/github.com/andybalholm/brotli/h5.go +214 -0
- package/govendor/github.com/andybalholm/brotli/h6.go +216 -0
- package/govendor/github.com/andybalholm/brotli/hash.go +342 -0
- package/govendor/github.com/andybalholm/brotli/hash_composite.go +93 -0
- package/govendor/github.com/andybalholm/brotli/hash_forgetful_chain.go +252 -0
- package/govendor/github.com/andybalholm/brotli/hash_longest_match_quickly.go +214 -0
- package/govendor/github.com/andybalholm/brotli/hash_rolling.go +168 -0
- package/govendor/github.com/andybalholm/brotli/histogram.go +226 -0
- package/govendor/github.com/andybalholm/brotli/http.go +201 -0
- package/govendor/github.com/andybalholm/brotli/huffman.go +653 -0
- package/govendor/github.com/andybalholm/brotli/literal_cost.go +182 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/bargain1.go +212 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/bargain2.go +248 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/bargain3.go +278 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/emitter.go +34 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/m0.go +175 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/m4.go +339 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/matchfinder.go +103 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/pathfinder.go +328 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/textencoder.go +53 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/trio.go +327 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/zdfast.go +258 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/zfast.go +206 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/zm.go +302 -0
- package/govendor/github.com/andybalholm/brotli/memory.go +66 -0
- package/govendor/github.com/andybalholm/brotli/metablock.go +574 -0
- package/govendor/github.com/andybalholm/brotli/metablock_command.go +165 -0
- package/govendor/github.com/andybalholm/brotli/metablock_distance.go +165 -0
- package/govendor/github.com/andybalholm/brotli/metablock_literal.go +165 -0
- package/govendor/github.com/andybalholm/brotli/params.go +37 -0
- package/govendor/github.com/andybalholm/brotli/platform.go +103 -0
- package/govendor/github.com/andybalholm/brotli/prefix.go +30 -0
- package/govendor/github.com/andybalholm/brotli/prefix_dec.go +723 -0
- package/govendor/github.com/andybalholm/brotli/quality.go +196 -0
- package/govendor/github.com/andybalholm/brotli/reader.go +127 -0
- package/govendor/github.com/andybalholm/brotli/ringbuffer.go +134 -0
- package/govendor/github.com/andybalholm/brotli/state.go +313 -0
- package/govendor/github.com/andybalholm/brotli/static_dict.go +662 -0
- package/govendor/github.com/andybalholm/brotli/static_dict_lut.go +75094 -0
- package/govendor/github.com/andybalholm/brotli/symbol_list.go +22 -0
- package/govendor/github.com/andybalholm/brotli/transform.go +641 -0
- package/govendor/github.com/andybalholm/brotli/utf8_util.go +70 -0
- package/govendor/github.com/andybalholm/brotli/util.go +7 -0
- package/govendor/github.com/andybalholm/brotli/write_bits.go +52 -0
- package/govendor/github.com/andybalholm/brotli/writer.go +162 -0
- package/govendor/github.com/fsnotify/fsnotify/.mailmap +2 -0
- package/govendor/github.com/fsnotify/fsnotify/CHANGELOG.md +651 -0
- package/govendor/github.com/fsnotify/fsnotify/CONTRIBUTING.md +147 -0
- package/govendor/github.com/fsnotify/fsnotify/LICENSE +25 -0
- package/govendor/github.com/fsnotify/fsnotify/README.md +214 -0
- package/govendor/github.com/fsnotify/fsnotify/backend_fen.go +469 -0
- package/govendor/github.com/fsnotify/fsnotify/backend_inotify.go +597 -0
- package/govendor/github.com/fsnotify/fsnotify/backend_kqueue.go +735 -0
- package/govendor/github.com/fsnotify/fsnotify/backend_other.go +22 -0
- package/govendor/github.com/fsnotify/fsnotify/backend_windows.go +702 -0
- package/govendor/github.com/fsnotify/fsnotify/fsnotify.go +526 -0
- package/govendor/github.com/fsnotify/fsnotify/go.mod +3 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/darwin.go +20 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_darwin.go +15 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_dragonfly.go +15 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_freebsd.go +20 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_kqueue.go +32 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_linux.go +56 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_netbsd.go +15 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_openbsd.go +16 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_solaris.go +45 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_windows.go +40 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/freebsd.go +20 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/internal.go +2 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/unix.go +20 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/unix2.go +25 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/windows.go +40 -0
- package/govendor/github.com/fsnotify/fsnotify/shared.go +64 -0
- package/govendor/github.com/fsnotify/fsnotify/staticcheck.conf +3 -0
- package/govendor/github.com/fsnotify/fsnotify/system_bsd.go +7 -0
- package/govendor/github.com/fsnotify/fsnotify/system_darwin.go +8 -0
- package/govendor/manifest.txt +3 -1
- package/package.json +1 -1
- package/types/typenative-go.d.ts +23 -0
- package/types/typenative-npm.d.ts +241 -5
- package/types/typenative.d.ts +302 -12
|
@@ -0,0 +1,436 @@
|
|
|
1
|
+
package brotli
|
|
2
|
+
|
|
3
|
+
/* Copyright 2013 Google Inc. All Rights Reserved.
|
|
4
|
+
|
|
5
|
+
Distributed under MIT license.
|
|
6
|
+
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
/* Functions to estimate the bit cost of Huffman trees. */
|
|
10
|
+
func shannonEntropy(population []uint32, size uint, total *uint) float64 {
|
|
11
|
+
var sum uint = 0
|
|
12
|
+
var retval float64 = 0
|
|
13
|
+
var population_end []uint32 = population[size:]
|
|
14
|
+
var p uint
|
|
15
|
+
for -cap(population) < -cap(population_end) {
|
|
16
|
+
p = uint(population[0])
|
|
17
|
+
population = population[1:]
|
|
18
|
+
sum += p
|
|
19
|
+
retval -= float64(p) * fastLog2(p)
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
if sum != 0 {
|
|
23
|
+
retval += float64(sum) * fastLog2(sum)
|
|
24
|
+
}
|
|
25
|
+
*total = sum
|
|
26
|
+
return retval
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
func bitsEntropy(population []uint32, size uint) float64 {
|
|
30
|
+
var sum uint
|
|
31
|
+
var retval float64 = shannonEntropy(population, size, &sum)
|
|
32
|
+
if retval < float64(sum) {
|
|
33
|
+
/* At least one bit per literal is needed. */
|
|
34
|
+
retval = float64(sum)
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
return retval
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
const kOneSymbolHistogramCost float64 = 12
|
|
41
|
+
const kTwoSymbolHistogramCost float64 = 20
|
|
42
|
+
const kThreeSymbolHistogramCost float64 = 28
|
|
43
|
+
const kFourSymbolHistogramCost float64 = 37
|
|
44
|
+
|
|
45
|
+
func populationCostLiteral(histogram *histogramLiteral) float64 {
|
|
46
|
+
var data_size uint = histogramDataSizeLiteral()
|
|
47
|
+
var count int = 0
|
|
48
|
+
var s [5]uint
|
|
49
|
+
var bits float64 = 0.0
|
|
50
|
+
var i uint
|
|
51
|
+
if histogram.total_count_ == 0 {
|
|
52
|
+
return kOneSymbolHistogramCost
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
for i = 0; i < data_size; i++ {
|
|
56
|
+
if histogram.data_[i] > 0 {
|
|
57
|
+
s[count] = i
|
|
58
|
+
count++
|
|
59
|
+
if count > 4 {
|
|
60
|
+
break
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
if count == 1 {
|
|
66
|
+
return kOneSymbolHistogramCost
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
if count == 2 {
|
|
70
|
+
return kTwoSymbolHistogramCost + float64(histogram.total_count_)
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
if count == 3 {
|
|
74
|
+
var histo0 uint32 = histogram.data_[s[0]]
|
|
75
|
+
var histo1 uint32 = histogram.data_[s[1]]
|
|
76
|
+
var histo2 uint32 = histogram.data_[s[2]]
|
|
77
|
+
var histomax uint32 = brotli_max_uint32_t(histo0, brotli_max_uint32_t(histo1, histo2))
|
|
78
|
+
return kThreeSymbolHistogramCost + 2*(float64(histo0)+float64(histo1)+float64(histo2)) - float64(histomax)
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
if count == 4 {
|
|
82
|
+
var histo [4]uint32
|
|
83
|
+
var h23 uint32
|
|
84
|
+
var histomax uint32
|
|
85
|
+
for i = 0; i < 4; i++ {
|
|
86
|
+
histo[i] = histogram.data_[s[i]]
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/* Sort */
|
|
90
|
+
for i = 0; i < 4; i++ {
|
|
91
|
+
var j uint
|
|
92
|
+
for j = i + 1; j < 4; j++ {
|
|
93
|
+
if histo[j] > histo[i] {
|
|
94
|
+
var tmp uint32 = histo[j]
|
|
95
|
+
histo[j] = histo[i]
|
|
96
|
+
histo[i] = tmp
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
h23 = histo[2] + histo[3]
|
|
102
|
+
histomax = brotli_max_uint32_t(h23, histo[0])
|
|
103
|
+
return kFourSymbolHistogramCost + 3*float64(h23) + 2*(float64(histo[0])+float64(histo[1])) - float64(histomax)
|
|
104
|
+
}
|
|
105
|
+
{
|
|
106
|
+
var max_depth uint = 1
|
|
107
|
+
var depth_histo = [codeLengthCodes]uint32{0}
|
|
108
|
+
/* In this loop we compute the entropy of the histogram and simultaneously
|
|
109
|
+
build a simplified histogram of the code length codes where we use the
|
|
110
|
+
zero repeat code 17, but we don't use the non-zero repeat code 16. */
|
|
111
|
+
|
|
112
|
+
var log2total float64 = fastLog2(histogram.total_count_)
|
|
113
|
+
for i = 0; i < data_size; {
|
|
114
|
+
if histogram.data_[i] > 0 {
|
|
115
|
+
var log2p float64 = log2total - fastLog2(uint(histogram.data_[i]))
|
|
116
|
+
/* Compute -log2(P(symbol)) = -log2(count(symbol)/total_count) =
|
|
117
|
+
= log2(total_count) - log2(count(symbol)) */
|
|
118
|
+
|
|
119
|
+
var depth uint = uint(log2p + 0.5)
|
|
120
|
+
/* Approximate the bit depth by round(-log2(P(symbol))) */
|
|
121
|
+
bits += float64(histogram.data_[i]) * log2p
|
|
122
|
+
|
|
123
|
+
if depth > 15 {
|
|
124
|
+
depth = 15
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
if depth > max_depth {
|
|
128
|
+
max_depth = depth
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
depth_histo[depth]++
|
|
132
|
+
i++
|
|
133
|
+
} else {
|
|
134
|
+
var reps uint32 = 1
|
|
135
|
+
/* Compute the run length of zeros and add the appropriate number of 0
|
|
136
|
+
and 17 code length codes to the code length code histogram. */
|
|
137
|
+
|
|
138
|
+
var k uint
|
|
139
|
+
for k = i + 1; k < data_size && histogram.data_[k] == 0; k++ {
|
|
140
|
+
reps++
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
i += uint(reps)
|
|
144
|
+
if i == data_size {
|
|
145
|
+
/* Don't add any cost for the last zero run, since these are encoded
|
|
146
|
+
only implicitly. */
|
|
147
|
+
break
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
if reps < 3 {
|
|
151
|
+
depth_histo[0] += reps
|
|
152
|
+
} else {
|
|
153
|
+
reps -= 2
|
|
154
|
+
for reps > 0 {
|
|
155
|
+
depth_histo[repeatZeroCodeLength]++
|
|
156
|
+
|
|
157
|
+
/* Add the 3 extra bits for the 17 code length code. */
|
|
158
|
+
bits += 3
|
|
159
|
+
|
|
160
|
+
reps >>= 3
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
/* Add the estimated encoding cost of the code length code histogram. */
|
|
167
|
+
bits += float64(18 + 2*max_depth)
|
|
168
|
+
|
|
169
|
+
/* Add the entropy of the code length code histogram. */
|
|
170
|
+
bits += bitsEntropy(depth_histo[:], codeLengthCodes)
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
return bits
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
func populationCostCommand(histogram *histogramCommand) float64 {
|
|
177
|
+
var data_size uint = histogramDataSizeCommand()
|
|
178
|
+
var count int = 0
|
|
179
|
+
var s [5]uint
|
|
180
|
+
var bits float64 = 0.0
|
|
181
|
+
var i uint
|
|
182
|
+
if histogram.total_count_ == 0 {
|
|
183
|
+
return kOneSymbolHistogramCost
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
for i = 0; i < data_size; i++ {
|
|
187
|
+
if histogram.data_[i] > 0 {
|
|
188
|
+
s[count] = i
|
|
189
|
+
count++
|
|
190
|
+
if count > 4 {
|
|
191
|
+
break
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
if count == 1 {
|
|
197
|
+
return kOneSymbolHistogramCost
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
if count == 2 {
|
|
201
|
+
return kTwoSymbolHistogramCost + float64(histogram.total_count_)
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
if count == 3 {
|
|
205
|
+
var histo0 uint32 = histogram.data_[s[0]]
|
|
206
|
+
var histo1 uint32 = histogram.data_[s[1]]
|
|
207
|
+
var histo2 uint32 = histogram.data_[s[2]]
|
|
208
|
+
var histomax uint32 = brotli_max_uint32_t(histo0, brotli_max_uint32_t(histo1, histo2))
|
|
209
|
+
return kThreeSymbolHistogramCost + 2*(float64(histo0)+float64(histo1)+float64(histo2)) - float64(histomax)
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
if count == 4 {
|
|
213
|
+
var histo [4]uint32
|
|
214
|
+
var h23 uint32
|
|
215
|
+
var histomax uint32
|
|
216
|
+
for i = 0; i < 4; i++ {
|
|
217
|
+
histo[i] = histogram.data_[s[i]]
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
/* Sort */
|
|
221
|
+
for i = 0; i < 4; i++ {
|
|
222
|
+
var j uint
|
|
223
|
+
for j = i + 1; j < 4; j++ {
|
|
224
|
+
if histo[j] > histo[i] {
|
|
225
|
+
var tmp uint32 = histo[j]
|
|
226
|
+
histo[j] = histo[i]
|
|
227
|
+
histo[i] = tmp
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
h23 = histo[2] + histo[3]
|
|
233
|
+
histomax = brotli_max_uint32_t(h23, histo[0])
|
|
234
|
+
return kFourSymbolHistogramCost + 3*float64(h23) + 2*(float64(histo[0])+float64(histo[1])) - float64(histomax)
|
|
235
|
+
}
|
|
236
|
+
{
|
|
237
|
+
var max_depth uint = 1
|
|
238
|
+
var depth_histo = [codeLengthCodes]uint32{0}
|
|
239
|
+
/* In this loop we compute the entropy of the histogram and simultaneously
|
|
240
|
+
build a simplified histogram of the code length codes where we use the
|
|
241
|
+
zero repeat code 17, but we don't use the non-zero repeat code 16. */
|
|
242
|
+
|
|
243
|
+
var log2total float64 = fastLog2(histogram.total_count_)
|
|
244
|
+
for i = 0; i < data_size; {
|
|
245
|
+
if histogram.data_[i] > 0 {
|
|
246
|
+
var log2p float64 = log2total - fastLog2(uint(histogram.data_[i]))
|
|
247
|
+
/* Compute -log2(P(symbol)) = -log2(count(symbol)/total_count) =
|
|
248
|
+
= log2(total_count) - log2(count(symbol)) */
|
|
249
|
+
|
|
250
|
+
var depth uint = uint(log2p + 0.5)
|
|
251
|
+
/* Approximate the bit depth by round(-log2(P(symbol))) */
|
|
252
|
+
bits += float64(histogram.data_[i]) * log2p
|
|
253
|
+
|
|
254
|
+
if depth > 15 {
|
|
255
|
+
depth = 15
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
if depth > max_depth {
|
|
259
|
+
max_depth = depth
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
depth_histo[depth]++
|
|
263
|
+
i++
|
|
264
|
+
} else {
|
|
265
|
+
var reps uint32 = 1
|
|
266
|
+
/* Compute the run length of zeros and add the appropriate number of 0
|
|
267
|
+
and 17 code length codes to the code length code histogram. */
|
|
268
|
+
|
|
269
|
+
var k uint
|
|
270
|
+
for k = i + 1; k < data_size && histogram.data_[k] == 0; k++ {
|
|
271
|
+
reps++
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
i += uint(reps)
|
|
275
|
+
if i == data_size {
|
|
276
|
+
/* Don't add any cost for the last zero run, since these are encoded
|
|
277
|
+
only implicitly. */
|
|
278
|
+
break
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
if reps < 3 {
|
|
282
|
+
depth_histo[0] += reps
|
|
283
|
+
} else {
|
|
284
|
+
reps -= 2
|
|
285
|
+
for reps > 0 {
|
|
286
|
+
depth_histo[repeatZeroCodeLength]++
|
|
287
|
+
|
|
288
|
+
/* Add the 3 extra bits for the 17 code length code. */
|
|
289
|
+
bits += 3
|
|
290
|
+
|
|
291
|
+
reps >>= 3
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
/* Add the estimated encoding cost of the code length code histogram. */
|
|
298
|
+
bits += float64(18 + 2*max_depth)
|
|
299
|
+
|
|
300
|
+
/* Add the entropy of the code length code histogram. */
|
|
301
|
+
bits += bitsEntropy(depth_histo[:], codeLengthCodes)
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
return bits
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
func populationCostDistance(histogram *histogramDistance) float64 {
|
|
308
|
+
var data_size uint = histogramDataSizeDistance()
|
|
309
|
+
var count int = 0
|
|
310
|
+
var s [5]uint
|
|
311
|
+
var bits float64 = 0.0
|
|
312
|
+
var i uint
|
|
313
|
+
if histogram.total_count_ == 0 {
|
|
314
|
+
return kOneSymbolHistogramCost
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
for i = 0; i < data_size; i++ {
|
|
318
|
+
if histogram.data_[i] > 0 {
|
|
319
|
+
s[count] = i
|
|
320
|
+
count++
|
|
321
|
+
if count > 4 {
|
|
322
|
+
break
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
if count == 1 {
|
|
328
|
+
return kOneSymbolHistogramCost
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
if count == 2 {
|
|
332
|
+
return kTwoSymbolHistogramCost + float64(histogram.total_count_)
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
if count == 3 {
|
|
336
|
+
var histo0 uint32 = histogram.data_[s[0]]
|
|
337
|
+
var histo1 uint32 = histogram.data_[s[1]]
|
|
338
|
+
var histo2 uint32 = histogram.data_[s[2]]
|
|
339
|
+
var histomax uint32 = brotli_max_uint32_t(histo0, brotli_max_uint32_t(histo1, histo2))
|
|
340
|
+
return kThreeSymbolHistogramCost + 2*(float64(histo0)+float64(histo1)+float64(histo2)) - float64(histomax)
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
if count == 4 {
|
|
344
|
+
var histo [4]uint32
|
|
345
|
+
var h23 uint32
|
|
346
|
+
var histomax uint32
|
|
347
|
+
for i = 0; i < 4; i++ {
|
|
348
|
+
histo[i] = histogram.data_[s[i]]
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
/* Sort */
|
|
352
|
+
for i = 0; i < 4; i++ {
|
|
353
|
+
var j uint
|
|
354
|
+
for j = i + 1; j < 4; j++ {
|
|
355
|
+
if histo[j] > histo[i] {
|
|
356
|
+
var tmp uint32 = histo[j]
|
|
357
|
+
histo[j] = histo[i]
|
|
358
|
+
histo[i] = tmp
|
|
359
|
+
}
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
h23 = histo[2] + histo[3]
|
|
364
|
+
histomax = brotli_max_uint32_t(h23, histo[0])
|
|
365
|
+
return kFourSymbolHistogramCost + 3*float64(h23) + 2*(float64(histo[0])+float64(histo[1])) - float64(histomax)
|
|
366
|
+
}
|
|
367
|
+
{
|
|
368
|
+
var max_depth uint = 1
|
|
369
|
+
var depth_histo = [codeLengthCodes]uint32{0}
|
|
370
|
+
/* In this loop we compute the entropy of the histogram and simultaneously
|
|
371
|
+
build a simplified histogram of the code length codes where we use the
|
|
372
|
+
zero repeat code 17, but we don't use the non-zero repeat code 16. */
|
|
373
|
+
|
|
374
|
+
var log2total float64 = fastLog2(histogram.total_count_)
|
|
375
|
+
for i = 0; i < data_size; {
|
|
376
|
+
if histogram.data_[i] > 0 {
|
|
377
|
+
var log2p float64 = log2total - fastLog2(uint(histogram.data_[i]))
|
|
378
|
+
/* Compute -log2(P(symbol)) = -log2(count(symbol)/total_count) =
|
|
379
|
+
= log2(total_count) - log2(count(symbol)) */
|
|
380
|
+
|
|
381
|
+
var depth uint = uint(log2p + 0.5)
|
|
382
|
+
/* Approximate the bit depth by round(-log2(P(symbol))) */
|
|
383
|
+
bits += float64(histogram.data_[i]) * log2p
|
|
384
|
+
|
|
385
|
+
if depth > 15 {
|
|
386
|
+
depth = 15
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
if depth > max_depth {
|
|
390
|
+
max_depth = depth
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
depth_histo[depth]++
|
|
394
|
+
i++
|
|
395
|
+
} else {
|
|
396
|
+
var reps uint32 = 1
|
|
397
|
+
/* Compute the run length of zeros and add the appropriate number of 0
|
|
398
|
+
and 17 code length codes to the code length code histogram. */
|
|
399
|
+
|
|
400
|
+
var k uint
|
|
401
|
+
for k = i + 1; k < data_size && histogram.data_[k] == 0; k++ {
|
|
402
|
+
reps++
|
|
403
|
+
}
|
|
404
|
+
|
|
405
|
+
i += uint(reps)
|
|
406
|
+
if i == data_size {
|
|
407
|
+
/* Don't add any cost for the last zero run, since these are encoded
|
|
408
|
+
only implicitly. */
|
|
409
|
+
break
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
if reps < 3 {
|
|
413
|
+
depth_histo[0] += reps
|
|
414
|
+
} else {
|
|
415
|
+
reps -= 2
|
|
416
|
+
for reps > 0 {
|
|
417
|
+
depth_histo[repeatZeroCodeLength]++
|
|
418
|
+
|
|
419
|
+
/* Add the 3 extra bits for the 17 code length code. */
|
|
420
|
+
bits += 3
|
|
421
|
+
|
|
422
|
+
reps >>= 3
|
|
423
|
+
}
|
|
424
|
+
}
|
|
425
|
+
}
|
|
426
|
+
}
|
|
427
|
+
|
|
428
|
+
/* Add the estimated encoding cost of the code length code histogram. */
|
|
429
|
+
bits += float64(18 + 2*max_depth)
|
|
430
|
+
|
|
431
|
+
/* Add the entropy of the code length code histogram. */
|
|
432
|
+
bits += bitsEntropy(depth_histo[:], codeLengthCodes)
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
return bits
|
|
436
|
+
}
|
|
@@ -0,0 +1,266 @@
|
|
|
1
|
+
package brotli
|
|
2
|
+
|
|
3
|
+
import "encoding/binary"
|
|
4
|
+
|
|
5
|
+
/* Copyright 2013 Google Inc. All Rights Reserved.
|
|
6
|
+
|
|
7
|
+
Distributed under MIT license.
|
|
8
|
+
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
/* Bit reading helpers */
|
|
12
|
+
|
|
13
|
+
const shortFillBitWindowRead = (8 >> 1)
|
|
14
|
+
|
|
15
|
+
var kBitMask = [33]uint32{
|
|
16
|
+
0x00000000,
|
|
17
|
+
0x00000001,
|
|
18
|
+
0x00000003,
|
|
19
|
+
0x00000007,
|
|
20
|
+
0x0000000F,
|
|
21
|
+
0x0000001F,
|
|
22
|
+
0x0000003F,
|
|
23
|
+
0x0000007F,
|
|
24
|
+
0x000000FF,
|
|
25
|
+
0x000001FF,
|
|
26
|
+
0x000003FF,
|
|
27
|
+
0x000007FF,
|
|
28
|
+
0x00000FFF,
|
|
29
|
+
0x00001FFF,
|
|
30
|
+
0x00003FFF,
|
|
31
|
+
0x00007FFF,
|
|
32
|
+
0x0000FFFF,
|
|
33
|
+
0x0001FFFF,
|
|
34
|
+
0x0003FFFF,
|
|
35
|
+
0x0007FFFF,
|
|
36
|
+
0x000FFFFF,
|
|
37
|
+
0x001FFFFF,
|
|
38
|
+
0x003FFFFF,
|
|
39
|
+
0x007FFFFF,
|
|
40
|
+
0x00FFFFFF,
|
|
41
|
+
0x01FFFFFF,
|
|
42
|
+
0x03FFFFFF,
|
|
43
|
+
0x07FFFFFF,
|
|
44
|
+
0x0FFFFFFF,
|
|
45
|
+
0x1FFFFFFF,
|
|
46
|
+
0x3FFFFFFF,
|
|
47
|
+
0x7FFFFFFF,
|
|
48
|
+
0xFFFFFFFF,
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
func bitMask(n uint32) uint32 {
|
|
52
|
+
return kBitMask[n]
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
type bitReader struct {
|
|
56
|
+
val_ uint64
|
|
57
|
+
bit_pos_ uint32
|
|
58
|
+
input []byte
|
|
59
|
+
input_len uint
|
|
60
|
+
byte_pos uint
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
type bitReaderState struct {
|
|
64
|
+
val_ uint64
|
|
65
|
+
bit_pos_ uint32
|
|
66
|
+
input []byte
|
|
67
|
+
input_len uint
|
|
68
|
+
byte_pos uint
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/* Initializes the BrotliBitReader fields. */
|
|
72
|
+
|
|
73
|
+
/* Ensures that accumulator is not empty.
|
|
74
|
+
May consume up to sizeof(brotli_reg_t) - 1 bytes of input.
|
|
75
|
+
Returns false if data is required but there is no input available.
|
|
76
|
+
For BROTLI_ALIGNED_READ this function also prepares bit reader for aligned
|
|
77
|
+
reading. */
|
|
78
|
+
func bitReaderSaveState(from *bitReader, to *bitReaderState) {
|
|
79
|
+
to.val_ = from.val_
|
|
80
|
+
to.bit_pos_ = from.bit_pos_
|
|
81
|
+
to.input = from.input
|
|
82
|
+
to.input_len = from.input_len
|
|
83
|
+
to.byte_pos = from.byte_pos
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
func bitReaderRestoreState(to *bitReader, from *bitReaderState) {
|
|
87
|
+
to.val_ = from.val_
|
|
88
|
+
to.bit_pos_ = from.bit_pos_
|
|
89
|
+
to.input = from.input
|
|
90
|
+
to.input_len = from.input_len
|
|
91
|
+
to.byte_pos = from.byte_pos
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
func getAvailableBits(br *bitReader) uint32 {
|
|
95
|
+
return 64 - br.bit_pos_
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/* Returns amount of unread bytes the bit reader still has buffered from the
|
|
99
|
+
BrotliInput, including whole bytes in br->val_. */
|
|
100
|
+
func getRemainingBytes(br *bitReader) uint {
|
|
101
|
+
return uint(uint32(br.input_len-br.byte_pos) + (getAvailableBits(br) >> 3))
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/* Checks if there is at least |num| bytes left in the input ring-buffer
|
|
105
|
+
(excluding the bits remaining in br->val_). */
|
|
106
|
+
func checkInputAmount(br *bitReader, num uint) bool {
|
|
107
|
+
return br.input_len-br.byte_pos >= num
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/* Guarantees that there are at least |n_bits| + 1 bits in accumulator.
|
|
111
|
+
Precondition: accumulator contains at least 1 bit.
|
|
112
|
+
|n_bits| should be in the range [1..24] for regular build. For portable
|
|
113
|
+
non-64-bit little-endian build only 16 bits are safe to request. */
|
|
114
|
+
func fillBitWindow(br *bitReader, n_bits uint32) {
|
|
115
|
+
if br.bit_pos_ >= 32 {
|
|
116
|
+
br.val_ >>= 32
|
|
117
|
+
br.bit_pos_ ^= 32 /* here same as -= 32 because of the if condition */
|
|
118
|
+
br.val_ |= (uint64(binary.LittleEndian.Uint32(br.input[br.byte_pos:]))) << 32
|
|
119
|
+
br.byte_pos += 4
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/* Mostly like BrotliFillBitWindow, but guarantees only 16 bits and reads no
|
|
124
|
+
more than BROTLI_SHORT_FILL_BIT_WINDOW_READ bytes of input. */
|
|
125
|
+
func fillBitWindow16(br *bitReader) {
|
|
126
|
+
fillBitWindow(br, 17)
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/* Tries to pull one byte of input to accumulator.
|
|
130
|
+
Returns false if there is no input available. */
|
|
131
|
+
func pullByte(br *bitReader) bool {
|
|
132
|
+
if br.byte_pos == br.input_len {
|
|
133
|
+
return false
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
br.val_ >>= 8
|
|
137
|
+
br.val_ |= (uint64(br.input[br.byte_pos])) << 56
|
|
138
|
+
br.bit_pos_ -= 8
|
|
139
|
+
br.byte_pos++
|
|
140
|
+
return true
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/* Returns currently available bits.
|
|
144
|
+
The number of valid bits could be calculated by BrotliGetAvailableBits. */
|
|
145
|
+
func getBitsUnmasked(br *bitReader) uint64 {
|
|
146
|
+
return br.val_ >> br.bit_pos_
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/* Like BrotliGetBits, but does not mask the result.
|
|
150
|
+
The result contains at least 16 valid bits. */
|
|
151
|
+
func get16BitsUnmasked(br *bitReader) uint32 {
|
|
152
|
+
fillBitWindow(br, 16)
|
|
153
|
+
return uint32(getBitsUnmasked(br))
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
/* Returns the specified number of bits from |br| without advancing bit
|
|
157
|
+
position. */
|
|
158
|
+
func getBits(br *bitReader, n_bits uint32) uint32 {
|
|
159
|
+
fillBitWindow(br, n_bits)
|
|
160
|
+
return uint32(getBitsUnmasked(br)) & bitMask(n_bits)
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
/* Tries to peek the specified amount of bits. Returns false, if there
|
|
164
|
+
is not enough input. */
|
|
165
|
+
func safeGetBits(br *bitReader, n_bits uint32, val *uint32) bool {
|
|
166
|
+
for getAvailableBits(br) < n_bits {
|
|
167
|
+
if !pullByte(br) {
|
|
168
|
+
return false
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
*val = uint32(getBitsUnmasked(br)) & bitMask(n_bits)
|
|
173
|
+
return true
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
/* Advances the bit pos by |n_bits|. */
|
|
177
|
+
func dropBits(br *bitReader, n_bits uint32) {
|
|
178
|
+
br.bit_pos_ += n_bits
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
func bitReaderUnload(br *bitReader) {
|
|
182
|
+
var unused_bytes uint32 = getAvailableBits(br) >> 3
|
|
183
|
+
var unused_bits uint32 = unused_bytes << 3
|
|
184
|
+
br.byte_pos -= uint(unused_bytes)
|
|
185
|
+
if unused_bits == 64 {
|
|
186
|
+
br.val_ = 0
|
|
187
|
+
} else {
|
|
188
|
+
br.val_ <<= unused_bits
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
br.bit_pos_ += unused_bits
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
/* Reads the specified number of bits from |br| and advances the bit pos.
|
|
195
|
+
Precondition: accumulator MUST contain at least |n_bits|. */
|
|
196
|
+
func takeBits(br *bitReader, n_bits uint32, val *uint32) {
|
|
197
|
+
*val = uint32(getBitsUnmasked(br)) & bitMask(n_bits)
|
|
198
|
+
dropBits(br, n_bits)
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
/* Reads the specified number of bits from |br| and advances the bit pos.
|
|
202
|
+
Assumes that there is enough input to perform BrotliFillBitWindow. */
|
|
203
|
+
func readBits(br *bitReader, n_bits uint32) uint32 {
|
|
204
|
+
var val uint32
|
|
205
|
+
fillBitWindow(br, n_bits)
|
|
206
|
+
takeBits(br, n_bits, &val)
|
|
207
|
+
return val
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
/* Tries to read the specified amount of bits. Returns false, if there
|
|
211
|
+
is not enough input. |n_bits| MUST be positive. */
|
|
212
|
+
func safeReadBits(br *bitReader, n_bits uint32, val *uint32) bool {
|
|
213
|
+
for getAvailableBits(br) < n_bits {
|
|
214
|
+
if !pullByte(br) {
|
|
215
|
+
return false
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
takeBits(br, n_bits, val)
|
|
220
|
+
return true
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
/* Advances the bit reader position to the next byte boundary and verifies
|
|
224
|
+
that any skipped bits are set to zero. */
|
|
225
|
+
func bitReaderJumpToByteBoundary(br *bitReader) bool {
|
|
226
|
+
var pad_bits_count uint32 = getAvailableBits(br) & 0x7
|
|
227
|
+
var pad_bits uint32 = 0
|
|
228
|
+
if pad_bits_count != 0 {
|
|
229
|
+
takeBits(br, pad_bits_count, &pad_bits)
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
return pad_bits == 0
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
/* Copies remaining input bytes stored in the bit reader to the output. Value
|
|
236
|
+
|num| may not be larger than BrotliGetRemainingBytes. The bit reader must be
|
|
237
|
+
warmed up again after this. */
|
|
238
|
+
func copyBytes(dest []byte, br *bitReader, num uint) {
|
|
239
|
+
for getAvailableBits(br) >= 8 && num > 0 {
|
|
240
|
+
dest[0] = byte(getBitsUnmasked(br))
|
|
241
|
+
dropBits(br, 8)
|
|
242
|
+
dest = dest[1:]
|
|
243
|
+
num--
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
copy(dest, br.input[br.byte_pos:][:num])
|
|
247
|
+
br.byte_pos += num
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
func initBitReader(br *bitReader) {
|
|
251
|
+
br.val_ = 0
|
|
252
|
+
br.bit_pos_ = 64
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
func warmupBitReader(br *bitReader) bool {
|
|
256
|
+
/* Fixing alignment after unaligned BrotliFillWindow would result accumulator
|
|
257
|
+
overflow. If unalignment is caused by BrotliSafeReadBits, then there is
|
|
258
|
+
enough space in accumulator to fix alignment. */
|
|
259
|
+
if getAvailableBits(br) == 0 {
|
|
260
|
+
if !pullByte(br) {
|
|
261
|
+
return false
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
return true
|
|
266
|
+
}
|