typenative 0.3.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +57 -0
- package/README.md +17 -1
- package/bin/cli.js +25 -11
- package/bin/go/convert.js +11 -1
- package/bin/index.js +221 -47
- package/bin/main.js +35 -14
- package/bin/node/assert.js +2 -2
- package/bin/node/buffer.js +9 -0
- package/bin/node/dns.js +1 -1
- package/bin/node/events.js +1 -1
- package/bin/node/fs.js +1 -1
- package/bin/node/fs_watch.js +407 -0
- package/bin/node/http2.js +33 -6
- package/bin/node/readline.js +1 -1
- package/bin/node/registry.js +2 -1
- package/bin/node/stream.js +6 -3
- package/bin/node/timers.js +1 -1
- package/bin/node/typed_arrays.js +608 -0
- package/bin/node/util.js +3 -0
- package/bin/node/zlib.js +131 -16
- package/bin/transpiler.js +635 -44
- package/bin/version.js +1 -1
- package/conformance.md +16 -4
- package/examples/README.md +54 -0
- package/examples/fibonacci/README.md +89 -0
- package/examples/fibonacci/bench.mjs +87 -0
- package/examples/fibonacci/main.ts +21 -0
- package/examples/hello-world/README.md +62 -0
- package/examples/hello-world/bench.mjs +66 -0
- package/examples/hello-world/main.ts +7 -0
- package/examples/json-server/README.md +125 -0
- package/examples/json-server/bench.mjs +171 -0
- package/examples/json-server/main.ts +67 -0
- package/govendor/github.com/andybalholm/brotli/LICENSE +19 -0
- package/govendor/github.com/andybalholm/brotli/README.md +19 -0
- package/govendor/github.com/andybalholm/brotli/backward_references.go +185 -0
- package/govendor/github.com/andybalholm/brotli/backward_references_hq.go +796 -0
- package/govendor/github.com/andybalholm/brotli/bit_cost.go +436 -0
- package/govendor/github.com/andybalholm/brotli/bit_reader.go +266 -0
- package/govendor/github.com/andybalholm/brotli/bitwriter.go +56 -0
- package/govendor/github.com/andybalholm/brotli/block_splitter.go +144 -0
- package/govendor/github.com/andybalholm/brotli/block_splitter_command.go +434 -0
- package/govendor/github.com/andybalholm/brotli/block_splitter_distance.go +433 -0
- package/govendor/github.com/andybalholm/brotli/block_splitter_literal.go +433 -0
- package/govendor/github.com/andybalholm/brotli/brotli_bit_stream.go +1435 -0
- package/govendor/github.com/andybalholm/brotli/cluster.go +30 -0
- package/govendor/github.com/andybalholm/brotli/cluster_command.go +164 -0
- package/govendor/github.com/andybalholm/brotli/cluster_distance.go +326 -0
- package/govendor/github.com/andybalholm/brotli/cluster_literal.go +326 -0
- package/govendor/github.com/andybalholm/brotli/command.go +254 -0
- package/govendor/github.com/andybalholm/brotli/compress_fragment.go +834 -0
- package/govendor/github.com/andybalholm/brotli/compress_fragment_two_pass.go +775 -0
- package/govendor/github.com/andybalholm/brotli/constants.go +77 -0
- package/govendor/github.com/andybalholm/brotli/context.go +2176 -0
- package/govendor/github.com/andybalholm/brotli/decode.go +2570 -0
- package/govendor/github.com/andybalholm/brotli/dictionary.go +122890 -0
- package/govendor/github.com/andybalholm/brotli/dictionary_hash.go +32779 -0
- package/govendor/github.com/andybalholm/brotli/encode.go +1220 -0
- package/govendor/github.com/andybalholm/brotli/encoder.go +177 -0
- package/govendor/github.com/andybalholm/brotli/encoder_dict.go +22 -0
- package/govendor/github.com/andybalholm/brotli/encoder_fast.go +201 -0
- package/govendor/github.com/andybalholm/brotli/entropy_encode.go +592 -0
- package/govendor/github.com/andybalholm/brotli/entropy_encode_static.go +4399 -0
- package/govendor/github.com/andybalholm/brotli/fast_log.go +290 -0
- package/govendor/github.com/andybalholm/brotli/find_match_length.go +45 -0
- package/govendor/github.com/andybalholm/brotli/go.mod +3 -0
- package/govendor/github.com/andybalholm/brotli/h10.go +287 -0
- package/govendor/github.com/andybalholm/brotli/h5.go +214 -0
- package/govendor/github.com/andybalholm/brotli/h6.go +216 -0
- package/govendor/github.com/andybalholm/brotli/hash.go +342 -0
- package/govendor/github.com/andybalholm/brotli/hash_composite.go +93 -0
- package/govendor/github.com/andybalholm/brotli/hash_forgetful_chain.go +252 -0
- package/govendor/github.com/andybalholm/brotli/hash_longest_match_quickly.go +214 -0
- package/govendor/github.com/andybalholm/brotli/hash_rolling.go +168 -0
- package/govendor/github.com/andybalholm/brotli/histogram.go +226 -0
- package/govendor/github.com/andybalholm/brotli/http.go +201 -0
- package/govendor/github.com/andybalholm/brotli/huffman.go +653 -0
- package/govendor/github.com/andybalholm/brotli/literal_cost.go +182 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/bargain1.go +212 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/bargain2.go +248 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/bargain3.go +278 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/emitter.go +34 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/m0.go +175 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/m4.go +339 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/matchfinder.go +103 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/pathfinder.go +328 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/textencoder.go +53 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/trio.go +327 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/zdfast.go +258 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/zfast.go +206 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/zm.go +302 -0
- package/govendor/github.com/andybalholm/brotli/memory.go +66 -0
- package/govendor/github.com/andybalholm/brotli/metablock.go +574 -0
- package/govendor/github.com/andybalholm/brotli/metablock_command.go +165 -0
- package/govendor/github.com/andybalholm/brotli/metablock_distance.go +165 -0
- package/govendor/github.com/andybalholm/brotli/metablock_literal.go +165 -0
- package/govendor/github.com/andybalholm/brotli/params.go +37 -0
- package/govendor/github.com/andybalholm/brotli/platform.go +103 -0
- package/govendor/github.com/andybalholm/brotli/prefix.go +30 -0
- package/govendor/github.com/andybalholm/brotli/prefix_dec.go +723 -0
- package/govendor/github.com/andybalholm/brotli/quality.go +196 -0
- package/govendor/github.com/andybalholm/brotli/reader.go +127 -0
- package/govendor/github.com/andybalholm/brotli/ringbuffer.go +134 -0
- package/govendor/github.com/andybalholm/brotli/state.go +313 -0
- package/govendor/github.com/andybalholm/brotli/static_dict.go +662 -0
- package/govendor/github.com/andybalholm/brotli/static_dict_lut.go +75094 -0
- package/govendor/github.com/andybalholm/brotli/symbol_list.go +22 -0
- package/govendor/github.com/andybalholm/brotli/transform.go +641 -0
- package/govendor/github.com/andybalholm/brotli/utf8_util.go +70 -0
- package/govendor/github.com/andybalholm/brotli/util.go +7 -0
- package/govendor/github.com/andybalholm/brotli/write_bits.go +52 -0
- package/govendor/github.com/andybalholm/brotli/writer.go +162 -0
- package/govendor/github.com/fsnotify/fsnotify/.mailmap +2 -0
- package/govendor/github.com/fsnotify/fsnotify/CHANGELOG.md +651 -0
- package/govendor/github.com/fsnotify/fsnotify/CONTRIBUTING.md +147 -0
- package/govendor/github.com/fsnotify/fsnotify/LICENSE +25 -0
- package/govendor/github.com/fsnotify/fsnotify/README.md +214 -0
- package/govendor/github.com/fsnotify/fsnotify/backend_fen.go +469 -0
- package/govendor/github.com/fsnotify/fsnotify/backend_inotify.go +597 -0
- package/govendor/github.com/fsnotify/fsnotify/backend_kqueue.go +735 -0
- package/govendor/github.com/fsnotify/fsnotify/backend_other.go +22 -0
- package/govendor/github.com/fsnotify/fsnotify/backend_windows.go +702 -0
- package/govendor/github.com/fsnotify/fsnotify/fsnotify.go +526 -0
- package/govendor/github.com/fsnotify/fsnotify/go.mod +3 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/darwin.go +20 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_darwin.go +15 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_dragonfly.go +15 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_freebsd.go +20 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_kqueue.go +32 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_linux.go +56 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_netbsd.go +15 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_openbsd.go +16 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_solaris.go +45 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_windows.go +40 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/freebsd.go +20 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/internal.go +2 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/unix.go +20 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/unix2.go +25 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/windows.go +40 -0
- package/govendor/github.com/fsnotify/fsnotify/shared.go +64 -0
- package/govendor/github.com/fsnotify/fsnotify/staticcheck.conf +3 -0
- package/govendor/github.com/fsnotify/fsnotify/system_bsd.go +7 -0
- package/govendor/github.com/fsnotify/fsnotify/system_darwin.go +8 -0
- package/govendor/manifest.txt +3 -1
- package/package.json +1 -1
- package/types/typenative-npm.d.ts +72 -0
- package/types/typenative.d.ts +117 -1
|
@@ -0,0 +1,574 @@
|
|
|
1
|
+
package brotli
|
|
2
|
+
|
|
3
|
+
import (
|
|
4
|
+
"sync"
|
|
5
|
+
)
|
|
6
|
+
|
|
7
|
+
/* Copyright 2014 Google Inc. All Rights Reserved.
|
|
8
|
+
|
|
9
|
+
Distributed under MIT license.
|
|
10
|
+
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
/* Algorithms for distributing the literals and commands of a metablock between
|
|
14
|
+
block types and contexts. */
|
|
15
|
+
|
|
16
|
+
type metaBlockSplit struct {
|
|
17
|
+
literal_split blockSplit
|
|
18
|
+
command_split blockSplit
|
|
19
|
+
distance_split blockSplit
|
|
20
|
+
literal_context_map []uint32
|
|
21
|
+
literal_context_map_size uint
|
|
22
|
+
distance_context_map []uint32
|
|
23
|
+
distance_context_map_size uint
|
|
24
|
+
literal_histograms []histogramLiteral
|
|
25
|
+
literal_histograms_size uint
|
|
26
|
+
command_histograms []histogramCommand
|
|
27
|
+
command_histograms_size uint
|
|
28
|
+
distance_histograms []histogramDistance
|
|
29
|
+
distance_histograms_size uint
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
var metaBlockPool sync.Pool
|
|
33
|
+
|
|
34
|
+
func getMetaBlockSplit() *metaBlockSplit {
|
|
35
|
+
mb, _ := metaBlockPool.Get().(*metaBlockSplit)
|
|
36
|
+
|
|
37
|
+
if mb == nil {
|
|
38
|
+
mb = &metaBlockSplit{}
|
|
39
|
+
} else {
|
|
40
|
+
initBlockSplit(&mb.literal_split)
|
|
41
|
+
initBlockSplit(&mb.command_split)
|
|
42
|
+
initBlockSplit(&mb.distance_split)
|
|
43
|
+
mb.literal_context_map = mb.literal_context_map[:0]
|
|
44
|
+
mb.literal_context_map_size = 0
|
|
45
|
+
mb.distance_context_map = mb.distance_context_map[:0]
|
|
46
|
+
mb.distance_context_map_size = 0
|
|
47
|
+
mb.literal_histograms = mb.literal_histograms[:0]
|
|
48
|
+
mb.command_histograms = mb.command_histograms[:0]
|
|
49
|
+
mb.distance_histograms = mb.distance_histograms[:0]
|
|
50
|
+
}
|
|
51
|
+
return mb
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
func freeMetaBlockSplit(mb *metaBlockSplit) {
|
|
55
|
+
metaBlockPool.Put(mb)
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
func initDistanceParams(params *encoderParams, npostfix uint32, ndirect uint32) {
|
|
59
|
+
var dist_params *distanceParams = ¶ms.dist
|
|
60
|
+
var alphabet_size uint32
|
|
61
|
+
var max_distance uint32
|
|
62
|
+
|
|
63
|
+
dist_params.distance_postfix_bits = npostfix
|
|
64
|
+
dist_params.num_direct_distance_codes = ndirect
|
|
65
|
+
|
|
66
|
+
alphabet_size = uint32(distanceAlphabetSize(uint(npostfix), uint(ndirect), maxDistanceBits))
|
|
67
|
+
max_distance = ndirect + (1 << (maxDistanceBits + npostfix + 2)) - (1 << (npostfix + 2))
|
|
68
|
+
|
|
69
|
+
if params.large_window {
|
|
70
|
+
var bound = [maxNpostfix + 1]uint32{0, 4, 12, 28}
|
|
71
|
+
var postfix uint32 = 1 << npostfix
|
|
72
|
+
alphabet_size = uint32(distanceAlphabetSize(uint(npostfix), uint(ndirect), largeMaxDistanceBits))
|
|
73
|
+
|
|
74
|
+
/* The maximum distance is set so that no distance symbol used can encode
|
|
75
|
+
a distance larger than BROTLI_MAX_ALLOWED_DISTANCE with all
|
|
76
|
+
its extra bits set. */
|
|
77
|
+
if ndirect < bound[npostfix] {
|
|
78
|
+
max_distance = maxAllowedDistance - (bound[npostfix] - ndirect)
|
|
79
|
+
} else if ndirect >= bound[npostfix]+postfix {
|
|
80
|
+
max_distance = (3 << 29) - 4 + (ndirect - bound[npostfix])
|
|
81
|
+
} else {
|
|
82
|
+
max_distance = maxAllowedDistance
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
dist_params.alphabet_size = alphabet_size
|
|
87
|
+
dist_params.max_distance = uint(max_distance)
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
func recomputeDistancePrefixes(cmds []command, orig_params *distanceParams, new_params *distanceParams) {
|
|
91
|
+
if orig_params.distance_postfix_bits == new_params.distance_postfix_bits && orig_params.num_direct_distance_codes == new_params.num_direct_distance_codes {
|
|
92
|
+
return
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
for i := range cmds {
|
|
96
|
+
var cmd *command = &cmds[i]
|
|
97
|
+
if commandCopyLen(cmd) != 0 && cmd.cmd_prefix_ >= 128 {
|
|
98
|
+
prefixEncodeCopyDistance(uint(commandRestoreDistanceCode(cmd, orig_params)), uint(new_params.num_direct_distance_codes), uint(new_params.distance_postfix_bits), &cmd.dist_prefix_, &cmd.dist_extra_)
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
func computeDistanceCost(cmds []command, orig_params *distanceParams, new_params *distanceParams, cost *float64) bool {
|
|
104
|
+
var equal_params bool = false
|
|
105
|
+
var dist_prefix uint16
|
|
106
|
+
var dist_extra uint32
|
|
107
|
+
var extra_bits float64 = 0.0
|
|
108
|
+
var histo histogramDistance
|
|
109
|
+
histogramClearDistance(&histo)
|
|
110
|
+
|
|
111
|
+
if orig_params.distance_postfix_bits == new_params.distance_postfix_bits && orig_params.num_direct_distance_codes == new_params.num_direct_distance_codes {
|
|
112
|
+
equal_params = true
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
for i := range cmds {
|
|
116
|
+
cmd := &cmds[i]
|
|
117
|
+
if commandCopyLen(cmd) != 0 && cmd.cmd_prefix_ >= 128 {
|
|
118
|
+
if equal_params {
|
|
119
|
+
dist_prefix = cmd.dist_prefix_
|
|
120
|
+
} else {
|
|
121
|
+
var distance uint32 = commandRestoreDistanceCode(cmd, orig_params)
|
|
122
|
+
if distance > uint32(new_params.max_distance) {
|
|
123
|
+
return false
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
prefixEncodeCopyDistance(uint(distance), uint(new_params.num_direct_distance_codes), uint(new_params.distance_postfix_bits), &dist_prefix, &dist_extra)
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
histogramAddDistance(&histo, uint(dist_prefix)&0x3FF)
|
|
130
|
+
extra_bits += float64(dist_prefix >> 10)
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
*cost = populationCostDistance(&histo) + extra_bits
|
|
135
|
+
return true
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
var buildMetaBlock_kMaxNumberOfHistograms uint = 256
|
|
139
|
+
|
|
140
|
+
func buildMetaBlock(ringbuffer []byte, pos uint, mask uint, params *encoderParams, prev_byte byte, prev_byte2 byte, cmds []command, literal_context_mode int, mb *metaBlockSplit) {
|
|
141
|
+
var distance_histograms []histogramDistance
|
|
142
|
+
var literal_histograms []histogramLiteral
|
|
143
|
+
var literal_context_modes []int = nil
|
|
144
|
+
var literal_histograms_size uint
|
|
145
|
+
var distance_histograms_size uint
|
|
146
|
+
var i uint
|
|
147
|
+
var literal_context_multiplier uint = 1
|
|
148
|
+
var npostfix uint32
|
|
149
|
+
var ndirect_msb uint32 = 0
|
|
150
|
+
var check_orig bool = true
|
|
151
|
+
var best_dist_cost float64 = 1e99
|
|
152
|
+
var orig_params encoderParams = *params
|
|
153
|
+
/* Histogram ids need to fit in one byte. */
|
|
154
|
+
|
|
155
|
+
var new_params encoderParams = *params
|
|
156
|
+
|
|
157
|
+
for npostfix = 0; npostfix <= maxNpostfix; npostfix++ {
|
|
158
|
+
for ; ndirect_msb < 16; ndirect_msb++ {
|
|
159
|
+
var ndirect uint32 = ndirect_msb << npostfix
|
|
160
|
+
var skip bool
|
|
161
|
+
var dist_cost float64
|
|
162
|
+
initDistanceParams(&new_params, npostfix, ndirect)
|
|
163
|
+
if npostfix == orig_params.dist.distance_postfix_bits && ndirect == orig_params.dist.num_direct_distance_codes {
|
|
164
|
+
check_orig = false
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
skip = !computeDistanceCost(cmds, &orig_params.dist, &new_params.dist, &dist_cost)
|
|
168
|
+
if skip || (dist_cost > best_dist_cost) {
|
|
169
|
+
break
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
best_dist_cost = dist_cost
|
|
173
|
+
params.dist = new_params.dist
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
if ndirect_msb > 0 {
|
|
177
|
+
ndirect_msb--
|
|
178
|
+
}
|
|
179
|
+
ndirect_msb /= 2
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
if check_orig {
|
|
183
|
+
var dist_cost float64
|
|
184
|
+
computeDistanceCost(cmds, &orig_params.dist, &orig_params.dist, &dist_cost)
|
|
185
|
+
if dist_cost < best_dist_cost {
|
|
186
|
+
/* NB: currently unused; uncomment when more param tuning is added. */
|
|
187
|
+
/* best_dist_cost = dist_cost; */
|
|
188
|
+
params.dist = orig_params.dist
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
recomputeDistancePrefixes(cmds, &orig_params.dist, ¶ms.dist)
|
|
193
|
+
|
|
194
|
+
splitBlock(cmds, ringbuffer, pos, mask, params, &mb.literal_split, &mb.command_split, &mb.distance_split)
|
|
195
|
+
|
|
196
|
+
if !params.disable_literal_context_modeling {
|
|
197
|
+
literal_context_multiplier = 1 << literalContextBits
|
|
198
|
+
literal_context_modes = make([]int, (mb.literal_split.num_types))
|
|
199
|
+
for i = 0; i < mb.literal_split.num_types; i++ {
|
|
200
|
+
literal_context_modes[i] = literal_context_mode
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
literal_histograms_size = mb.literal_split.num_types * literal_context_multiplier
|
|
205
|
+
literal_histograms = make([]histogramLiteral, literal_histograms_size)
|
|
206
|
+
clearHistogramsLiteral(literal_histograms, literal_histograms_size)
|
|
207
|
+
|
|
208
|
+
distance_histograms_size = mb.distance_split.num_types << distanceContextBits
|
|
209
|
+
distance_histograms = make([]histogramDistance, distance_histograms_size)
|
|
210
|
+
clearHistogramsDistance(distance_histograms, distance_histograms_size)
|
|
211
|
+
|
|
212
|
+
mb.command_histograms_size = mb.command_split.num_types
|
|
213
|
+
if cap(mb.command_histograms) < int(mb.command_histograms_size) {
|
|
214
|
+
mb.command_histograms = make([]histogramCommand, (mb.command_histograms_size))
|
|
215
|
+
} else {
|
|
216
|
+
mb.command_histograms = mb.command_histograms[:mb.command_histograms_size]
|
|
217
|
+
}
|
|
218
|
+
clearHistogramsCommand(mb.command_histograms, mb.command_histograms_size)
|
|
219
|
+
|
|
220
|
+
buildHistogramsWithContext(cmds, &mb.literal_split, &mb.command_split, &mb.distance_split, ringbuffer, pos, mask, prev_byte, prev_byte2, literal_context_modes, literal_histograms, mb.command_histograms, distance_histograms)
|
|
221
|
+
literal_context_modes = nil
|
|
222
|
+
|
|
223
|
+
mb.literal_context_map_size = mb.literal_split.num_types << literalContextBits
|
|
224
|
+
if cap(mb.literal_context_map) < int(mb.literal_context_map_size) {
|
|
225
|
+
mb.literal_context_map = make([]uint32, (mb.literal_context_map_size))
|
|
226
|
+
} else {
|
|
227
|
+
mb.literal_context_map = mb.literal_context_map[:mb.literal_context_map_size]
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
mb.literal_histograms_size = mb.literal_context_map_size
|
|
231
|
+
if cap(mb.literal_histograms) < int(mb.literal_histograms_size) {
|
|
232
|
+
mb.literal_histograms = make([]histogramLiteral, (mb.literal_histograms_size))
|
|
233
|
+
} else {
|
|
234
|
+
mb.literal_histograms = mb.literal_histograms[:mb.literal_histograms_size]
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
clusterHistogramsLiteral(literal_histograms, literal_histograms_size, buildMetaBlock_kMaxNumberOfHistograms, mb.literal_histograms, &mb.literal_histograms_size, mb.literal_context_map)
|
|
238
|
+
literal_histograms = nil
|
|
239
|
+
|
|
240
|
+
if params.disable_literal_context_modeling {
|
|
241
|
+
/* Distribute assignment to all contexts. */
|
|
242
|
+
for i = mb.literal_split.num_types; i != 0; {
|
|
243
|
+
var j uint = 0
|
|
244
|
+
i--
|
|
245
|
+
for ; j < 1<<literalContextBits; j++ {
|
|
246
|
+
mb.literal_context_map[(i<<literalContextBits)+j] = mb.literal_context_map[i]
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
mb.distance_context_map_size = mb.distance_split.num_types << distanceContextBits
|
|
252
|
+
if cap(mb.distance_context_map) < int(mb.distance_context_map_size) {
|
|
253
|
+
mb.distance_context_map = make([]uint32, (mb.distance_context_map_size))
|
|
254
|
+
} else {
|
|
255
|
+
mb.distance_context_map = mb.distance_context_map[:mb.distance_context_map_size]
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
mb.distance_histograms_size = mb.distance_context_map_size
|
|
259
|
+
if cap(mb.distance_histograms) < int(mb.distance_histograms_size) {
|
|
260
|
+
mb.distance_histograms = make([]histogramDistance, (mb.distance_histograms_size))
|
|
261
|
+
} else {
|
|
262
|
+
mb.distance_histograms = mb.distance_histograms[:mb.distance_histograms_size]
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
clusterHistogramsDistance(distance_histograms, mb.distance_context_map_size, buildMetaBlock_kMaxNumberOfHistograms, mb.distance_histograms, &mb.distance_histograms_size, mb.distance_context_map)
|
|
266
|
+
distance_histograms = nil
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
const maxStaticContexts = 13
|
|
270
|
+
|
|
271
|
+
/* Greedy block splitter for one block category (literal, command or distance).
|
|
272
|
+
Gathers histograms for all context buckets. */
|
|
273
|
+
type contextBlockSplitter struct {
|
|
274
|
+
alphabet_size_ uint
|
|
275
|
+
num_contexts_ uint
|
|
276
|
+
max_block_types_ uint
|
|
277
|
+
min_block_size_ uint
|
|
278
|
+
split_threshold_ float64
|
|
279
|
+
num_blocks_ uint
|
|
280
|
+
split_ *blockSplit
|
|
281
|
+
histograms_ []histogramLiteral
|
|
282
|
+
histograms_size_ *uint
|
|
283
|
+
target_block_size_ uint
|
|
284
|
+
block_size_ uint
|
|
285
|
+
curr_histogram_ix_ uint
|
|
286
|
+
last_histogram_ix_ [2]uint
|
|
287
|
+
last_entropy_ [2 * maxStaticContexts]float64
|
|
288
|
+
merge_last_count_ uint
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
func initContextBlockSplitter(self *contextBlockSplitter, alphabet_size uint, num_contexts uint, min_block_size uint, split_threshold float64, num_symbols uint, split *blockSplit, histograms *[]histogramLiteral, histograms_size *uint) {
|
|
292
|
+
var max_num_blocks uint = num_symbols/min_block_size + 1
|
|
293
|
+
var max_num_types uint
|
|
294
|
+
assert(num_contexts <= maxStaticContexts)
|
|
295
|
+
|
|
296
|
+
self.alphabet_size_ = alphabet_size
|
|
297
|
+
self.num_contexts_ = num_contexts
|
|
298
|
+
self.max_block_types_ = maxNumberOfBlockTypes / num_contexts
|
|
299
|
+
self.min_block_size_ = min_block_size
|
|
300
|
+
self.split_threshold_ = split_threshold
|
|
301
|
+
self.num_blocks_ = 0
|
|
302
|
+
self.split_ = split
|
|
303
|
+
self.histograms_size_ = histograms_size
|
|
304
|
+
self.target_block_size_ = min_block_size
|
|
305
|
+
self.block_size_ = 0
|
|
306
|
+
self.curr_histogram_ix_ = 0
|
|
307
|
+
self.merge_last_count_ = 0
|
|
308
|
+
|
|
309
|
+
/* We have to allocate one more histogram than the maximum number of block
|
|
310
|
+
types for the current histogram when the meta-block is too big. */
|
|
311
|
+
max_num_types = brotli_min_size_t(max_num_blocks, self.max_block_types_+1)
|
|
312
|
+
|
|
313
|
+
brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, max_num_blocks)
|
|
314
|
+
brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, max_num_blocks)
|
|
315
|
+
split.num_blocks = max_num_blocks
|
|
316
|
+
*histograms_size = max_num_types * num_contexts
|
|
317
|
+
if histograms == nil || cap(*histograms) < int(*histograms_size) {
|
|
318
|
+
*histograms = make([]histogramLiteral, (*histograms_size))
|
|
319
|
+
} else {
|
|
320
|
+
*histograms = (*histograms)[:*histograms_size]
|
|
321
|
+
}
|
|
322
|
+
self.histograms_ = *histograms
|
|
323
|
+
|
|
324
|
+
/* Clear only current histogram. */
|
|
325
|
+
clearHistogramsLiteral(self.histograms_[0:], num_contexts)
|
|
326
|
+
|
|
327
|
+
self.last_histogram_ix_[1] = 0
|
|
328
|
+
self.last_histogram_ix_[0] = self.last_histogram_ix_[1]
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
/* Does either of three things:
|
|
332
|
+
(1) emits the current block with a new block type;
|
|
333
|
+
(2) emits the current block with the type of the second last block;
|
|
334
|
+
(3) merges the current block with the last block. */
|
|
335
|
+
func contextBlockSplitterFinishBlock(self *contextBlockSplitter, is_final bool) {
|
|
336
|
+
var split *blockSplit = self.split_
|
|
337
|
+
var num_contexts uint = self.num_contexts_
|
|
338
|
+
var last_entropy []float64 = self.last_entropy_[:]
|
|
339
|
+
var histograms []histogramLiteral = self.histograms_
|
|
340
|
+
|
|
341
|
+
if self.block_size_ < self.min_block_size_ {
|
|
342
|
+
self.block_size_ = self.min_block_size_
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
if self.num_blocks_ == 0 {
|
|
346
|
+
var i uint
|
|
347
|
+
|
|
348
|
+
/* Create first block. */
|
|
349
|
+
split.lengths[0] = uint32(self.block_size_)
|
|
350
|
+
|
|
351
|
+
split.types[0] = 0
|
|
352
|
+
|
|
353
|
+
for i = 0; i < num_contexts; i++ {
|
|
354
|
+
last_entropy[i] = bitsEntropy(histograms[i].data_[:], self.alphabet_size_)
|
|
355
|
+
last_entropy[num_contexts+i] = last_entropy[i]
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
self.num_blocks_++
|
|
359
|
+
split.num_types++
|
|
360
|
+
self.curr_histogram_ix_ += num_contexts
|
|
361
|
+
if self.curr_histogram_ix_ < *self.histograms_size_ {
|
|
362
|
+
clearHistogramsLiteral(self.histograms_[self.curr_histogram_ix_:], self.num_contexts_)
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
self.block_size_ = 0
|
|
366
|
+
} else if self.block_size_ > 0 {
|
|
367
|
+
var entropy [maxStaticContexts]float64
|
|
368
|
+
var combined_histo []histogramLiteral = make([]histogramLiteral, (2 * num_contexts))
|
|
369
|
+
var combined_entropy [2 * maxStaticContexts]float64
|
|
370
|
+
var diff = [2]float64{0.0}
|
|
371
|
+
/* Try merging the set of histograms for the current block type with the
|
|
372
|
+
respective set of histograms for the last and second last block types.
|
|
373
|
+
Decide over the split based on the total reduction of entropy across
|
|
374
|
+
all contexts. */
|
|
375
|
+
|
|
376
|
+
var i uint
|
|
377
|
+
for i = 0; i < num_contexts; i++ {
|
|
378
|
+
var curr_histo_ix uint = self.curr_histogram_ix_ + i
|
|
379
|
+
var j uint
|
|
380
|
+
entropy[i] = bitsEntropy(histograms[curr_histo_ix].data_[:], self.alphabet_size_)
|
|
381
|
+
for j = 0; j < 2; j++ {
|
|
382
|
+
var jx uint = j*num_contexts + i
|
|
383
|
+
var last_histogram_ix uint = self.last_histogram_ix_[j] + i
|
|
384
|
+
combined_histo[jx] = histograms[curr_histo_ix]
|
|
385
|
+
histogramAddHistogramLiteral(&combined_histo[jx], &histograms[last_histogram_ix])
|
|
386
|
+
combined_entropy[jx] = bitsEntropy(combined_histo[jx].data_[0:], self.alphabet_size_)
|
|
387
|
+
diff[j] += combined_entropy[jx] - entropy[i] - last_entropy[jx]
|
|
388
|
+
}
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
if split.num_types < self.max_block_types_ && diff[0] > self.split_threshold_ && diff[1] > self.split_threshold_ {
|
|
392
|
+
/* Create new block. */
|
|
393
|
+
split.lengths[self.num_blocks_] = uint32(self.block_size_)
|
|
394
|
+
|
|
395
|
+
split.types[self.num_blocks_] = byte(split.num_types)
|
|
396
|
+
self.last_histogram_ix_[1] = self.last_histogram_ix_[0]
|
|
397
|
+
self.last_histogram_ix_[0] = split.num_types * num_contexts
|
|
398
|
+
for i = 0; i < num_contexts; i++ {
|
|
399
|
+
last_entropy[num_contexts+i] = last_entropy[i]
|
|
400
|
+
last_entropy[i] = entropy[i]
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
self.num_blocks_++
|
|
404
|
+
split.num_types++
|
|
405
|
+
self.curr_histogram_ix_ += num_contexts
|
|
406
|
+
if self.curr_histogram_ix_ < *self.histograms_size_ {
|
|
407
|
+
clearHistogramsLiteral(self.histograms_[self.curr_histogram_ix_:], self.num_contexts_)
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
self.block_size_ = 0
|
|
411
|
+
self.merge_last_count_ = 0
|
|
412
|
+
self.target_block_size_ = self.min_block_size_
|
|
413
|
+
} else if diff[1] < diff[0]-20.0 {
|
|
414
|
+
split.lengths[self.num_blocks_] = uint32(self.block_size_)
|
|
415
|
+
split.types[self.num_blocks_] = split.types[self.num_blocks_-2]
|
|
416
|
+
/* Combine this block with second last block. */
|
|
417
|
+
|
|
418
|
+
var tmp uint = self.last_histogram_ix_[0]
|
|
419
|
+
self.last_histogram_ix_[0] = self.last_histogram_ix_[1]
|
|
420
|
+
self.last_histogram_ix_[1] = tmp
|
|
421
|
+
for i = 0; i < num_contexts; i++ {
|
|
422
|
+
histograms[self.last_histogram_ix_[0]+i] = combined_histo[num_contexts+i]
|
|
423
|
+
last_entropy[num_contexts+i] = last_entropy[i]
|
|
424
|
+
last_entropy[i] = combined_entropy[num_contexts+i]
|
|
425
|
+
histogramClearLiteral(&histograms[self.curr_histogram_ix_+i])
|
|
426
|
+
}
|
|
427
|
+
|
|
428
|
+
self.num_blocks_++
|
|
429
|
+
self.block_size_ = 0
|
|
430
|
+
self.merge_last_count_ = 0
|
|
431
|
+
self.target_block_size_ = self.min_block_size_
|
|
432
|
+
} else {
|
|
433
|
+
/* Combine this block with last block. */
|
|
434
|
+
split.lengths[self.num_blocks_-1] += uint32(self.block_size_)
|
|
435
|
+
|
|
436
|
+
for i = 0; i < num_contexts; i++ {
|
|
437
|
+
histograms[self.last_histogram_ix_[0]+i] = combined_histo[i]
|
|
438
|
+
last_entropy[i] = combined_entropy[i]
|
|
439
|
+
if split.num_types == 1 {
|
|
440
|
+
last_entropy[num_contexts+i] = last_entropy[i]
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
histogramClearLiteral(&histograms[self.curr_histogram_ix_+i])
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
self.block_size_ = 0
|
|
447
|
+
self.merge_last_count_++
|
|
448
|
+
if self.merge_last_count_ > 1 {
|
|
449
|
+
self.target_block_size_ += self.min_block_size_
|
|
450
|
+
}
|
|
451
|
+
}
|
|
452
|
+
|
|
453
|
+
combined_histo = nil
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
if is_final {
|
|
457
|
+
*self.histograms_size_ = split.num_types * num_contexts
|
|
458
|
+
split.num_blocks = self.num_blocks_
|
|
459
|
+
}
|
|
460
|
+
}
|
|
461
|
+
|
|
462
|
+
/* Adds the next symbol to the current block type and context. When the
|
|
463
|
+
current block reaches the target size, decides on merging the block. */
|
|
464
|
+
func contextBlockSplitterAddSymbol(self *contextBlockSplitter, symbol uint, context uint) {
|
|
465
|
+
histogramAddLiteral(&self.histograms_[self.curr_histogram_ix_+context], symbol)
|
|
466
|
+
self.block_size_++
|
|
467
|
+
if self.block_size_ == self.target_block_size_ {
|
|
468
|
+
contextBlockSplitterFinishBlock(self, false) /* is_final = */
|
|
469
|
+
}
|
|
470
|
+
}
|
|
471
|
+
|
|
472
|
+
func mapStaticContexts(num_contexts uint, static_context_map []uint32, mb *metaBlockSplit) {
|
|
473
|
+
var i uint
|
|
474
|
+
mb.literal_context_map_size = mb.literal_split.num_types << literalContextBits
|
|
475
|
+
if cap(mb.literal_context_map) < int(mb.literal_context_map_size) {
|
|
476
|
+
mb.literal_context_map = make([]uint32, (mb.literal_context_map_size))
|
|
477
|
+
} else {
|
|
478
|
+
mb.literal_context_map = mb.literal_context_map[:mb.literal_context_map_size]
|
|
479
|
+
}
|
|
480
|
+
|
|
481
|
+
for i = 0; i < mb.literal_split.num_types; i++ {
|
|
482
|
+
var offset uint32 = uint32(i * num_contexts)
|
|
483
|
+
var j uint
|
|
484
|
+
for j = 0; j < 1<<literalContextBits; j++ {
|
|
485
|
+
mb.literal_context_map[(i<<literalContextBits)+j] = offset + static_context_map[j]
|
|
486
|
+
}
|
|
487
|
+
}
|
|
488
|
+
}
|
|
489
|
+
|
|
490
|
+
func buildMetaBlockGreedyInternal(ringbuffer []byte, pos uint, mask uint, prev_byte byte, prev_byte2 byte, literal_context_lut contextLUT, num_contexts uint, static_context_map []uint32, commands []command, mb *metaBlockSplit) {
|
|
491
|
+
var lit_blocks struct {
|
|
492
|
+
plain blockSplitterLiteral
|
|
493
|
+
ctx contextBlockSplitter
|
|
494
|
+
}
|
|
495
|
+
var cmd_blocks blockSplitterCommand
|
|
496
|
+
var dist_blocks blockSplitterDistance
|
|
497
|
+
var num_literals uint = 0
|
|
498
|
+
for i := range commands {
|
|
499
|
+
num_literals += uint(commands[i].insert_len_)
|
|
500
|
+
}
|
|
501
|
+
|
|
502
|
+
if num_contexts == 1 {
|
|
503
|
+
initBlockSplitterLiteral(&lit_blocks.plain, 256, 512, 400.0, num_literals, &mb.literal_split, &mb.literal_histograms, &mb.literal_histograms_size)
|
|
504
|
+
} else {
|
|
505
|
+
initContextBlockSplitter(&lit_blocks.ctx, 256, num_contexts, 512, 400.0, num_literals, &mb.literal_split, &mb.literal_histograms, &mb.literal_histograms_size)
|
|
506
|
+
}
|
|
507
|
+
|
|
508
|
+
initBlockSplitterCommand(&cmd_blocks, numCommandSymbols, 1024, 500.0, uint(len(commands)), &mb.command_split, &mb.command_histograms, &mb.command_histograms_size)
|
|
509
|
+
initBlockSplitterDistance(&dist_blocks, 64, 512, 100.0, uint(len(commands)), &mb.distance_split, &mb.distance_histograms, &mb.distance_histograms_size)
|
|
510
|
+
|
|
511
|
+
for _, cmd := range commands {
|
|
512
|
+
var j uint
|
|
513
|
+
blockSplitterAddSymbolCommand(&cmd_blocks, uint(cmd.cmd_prefix_))
|
|
514
|
+
for j = uint(cmd.insert_len_); j != 0; j-- {
|
|
515
|
+
var literal byte = ringbuffer[pos&mask]
|
|
516
|
+
if num_contexts == 1 {
|
|
517
|
+
blockSplitterAddSymbolLiteral(&lit_blocks.plain, uint(literal))
|
|
518
|
+
} else {
|
|
519
|
+
var context uint = uint(getContext(prev_byte, prev_byte2, literal_context_lut))
|
|
520
|
+
contextBlockSplitterAddSymbol(&lit_blocks.ctx, uint(literal), uint(static_context_map[context]))
|
|
521
|
+
}
|
|
522
|
+
|
|
523
|
+
prev_byte2 = prev_byte
|
|
524
|
+
prev_byte = literal
|
|
525
|
+
pos++
|
|
526
|
+
}
|
|
527
|
+
|
|
528
|
+
pos += uint(commandCopyLen(&cmd))
|
|
529
|
+
if commandCopyLen(&cmd) != 0 {
|
|
530
|
+
prev_byte2 = ringbuffer[(pos-2)&mask]
|
|
531
|
+
prev_byte = ringbuffer[(pos-1)&mask]
|
|
532
|
+
if cmd.cmd_prefix_ >= 128 {
|
|
533
|
+
blockSplitterAddSymbolDistance(&dist_blocks, uint(cmd.dist_prefix_)&0x3FF)
|
|
534
|
+
}
|
|
535
|
+
}
|
|
536
|
+
}
|
|
537
|
+
|
|
538
|
+
if num_contexts == 1 {
|
|
539
|
+
blockSplitterFinishBlockLiteral(&lit_blocks.plain, true) /* is_final = */
|
|
540
|
+
} else {
|
|
541
|
+
contextBlockSplitterFinishBlock(&lit_blocks.ctx, true) /* is_final = */
|
|
542
|
+
}
|
|
543
|
+
|
|
544
|
+
blockSplitterFinishBlockCommand(&cmd_blocks, true) /* is_final = */
|
|
545
|
+
blockSplitterFinishBlockDistance(&dist_blocks, true) /* is_final = */
|
|
546
|
+
|
|
547
|
+
if num_contexts > 1 {
|
|
548
|
+
mapStaticContexts(num_contexts, static_context_map, mb)
|
|
549
|
+
}
|
|
550
|
+
}
|
|
551
|
+
|
|
552
|
+
func buildMetaBlockGreedy(ringbuffer []byte, pos uint, mask uint, prev_byte byte, prev_byte2 byte, literal_context_lut contextLUT, num_contexts uint, static_context_map []uint32, commands []command, mb *metaBlockSplit) {
|
|
553
|
+
if num_contexts == 1 {
|
|
554
|
+
buildMetaBlockGreedyInternal(ringbuffer, pos, mask, prev_byte, prev_byte2, literal_context_lut, 1, nil, commands, mb)
|
|
555
|
+
} else {
|
|
556
|
+
buildMetaBlockGreedyInternal(ringbuffer, pos, mask, prev_byte, prev_byte2, literal_context_lut, num_contexts, static_context_map, commands, mb)
|
|
557
|
+
}
|
|
558
|
+
}
|
|
559
|
+
|
|
560
|
+
func optimizeHistograms(num_distance_codes uint32, mb *metaBlockSplit) {
|
|
561
|
+
var good_for_rle [numCommandSymbols]byte
|
|
562
|
+
var i uint
|
|
563
|
+
for i = 0; i < mb.literal_histograms_size; i++ {
|
|
564
|
+
optimizeHuffmanCountsForRLE(256, mb.literal_histograms[i].data_[:], good_for_rle[:])
|
|
565
|
+
}
|
|
566
|
+
|
|
567
|
+
for i = 0; i < mb.command_histograms_size; i++ {
|
|
568
|
+
optimizeHuffmanCountsForRLE(numCommandSymbols, mb.command_histograms[i].data_[:], good_for_rle[:])
|
|
569
|
+
}
|
|
570
|
+
|
|
571
|
+
for i = 0; i < mb.distance_histograms_size; i++ {
|
|
572
|
+
optimizeHuffmanCountsForRLE(uint(num_distance_codes), mb.distance_histograms[i].data_[:], good_for_rle[:])
|
|
573
|
+
}
|
|
574
|
+
}
|