typenative 0.3.1 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/ci.yml +2 -0
- package/CHANGELOG.md +258 -0
- package/CONTRIBUTION.md +1 -1
- package/README.md +17 -1
- package/bin/cli.js +26 -11
- package/bin/go/bridge.js +74 -14
- package/bin/go/convert.js +59 -1
- package/bin/go/metadata.js +13 -4
- package/bin/go/overrides.js +4 -6
- package/bin/go/stdlib.generated.js +2 -0
- package/bin/index.js +338 -47
- package/bin/main.js +36 -14
- package/bin/node/assert.js +2 -2
- package/bin/node/buffer.js +9 -0
- package/bin/node/child_process.js +51 -3
- package/bin/node/cluster.js +41 -3
- package/bin/node/core.js +1 -1
- package/bin/node/crypto.js +205 -4
- package/bin/node/dgram.js +230 -6
- package/bin/node/dns.js +471 -8
- package/bin/node/errno_tables.js +445 -0
- package/bin/node/events.js +1 -1
- package/bin/node/fs.js +22 -15
- package/bin/node/fs_watch.js +407 -0
- package/bin/node/http.js +128 -30
- package/bin/node/http2.js +37 -7
- package/bin/node/module_v8.js +142 -0
- package/bin/node/net.js +9 -6
- package/bin/node/process.js +242 -0
- package/bin/node/readline.js +272 -1
- package/bin/node/registry.js +3 -1
- package/bin/node/sqlite.js +60 -12
- package/bin/node/stream.js +145 -10
- package/bin/node/test.js +311 -19
- package/bin/node/timers.js +1 -1
- package/bin/node/typed_arrays.js +608 -0
- package/bin/node/url.js +23 -5
- package/bin/node/util.js +21 -1
- package/bin/node/util_extra.js +6 -30
- package/bin/node/worker_threads.js +79 -19
- package/bin/node/zlib.js +131 -16
- package/bin/transpiler.js +4729 -616
- package/bin/typecheck-node.js +1 -0
- package/bin/version.js +1 -1
- package/conformance.md +1030 -877
- package/examples/README.md +54 -0
- package/examples/fibonacci/README.md +89 -0
- package/examples/fibonacci/bench.mjs +87 -0
- package/examples/fibonacci/main.ts +21 -0
- package/examples/hello-world/README.md +62 -0
- package/examples/hello-world/bench.mjs +66 -0
- package/examples/hello-world/main.ts +7 -0
- package/examples/json-server/README.md +127 -0
- package/examples/json-server/bench.mjs +263 -0
- package/examples/json-server/main.ts +67 -0
- package/govendor/github.com/andybalholm/brotli/LICENSE +19 -0
- package/govendor/github.com/andybalholm/brotli/README.md +19 -0
- package/govendor/github.com/andybalholm/brotli/backward_references.go +185 -0
- package/govendor/github.com/andybalholm/brotli/backward_references_hq.go +796 -0
- package/govendor/github.com/andybalholm/brotli/bit_cost.go +436 -0
- package/govendor/github.com/andybalholm/brotli/bit_reader.go +266 -0
- package/govendor/github.com/andybalholm/brotli/bitwriter.go +56 -0
- package/govendor/github.com/andybalholm/brotli/block_splitter.go +144 -0
- package/govendor/github.com/andybalholm/brotli/block_splitter_command.go +434 -0
- package/govendor/github.com/andybalholm/brotli/block_splitter_distance.go +433 -0
- package/govendor/github.com/andybalholm/brotli/block_splitter_literal.go +433 -0
- package/govendor/github.com/andybalholm/brotli/brotli_bit_stream.go +1435 -0
- package/govendor/github.com/andybalholm/brotli/cluster.go +30 -0
- package/govendor/github.com/andybalholm/brotli/cluster_command.go +164 -0
- package/govendor/github.com/andybalholm/brotli/cluster_distance.go +326 -0
- package/govendor/github.com/andybalholm/brotli/cluster_literal.go +326 -0
- package/govendor/github.com/andybalholm/brotli/command.go +254 -0
- package/govendor/github.com/andybalholm/brotli/compress_fragment.go +834 -0
- package/govendor/github.com/andybalholm/brotli/compress_fragment_two_pass.go +775 -0
- package/govendor/github.com/andybalholm/brotli/constants.go +77 -0
- package/govendor/github.com/andybalholm/brotli/context.go +2176 -0
- package/govendor/github.com/andybalholm/brotli/decode.go +2570 -0
- package/govendor/github.com/andybalholm/brotli/dictionary.go +122890 -0
- package/govendor/github.com/andybalholm/brotli/dictionary_hash.go +32779 -0
- package/govendor/github.com/andybalholm/brotli/encode.go +1220 -0
- package/govendor/github.com/andybalholm/brotli/encoder.go +177 -0
- package/govendor/github.com/andybalholm/brotli/encoder_dict.go +22 -0
- package/govendor/github.com/andybalholm/brotli/encoder_fast.go +201 -0
- package/govendor/github.com/andybalholm/brotli/entropy_encode.go +592 -0
- package/govendor/github.com/andybalholm/brotli/entropy_encode_static.go +4399 -0
- package/govendor/github.com/andybalholm/brotli/fast_log.go +290 -0
- package/govendor/github.com/andybalholm/brotli/find_match_length.go +45 -0
- package/govendor/github.com/andybalholm/brotli/go.mod +3 -0
- package/govendor/github.com/andybalholm/brotli/h10.go +287 -0
- package/govendor/github.com/andybalholm/brotli/h5.go +214 -0
- package/govendor/github.com/andybalholm/brotli/h6.go +216 -0
- package/govendor/github.com/andybalholm/brotli/hash.go +342 -0
- package/govendor/github.com/andybalholm/brotli/hash_composite.go +93 -0
- package/govendor/github.com/andybalholm/brotli/hash_forgetful_chain.go +252 -0
- package/govendor/github.com/andybalholm/brotli/hash_longest_match_quickly.go +214 -0
- package/govendor/github.com/andybalholm/brotli/hash_rolling.go +168 -0
- package/govendor/github.com/andybalholm/brotli/histogram.go +226 -0
- package/govendor/github.com/andybalholm/brotli/http.go +201 -0
- package/govendor/github.com/andybalholm/brotli/huffman.go +653 -0
- package/govendor/github.com/andybalholm/brotli/literal_cost.go +182 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/bargain1.go +212 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/bargain2.go +248 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/bargain3.go +278 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/emitter.go +34 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/m0.go +175 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/m4.go +339 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/matchfinder.go +103 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/pathfinder.go +328 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/textencoder.go +53 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/trio.go +327 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/zdfast.go +258 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/zfast.go +206 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/zm.go +302 -0
- package/govendor/github.com/andybalholm/brotli/memory.go +66 -0
- package/govendor/github.com/andybalholm/brotli/metablock.go +574 -0
- package/govendor/github.com/andybalholm/brotli/metablock_command.go +165 -0
- package/govendor/github.com/andybalholm/brotli/metablock_distance.go +165 -0
- package/govendor/github.com/andybalholm/brotli/metablock_literal.go +165 -0
- package/govendor/github.com/andybalholm/brotli/params.go +37 -0
- package/govendor/github.com/andybalholm/brotli/platform.go +103 -0
- package/govendor/github.com/andybalholm/brotli/prefix.go +30 -0
- package/govendor/github.com/andybalholm/brotli/prefix_dec.go +723 -0
- package/govendor/github.com/andybalholm/brotli/quality.go +196 -0
- package/govendor/github.com/andybalholm/brotli/reader.go +127 -0
- package/govendor/github.com/andybalholm/brotli/ringbuffer.go +134 -0
- package/govendor/github.com/andybalholm/brotli/state.go +313 -0
- package/govendor/github.com/andybalholm/brotli/static_dict.go +662 -0
- package/govendor/github.com/andybalholm/brotli/static_dict_lut.go +75094 -0
- package/govendor/github.com/andybalholm/brotli/symbol_list.go +22 -0
- package/govendor/github.com/andybalholm/brotli/transform.go +641 -0
- package/govendor/github.com/andybalholm/brotli/utf8_util.go +70 -0
- package/govendor/github.com/andybalholm/brotli/util.go +7 -0
- package/govendor/github.com/andybalholm/brotli/write_bits.go +52 -0
- package/govendor/github.com/andybalholm/brotli/writer.go +162 -0
- package/govendor/github.com/fsnotify/fsnotify/.mailmap +2 -0
- package/govendor/github.com/fsnotify/fsnotify/CHANGELOG.md +651 -0
- package/govendor/github.com/fsnotify/fsnotify/CONTRIBUTING.md +147 -0
- package/govendor/github.com/fsnotify/fsnotify/LICENSE +25 -0
- package/govendor/github.com/fsnotify/fsnotify/README.md +214 -0
- package/govendor/github.com/fsnotify/fsnotify/backend_fen.go +469 -0
- package/govendor/github.com/fsnotify/fsnotify/backend_inotify.go +597 -0
- package/govendor/github.com/fsnotify/fsnotify/backend_kqueue.go +735 -0
- package/govendor/github.com/fsnotify/fsnotify/backend_other.go +22 -0
- package/govendor/github.com/fsnotify/fsnotify/backend_windows.go +702 -0
- package/govendor/github.com/fsnotify/fsnotify/fsnotify.go +526 -0
- package/govendor/github.com/fsnotify/fsnotify/go.mod +3 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/darwin.go +20 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_darwin.go +15 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_dragonfly.go +15 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_freebsd.go +20 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_kqueue.go +32 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_linux.go +56 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_netbsd.go +15 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_openbsd.go +16 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_solaris.go +45 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_windows.go +40 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/freebsd.go +20 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/internal.go +2 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/unix.go +20 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/unix2.go +25 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/windows.go +40 -0
- package/govendor/github.com/fsnotify/fsnotify/shared.go +64 -0
- package/govendor/github.com/fsnotify/fsnotify/staticcheck.conf +3 -0
- package/govendor/github.com/fsnotify/fsnotify/system_bsd.go +7 -0
- package/govendor/github.com/fsnotify/fsnotify/system_darwin.go +8 -0
- package/govendor/manifest.txt +3 -1
- package/package.json +1 -1
- package/types/typenative-go.d.ts +23 -0
- package/types/typenative-npm.d.ts +241 -5
- package/types/typenative.d.ts +302 -12
|
@@ -0,0 +1,796 @@
|
|
|
1
|
+
package brotli
|
|
2
|
+
|
|
3
|
+
import "math"
|
|
4
|
+
|
|
5
|
+
type zopfliNode struct {
|
|
6
|
+
length uint32
|
|
7
|
+
distance uint32
|
|
8
|
+
dcode_insert_length uint32
|
|
9
|
+
u struct {
|
|
10
|
+
cost float32
|
|
11
|
+
next uint32
|
|
12
|
+
shortcut uint32
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
const maxEffectiveDistanceAlphabetSize = 544
|
|
17
|
+
|
|
18
|
+
const kInfinity float32 = 1.7e38 /* ~= 2 ^ 127 */
|
|
19
|
+
|
|
20
|
+
var kDistanceCacheIndex = []uint32{0, 1, 2, 3, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1}
|
|
21
|
+
|
|
22
|
+
var kDistanceCacheOffset = []int{0, 0, 0, 0, -1, 1, -2, 2, -3, 3, -1, 1, -2, 2, -3, 3}
|
|
23
|
+
|
|
24
|
+
func initZopfliNodes(array []zopfliNode, length uint) {
|
|
25
|
+
var stub zopfliNode
|
|
26
|
+
var i uint
|
|
27
|
+
stub.length = 1
|
|
28
|
+
stub.distance = 0
|
|
29
|
+
stub.dcode_insert_length = 0
|
|
30
|
+
stub.u.cost = kInfinity
|
|
31
|
+
for i = 0; i < length; i++ {
|
|
32
|
+
array[i] = stub
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
func zopfliNodeCopyLength(self *zopfliNode) uint32 {
|
|
37
|
+
return self.length & 0x1FFFFFF
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
func zopfliNodeLengthCode(self *zopfliNode) uint32 {
|
|
41
|
+
var modifier uint32 = self.length >> 25
|
|
42
|
+
return zopfliNodeCopyLength(self) + 9 - modifier
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
func zopfliNodeCopyDistance(self *zopfliNode) uint32 {
|
|
46
|
+
return self.distance
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
func zopfliNodeDistanceCode(self *zopfliNode) uint32 {
|
|
50
|
+
var short_code uint32 = self.dcode_insert_length >> 27
|
|
51
|
+
if short_code == 0 {
|
|
52
|
+
return zopfliNodeCopyDistance(self) + numDistanceShortCodes - 1
|
|
53
|
+
} else {
|
|
54
|
+
return short_code - 1
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
func zopfliNodeCommandLength(self *zopfliNode) uint32 {
|
|
59
|
+
return zopfliNodeCopyLength(self) + (self.dcode_insert_length & 0x7FFFFFF)
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/* Histogram based cost model for zopflification. */
|
|
63
|
+
type zopfliCostModel struct {
|
|
64
|
+
cost_cmd_ [numCommandSymbols]float32
|
|
65
|
+
cost_dist_ []float32
|
|
66
|
+
distance_histogram_size uint32
|
|
67
|
+
literal_costs_ []float32
|
|
68
|
+
min_cost_cmd_ float32
|
|
69
|
+
num_bytes_ uint
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
func initZopfliCostModel(self *zopfliCostModel, dist *distanceParams, num_bytes uint) {
|
|
73
|
+
var distance_histogram_size uint32 = dist.alphabet_size
|
|
74
|
+
if distance_histogram_size > maxEffectiveDistanceAlphabetSize {
|
|
75
|
+
distance_histogram_size = maxEffectiveDistanceAlphabetSize
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
self.num_bytes_ = num_bytes
|
|
79
|
+
self.literal_costs_ = make([]float32, (num_bytes + 2))
|
|
80
|
+
self.cost_dist_ = make([]float32, (dist.alphabet_size))
|
|
81
|
+
self.distance_histogram_size = distance_histogram_size
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
func cleanupZopfliCostModel(self *zopfliCostModel) {
|
|
85
|
+
self.literal_costs_ = nil
|
|
86
|
+
self.cost_dist_ = nil
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
func setCost(histogram []uint32, histogram_size uint, literal_histogram bool, cost []float32) {
|
|
90
|
+
var sum uint = 0
|
|
91
|
+
var missing_symbol_sum uint
|
|
92
|
+
var log2sum float32
|
|
93
|
+
var missing_symbol_cost float32
|
|
94
|
+
var i uint
|
|
95
|
+
for i = 0; i < histogram_size; i++ {
|
|
96
|
+
sum += uint(histogram[i])
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
log2sum = float32(fastLog2(sum))
|
|
100
|
+
missing_symbol_sum = sum
|
|
101
|
+
if !literal_histogram {
|
|
102
|
+
for i = 0; i < histogram_size; i++ {
|
|
103
|
+
if histogram[i] == 0 {
|
|
104
|
+
missing_symbol_sum++
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
missing_symbol_cost = float32(fastLog2(missing_symbol_sum)) + 2
|
|
110
|
+
for i = 0; i < histogram_size; i++ {
|
|
111
|
+
if histogram[i] == 0 {
|
|
112
|
+
cost[i] = missing_symbol_cost
|
|
113
|
+
continue
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/* Shannon bits for this symbol. */
|
|
117
|
+
cost[i] = log2sum - float32(fastLog2(uint(histogram[i])))
|
|
118
|
+
|
|
119
|
+
/* Cannot be coded with less than 1 bit */
|
|
120
|
+
if cost[i] < 1 {
|
|
121
|
+
cost[i] = 1
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
func zopfliCostModelSetFromCommands(self *zopfliCostModel, position uint, ringbuffer []byte, ringbuffer_mask uint, commands []command, last_insert_len uint) {
|
|
127
|
+
var histogram_literal [numLiteralSymbols]uint32
|
|
128
|
+
var histogram_cmd [numCommandSymbols]uint32
|
|
129
|
+
var histogram_dist [maxEffectiveDistanceAlphabetSize]uint32
|
|
130
|
+
var cost_literal [numLiteralSymbols]float32
|
|
131
|
+
var pos uint = position - last_insert_len
|
|
132
|
+
var min_cost_cmd float32 = kInfinity
|
|
133
|
+
var cost_cmd []float32 = self.cost_cmd_[:]
|
|
134
|
+
var literal_costs []float32
|
|
135
|
+
|
|
136
|
+
histogram_literal = [numLiteralSymbols]uint32{}
|
|
137
|
+
histogram_cmd = [numCommandSymbols]uint32{}
|
|
138
|
+
histogram_dist = [maxEffectiveDistanceAlphabetSize]uint32{}
|
|
139
|
+
|
|
140
|
+
for i := range commands {
|
|
141
|
+
var inslength uint = uint(commands[i].insert_len_)
|
|
142
|
+
var copylength uint = uint(commandCopyLen(&commands[i]))
|
|
143
|
+
var distcode uint = uint(commands[i].dist_prefix_) & 0x3FF
|
|
144
|
+
var cmdcode uint = uint(commands[i].cmd_prefix_)
|
|
145
|
+
var j uint
|
|
146
|
+
|
|
147
|
+
histogram_cmd[cmdcode]++
|
|
148
|
+
if cmdcode >= 128 {
|
|
149
|
+
histogram_dist[distcode]++
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
for j = 0; j < inslength; j++ {
|
|
153
|
+
histogram_literal[ringbuffer[(pos+j)&ringbuffer_mask]]++
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
pos += inslength + copylength
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
setCost(histogram_literal[:], numLiteralSymbols, true, cost_literal[:])
|
|
160
|
+
setCost(histogram_cmd[:], numCommandSymbols, false, cost_cmd)
|
|
161
|
+
setCost(histogram_dist[:], uint(self.distance_histogram_size), false, self.cost_dist_)
|
|
162
|
+
|
|
163
|
+
for i := 0; i < numCommandSymbols; i++ {
|
|
164
|
+
min_cost_cmd = brotli_min_float(min_cost_cmd, cost_cmd[i])
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
self.min_cost_cmd_ = min_cost_cmd
|
|
168
|
+
{
|
|
169
|
+
literal_costs = self.literal_costs_
|
|
170
|
+
var literal_carry float32 = 0.0
|
|
171
|
+
num_bytes := int(self.num_bytes_)
|
|
172
|
+
literal_costs[0] = 0.0
|
|
173
|
+
for i := 0; i < num_bytes; i++ {
|
|
174
|
+
literal_carry += cost_literal[ringbuffer[(position+uint(i))&ringbuffer_mask]]
|
|
175
|
+
literal_costs[i+1] = literal_costs[i] + literal_carry
|
|
176
|
+
literal_carry -= literal_costs[i+1] - literal_costs[i]
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
func zopfliCostModelSetFromLiteralCosts(self *zopfliCostModel, position uint, ringbuffer []byte, ringbuffer_mask uint) {
|
|
182
|
+
var literal_costs []float32 = self.literal_costs_
|
|
183
|
+
var literal_carry float32 = 0.0
|
|
184
|
+
var cost_dist []float32 = self.cost_dist_
|
|
185
|
+
var cost_cmd []float32 = self.cost_cmd_[:]
|
|
186
|
+
var num_bytes uint = self.num_bytes_
|
|
187
|
+
var i uint
|
|
188
|
+
estimateBitCostsForLiterals(position, num_bytes, ringbuffer_mask, ringbuffer, literal_costs[1:])
|
|
189
|
+
literal_costs[0] = 0.0
|
|
190
|
+
for i = 0; i < num_bytes; i++ {
|
|
191
|
+
literal_carry += literal_costs[i+1]
|
|
192
|
+
literal_costs[i+1] = literal_costs[i] + literal_carry
|
|
193
|
+
literal_carry -= literal_costs[i+1] - literal_costs[i]
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
for i = 0; i < numCommandSymbols; i++ {
|
|
197
|
+
cost_cmd[i] = float32(fastLog2(uint(11 + uint32(i))))
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
for i = 0; uint32(i) < self.distance_histogram_size; i++ {
|
|
201
|
+
cost_dist[i] = float32(fastLog2(uint(20 + uint32(i))))
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
self.min_cost_cmd_ = float32(fastLog2(11))
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
func zopfliCostModelGetCommandCost(self *zopfliCostModel, cmdcode uint16) float32 {
|
|
208
|
+
return self.cost_cmd_[cmdcode]
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
func zopfliCostModelGetDistanceCost(self *zopfliCostModel, distcode uint) float32 {
|
|
212
|
+
return self.cost_dist_[distcode]
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
func zopfliCostModelGetLiteralCosts(self *zopfliCostModel, from uint, to uint) float32 {
|
|
216
|
+
return self.literal_costs_[to] - self.literal_costs_[from]
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
func zopfliCostModelGetMinCostCmd(self *zopfliCostModel) float32 {
|
|
220
|
+
return self.min_cost_cmd_
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
/* REQUIRES: len >= 2, start_pos <= pos */
|
|
224
|
+
/* REQUIRES: cost < kInfinity, nodes[start_pos].cost < kInfinity */
|
|
225
|
+
/* Maintains the "ZopfliNode array invariant". */
|
|
226
|
+
func updateZopfliNode(nodes []zopfliNode, pos uint, start_pos uint, len uint, len_code uint, dist uint, short_code uint, cost float32) {
|
|
227
|
+
var next *zopfliNode = &nodes[pos+len]
|
|
228
|
+
next.length = uint32(len | (len+9-len_code)<<25)
|
|
229
|
+
next.distance = uint32(dist)
|
|
230
|
+
next.dcode_insert_length = uint32(short_code<<27 | (pos - start_pos))
|
|
231
|
+
next.u.cost = cost
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
type posData struct {
|
|
235
|
+
pos uint
|
|
236
|
+
distance_cache [4]int
|
|
237
|
+
costdiff float32
|
|
238
|
+
cost float32
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
/* Maintains the smallest 8 cost difference together with their positions */
|
|
242
|
+
type startPosQueue struct {
|
|
243
|
+
q_ [8]posData
|
|
244
|
+
idx_ uint
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
func initStartPosQueue(self *startPosQueue) {
|
|
248
|
+
self.idx_ = 0
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
func startPosQueueSize(self *startPosQueue) uint {
|
|
252
|
+
return brotli_min_size_t(self.idx_, 8)
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
func startPosQueuePush(self *startPosQueue, posdata *posData) {
|
|
256
|
+
var offset uint = ^(self.idx_) & 7
|
|
257
|
+
self.idx_++
|
|
258
|
+
var len uint = startPosQueueSize(self)
|
|
259
|
+
var i uint
|
|
260
|
+
var q []posData = self.q_[:]
|
|
261
|
+
q[offset] = *posdata
|
|
262
|
+
|
|
263
|
+
/* Restore the sorted order. In the list of |len| items at most |len - 1|
|
|
264
|
+
adjacent element comparisons / swaps are required. */
|
|
265
|
+
for i = 1; i < len; i++ {
|
|
266
|
+
if q[offset&7].costdiff > q[(offset+1)&7].costdiff {
|
|
267
|
+
var tmp posData = q[offset&7]
|
|
268
|
+
q[offset&7] = q[(offset+1)&7]
|
|
269
|
+
q[(offset+1)&7] = tmp
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
offset++
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
func startPosQueueAt(self *startPosQueue, k uint) *posData {
|
|
277
|
+
return &self.q_[(k-self.idx_)&7]
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
/* Returns the minimum possible copy length that can improve the cost of any */
|
|
281
|
+
/* future position. */
|
|
282
|
+
func computeMinimumCopyLength(start_cost float32, nodes []zopfliNode, num_bytes uint, pos uint) uint {
|
|
283
|
+
var min_cost float32 = start_cost
|
|
284
|
+
var len uint = 2
|
|
285
|
+
var next_len_bucket uint = 4
|
|
286
|
+
/* Compute the minimum possible cost of reaching any future position. */
|
|
287
|
+
|
|
288
|
+
var next_len_offset uint = 10
|
|
289
|
+
for pos+len <= num_bytes && nodes[pos+len].u.cost <= min_cost {
|
|
290
|
+
/* We already reached (pos + len) with no more cost than the minimum
|
|
291
|
+
possible cost of reaching anything from this pos, so there is no point in
|
|
292
|
+
looking for lengths <= len. */
|
|
293
|
+
len++
|
|
294
|
+
|
|
295
|
+
if len == next_len_offset {
|
|
296
|
+
/* We reached the next copy length code bucket, so we add one more
|
|
297
|
+
extra bit to the minimum cost. */
|
|
298
|
+
min_cost += 1.0
|
|
299
|
+
|
|
300
|
+
next_len_offset += next_len_bucket
|
|
301
|
+
next_len_bucket *= 2
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
return uint(len)
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
/* REQUIRES: nodes[pos].cost < kInfinity
|
|
309
|
+
REQUIRES: nodes[0..pos] satisfies that "ZopfliNode array invariant". */
|
|
310
|
+
func computeDistanceShortcut(block_start uint, pos uint, max_backward_limit uint, gap uint, nodes []zopfliNode) uint32 {
|
|
311
|
+
var clen uint = uint(zopfliNodeCopyLength(&nodes[pos]))
|
|
312
|
+
var ilen uint = uint(nodes[pos].dcode_insert_length & 0x7FFFFFF)
|
|
313
|
+
var dist uint = uint(zopfliNodeCopyDistance(&nodes[pos]))
|
|
314
|
+
|
|
315
|
+
/* Since |block_start + pos| is the end position of the command, the copy part
|
|
316
|
+
starts from |block_start + pos - clen|. Distances that are greater than
|
|
317
|
+
this or greater than |max_backward_limit| + |gap| are static dictionary
|
|
318
|
+
references, and do not update the last distances.
|
|
319
|
+
Also distance code 0 (last distance) does not update the last distances. */
|
|
320
|
+
if pos == 0 {
|
|
321
|
+
return 0
|
|
322
|
+
} else if dist+clen <= block_start+pos+gap && dist <= max_backward_limit+gap && zopfliNodeDistanceCode(&nodes[pos]) > 0 {
|
|
323
|
+
return uint32(pos)
|
|
324
|
+
} else {
|
|
325
|
+
return nodes[pos-clen-ilen].u.shortcut
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
/* Fills in dist_cache[0..3] with the last four distances (as defined by
|
|
330
|
+
Section 4. of the Spec) that would be used at (block_start + pos) if we
|
|
331
|
+
used the shortest path of commands from block_start, computed from
|
|
332
|
+
nodes[0..pos]. The last four distances at block_start are in
|
|
333
|
+
starting_dist_cache[0..3].
|
|
334
|
+
REQUIRES: nodes[pos].cost < kInfinity
|
|
335
|
+
REQUIRES: nodes[0..pos] satisfies that "ZopfliNode array invariant". */
|
|
336
|
+
func computeDistanceCache(pos uint, starting_dist_cache []int, nodes []zopfliNode, dist_cache []int) {
|
|
337
|
+
var idx int = 0
|
|
338
|
+
var p uint = uint(nodes[pos].u.shortcut)
|
|
339
|
+
for idx < 4 && p > 0 {
|
|
340
|
+
var ilen uint = uint(nodes[p].dcode_insert_length & 0x7FFFFFF)
|
|
341
|
+
var clen uint = uint(zopfliNodeCopyLength(&nodes[p]))
|
|
342
|
+
var dist uint = uint(zopfliNodeCopyDistance(&nodes[p]))
|
|
343
|
+
dist_cache[idx] = int(dist)
|
|
344
|
+
idx++
|
|
345
|
+
|
|
346
|
+
/* Because of prerequisite, p >= clen + ilen >= 2. */
|
|
347
|
+
p = uint(nodes[p-clen-ilen].u.shortcut)
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
for ; idx < 4; idx++ {
|
|
351
|
+
dist_cache[idx] = starting_dist_cache[0]
|
|
352
|
+
starting_dist_cache = starting_dist_cache[1:]
|
|
353
|
+
}
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
/* Maintains "ZopfliNode array invariant" and pushes node to the queue, if it
|
|
357
|
+
is eligible. */
|
|
358
|
+
func evaluateNode(block_start uint, pos uint, max_backward_limit uint, gap uint, starting_dist_cache []int, model *zopfliCostModel, queue *startPosQueue, nodes []zopfliNode) {
|
|
359
|
+
/* Save cost, because ComputeDistanceCache invalidates it. */
|
|
360
|
+
var node_cost float32 = nodes[pos].u.cost
|
|
361
|
+
nodes[pos].u.shortcut = computeDistanceShortcut(block_start, pos, max_backward_limit, gap, nodes)
|
|
362
|
+
if node_cost <= zopfliCostModelGetLiteralCosts(model, 0, pos) {
|
|
363
|
+
var posdata posData
|
|
364
|
+
posdata.pos = pos
|
|
365
|
+
posdata.cost = node_cost
|
|
366
|
+
posdata.costdiff = node_cost - zopfliCostModelGetLiteralCosts(model, 0, pos)
|
|
367
|
+
computeDistanceCache(pos, starting_dist_cache, nodes, posdata.distance_cache[:])
|
|
368
|
+
startPosQueuePush(queue, &posdata)
|
|
369
|
+
}
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
/* Returns longest copy length. */
|
|
373
|
+
func updateNodes(num_bytes uint, block_start uint, pos uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, max_backward_limit uint, starting_dist_cache []int, num_matches uint, matches []backwardMatch, model *zopfliCostModel, queue *startPosQueue, nodes []zopfliNode) uint {
|
|
374
|
+
var cur_ix uint = block_start + pos
|
|
375
|
+
var cur_ix_masked uint = cur_ix & ringbuffer_mask
|
|
376
|
+
var max_distance uint = brotli_min_size_t(cur_ix, max_backward_limit)
|
|
377
|
+
var max_len uint = num_bytes - pos
|
|
378
|
+
var max_zopfli_len uint = maxZopfliLen(params)
|
|
379
|
+
var max_iters uint = maxZopfliCandidates(params)
|
|
380
|
+
var min_len uint
|
|
381
|
+
var result uint = 0
|
|
382
|
+
var k uint
|
|
383
|
+
var gap uint = 0
|
|
384
|
+
|
|
385
|
+
evaluateNode(block_start, pos, max_backward_limit, gap, starting_dist_cache, model, queue, nodes)
|
|
386
|
+
{
|
|
387
|
+
var posdata *posData = startPosQueueAt(queue, 0)
|
|
388
|
+
var min_cost float32 = (posdata.cost + zopfliCostModelGetMinCostCmd(model) + zopfliCostModelGetLiteralCosts(model, posdata.pos, pos))
|
|
389
|
+
min_len = computeMinimumCopyLength(min_cost, nodes, num_bytes, pos)
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
/* Go over the command starting positions in order of increasing cost
|
|
393
|
+
difference. */
|
|
394
|
+
for k = 0; k < max_iters && k < startPosQueueSize(queue); k++ {
|
|
395
|
+
var posdata *posData = startPosQueueAt(queue, k)
|
|
396
|
+
var start uint = posdata.pos
|
|
397
|
+
var inscode uint16 = getInsertLengthCode(pos - start)
|
|
398
|
+
var start_costdiff float32 = posdata.costdiff
|
|
399
|
+
var base_cost float32 = start_costdiff + float32(getInsertExtra(inscode)) + zopfliCostModelGetLiteralCosts(model, 0, pos)
|
|
400
|
+
var best_len uint = min_len - 1
|
|
401
|
+
var j uint = 0
|
|
402
|
+
/* Look for last distance matches using the distance cache from this
|
|
403
|
+
starting position. */
|
|
404
|
+
for ; j < numDistanceShortCodes && best_len < max_len; j++ {
|
|
405
|
+
var idx uint = uint(kDistanceCacheIndex[j])
|
|
406
|
+
var backward uint = uint(posdata.distance_cache[idx] + kDistanceCacheOffset[j])
|
|
407
|
+
var prev_ix uint = cur_ix - backward
|
|
408
|
+
var len uint = 0
|
|
409
|
+
var continuation byte = ringbuffer[cur_ix_masked+best_len]
|
|
410
|
+
if cur_ix_masked+best_len > ringbuffer_mask {
|
|
411
|
+
break
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
if backward > max_distance+gap {
|
|
415
|
+
/* Word dictionary -> ignore. */
|
|
416
|
+
continue
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
if backward <= max_distance {
|
|
420
|
+
/* Regular backward reference. */
|
|
421
|
+
if prev_ix >= cur_ix {
|
|
422
|
+
continue
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
prev_ix &= ringbuffer_mask
|
|
426
|
+
if prev_ix+best_len > ringbuffer_mask || continuation != ringbuffer[prev_ix+best_len] {
|
|
427
|
+
continue
|
|
428
|
+
}
|
|
429
|
+
|
|
430
|
+
len = findMatchLengthWithLimit(ringbuffer[prev_ix:], ringbuffer[cur_ix_masked:], max_len)
|
|
431
|
+
} else {
|
|
432
|
+
continue
|
|
433
|
+
}
|
|
434
|
+
{
|
|
435
|
+
var dist_cost float32 = base_cost + zopfliCostModelGetDistanceCost(model, j)
|
|
436
|
+
var l uint
|
|
437
|
+
for l = best_len + 1; l <= len; l++ {
|
|
438
|
+
var copycode uint16 = getCopyLengthCode(l)
|
|
439
|
+
var cmdcode uint16 = combineLengthCodes(inscode, copycode, j == 0)
|
|
440
|
+
var tmp float32
|
|
441
|
+
if cmdcode < 128 {
|
|
442
|
+
tmp = base_cost
|
|
443
|
+
} else {
|
|
444
|
+
tmp = dist_cost
|
|
445
|
+
}
|
|
446
|
+
var cost float32 = tmp + float32(getCopyExtra(copycode)) + zopfliCostModelGetCommandCost(model, cmdcode)
|
|
447
|
+
if cost < nodes[pos+l].u.cost {
|
|
448
|
+
updateZopfliNode(nodes, pos, start, l, l, backward, j+1, cost)
|
|
449
|
+
result = brotli_max_size_t(result, l)
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
best_len = l
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
/* At higher iterations look only for new last distance matches, since
|
|
458
|
+
looking only for new command start positions with the same distances
|
|
459
|
+
does not help much. */
|
|
460
|
+
if k >= 2 {
|
|
461
|
+
continue
|
|
462
|
+
}
|
|
463
|
+
{
|
|
464
|
+
/* Loop through all possible copy lengths at this position. */
|
|
465
|
+
var len uint = min_len
|
|
466
|
+
for j = 0; j < num_matches; j++ {
|
|
467
|
+
var match backwardMatch = matches[j]
|
|
468
|
+
var dist uint = uint(match.distance)
|
|
469
|
+
var is_dictionary_match bool = (dist > max_distance+gap)
|
|
470
|
+
var dist_code uint = dist + numDistanceShortCodes - 1
|
|
471
|
+
var dist_symbol uint16
|
|
472
|
+
var distextra uint32
|
|
473
|
+
var distnumextra uint32
|
|
474
|
+
var dist_cost float32
|
|
475
|
+
var max_match_len uint
|
|
476
|
+
/* We already tried all possible last distance matches, so we can use
|
|
477
|
+
normal distance code here. */
|
|
478
|
+
prefixEncodeCopyDistance(dist_code, uint(params.dist.num_direct_distance_codes), uint(params.dist.distance_postfix_bits), &dist_symbol, &distextra)
|
|
479
|
+
|
|
480
|
+
distnumextra = uint32(dist_symbol) >> 10
|
|
481
|
+
dist_cost = base_cost + float32(distnumextra) + zopfliCostModelGetDistanceCost(model, uint(dist_symbol)&0x3FF)
|
|
482
|
+
|
|
483
|
+
/* Try all copy lengths up until the maximum copy length corresponding
|
|
484
|
+
to this distance. If the distance refers to the static dictionary, or
|
|
485
|
+
the maximum length is long enough, try only one maximum length. */
|
|
486
|
+
max_match_len = backwardMatchLength(&match)
|
|
487
|
+
|
|
488
|
+
if len < max_match_len && (is_dictionary_match || max_match_len > max_zopfli_len) {
|
|
489
|
+
len = max_match_len
|
|
490
|
+
}
|
|
491
|
+
|
|
492
|
+
for ; len <= max_match_len; len++ {
|
|
493
|
+
var len_code uint
|
|
494
|
+
if is_dictionary_match {
|
|
495
|
+
len_code = backwardMatchLengthCode(&match)
|
|
496
|
+
} else {
|
|
497
|
+
len_code = len
|
|
498
|
+
}
|
|
499
|
+
var copycode uint16 = getCopyLengthCode(len_code)
|
|
500
|
+
var cmdcode uint16 = combineLengthCodes(inscode, copycode, false)
|
|
501
|
+
var cost float32 = dist_cost + float32(getCopyExtra(copycode)) + zopfliCostModelGetCommandCost(model, cmdcode)
|
|
502
|
+
if cost < nodes[pos+len].u.cost {
|
|
503
|
+
updateZopfliNode(nodes, pos, start, uint(len), len_code, dist, 0, cost)
|
|
504
|
+
if len > result {
|
|
505
|
+
result = len
|
|
506
|
+
}
|
|
507
|
+
}
|
|
508
|
+
}
|
|
509
|
+
}
|
|
510
|
+
}
|
|
511
|
+
}
|
|
512
|
+
|
|
513
|
+
return result
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
func computeShortestPathFromNodes(num_bytes uint, nodes []zopfliNode) uint {
|
|
517
|
+
var index uint = num_bytes
|
|
518
|
+
var num_commands uint = 0
|
|
519
|
+
for nodes[index].dcode_insert_length&0x7FFFFFF == 0 && nodes[index].length == 1 {
|
|
520
|
+
index--
|
|
521
|
+
}
|
|
522
|
+
nodes[index].u.next = math.MaxUint32
|
|
523
|
+
for index != 0 {
|
|
524
|
+
var len uint = uint(zopfliNodeCommandLength(&nodes[index]))
|
|
525
|
+
index -= uint(len)
|
|
526
|
+
nodes[index].u.next = uint32(len)
|
|
527
|
+
num_commands++
|
|
528
|
+
}
|
|
529
|
+
|
|
530
|
+
return num_commands
|
|
531
|
+
}
|
|
532
|
+
|
|
533
|
+
/* REQUIRES: nodes != NULL and len(nodes) >= num_bytes + 1 */
|
|
534
|
+
func zopfliCreateCommands(num_bytes uint, block_start uint, nodes []zopfliNode, dist_cache []int, last_insert_len *uint, params *encoderParams, commands *[]command, num_literals *uint) {
|
|
535
|
+
var max_backward_limit uint = maxBackwardLimit(params.lgwin)
|
|
536
|
+
var pos uint = 0
|
|
537
|
+
var offset uint32 = nodes[0].u.next
|
|
538
|
+
var i uint
|
|
539
|
+
var gap uint = 0
|
|
540
|
+
for i = 0; offset != math.MaxUint32; i++ {
|
|
541
|
+
var next *zopfliNode = &nodes[uint32(pos)+offset]
|
|
542
|
+
var copy_length uint = uint(zopfliNodeCopyLength(next))
|
|
543
|
+
var insert_length uint = uint(next.dcode_insert_length & 0x7FFFFFF)
|
|
544
|
+
pos += insert_length
|
|
545
|
+
offset = next.u.next
|
|
546
|
+
if i == 0 {
|
|
547
|
+
insert_length += *last_insert_len
|
|
548
|
+
*last_insert_len = 0
|
|
549
|
+
}
|
|
550
|
+
{
|
|
551
|
+
var distance uint = uint(zopfliNodeCopyDistance(next))
|
|
552
|
+
var len_code uint = uint(zopfliNodeLengthCode(next))
|
|
553
|
+
var max_distance uint = brotli_min_size_t(block_start+pos, max_backward_limit)
|
|
554
|
+
var is_dictionary bool = (distance > max_distance+gap)
|
|
555
|
+
var dist_code uint = uint(zopfliNodeDistanceCode(next))
|
|
556
|
+
*commands = append(*commands, makeCommand(¶ms.dist, insert_length, copy_length, int(len_code)-int(copy_length), dist_code))
|
|
557
|
+
|
|
558
|
+
if !is_dictionary && dist_code > 0 {
|
|
559
|
+
dist_cache[3] = dist_cache[2]
|
|
560
|
+
dist_cache[2] = dist_cache[1]
|
|
561
|
+
dist_cache[1] = dist_cache[0]
|
|
562
|
+
dist_cache[0] = int(distance)
|
|
563
|
+
}
|
|
564
|
+
}
|
|
565
|
+
|
|
566
|
+
*num_literals += insert_length
|
|
567
|
+
pos += copy_length
|
|
568
|
+
}
|
|
569
|
+
|
|
570
|
+
*last_insert_len += num_bytes - pos
|
|
571
|
+
}
|
|
572
|
+
|
|
573
|
+
func zopfliIterate(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, gap uint, dist_cache []int, model *zopfliCostModel, num_matches []uint32, matches []backwardMatch, nodes []zopfliNode) uint {
|
|
574
|
+
var max_backward_limit uint = maxBackwardLimit(params.lgwin)
|
|
575
|
+
var max_zopfli_len uint = maxZopfliLen(params)
|
|
576
|
+
var queue startPosQueue
|
|
577
|
+
var cur_match_pos uint = 0
|
|
578
|
+
var i uint
|
|
579
|
+
nodes[0].length = 0
|
|
580
|
+
nodes[0].u.cost = 0
|
|
581
|
+
initStartPosQueue(&queue)
|
|
582
|
+
for i = 0; i+3 < num_bytes; i++ {
|
|
583
|
+
var skip uint = updateNodes(num_bytes, position, i, ringbuffer, ringbuffer_mask, params, max_backward_limit, dist_cache, uint(num_matches[i]), matches[cur_match_pos:], model, &queue, nodes)
|
|
584
|
+
if skip < longCopyQuickStep {
|
|
585
|
+
skip = 0
|
|
586
|
+
}
|
|
587
|
+
cur_match_pos += uint(num_matches[i])
|
|
588
|
+
if num_matches[i] == 1 && backwardMatchLength(&matches[cur_match_pos-1]) > max_zopfli_len {
|
|
589
|
+
skip = brotli_max_size_t(backwardMatchLength(&matches[cur_match_pos-1]), skip)
|
|
590
|
+
}
|
|
591
|
+
|
|
592
|
+
if skip > 1 {
|
|
593
|
+
skip--
|
|
594
|
+
for skip != 0 {
|
|
595
|
+
i++
|
|
596
|
+
if i+3 >= num_bytes {
|
|
597
|
+
break
|
|
598
|
+
}
|
|
599
|
+
evaluateNode(position, i, max_backward_limit, gap, dist_cache, model, &queue, nodes)
|
|
600
|
+
cur_match_pos += uint(num_matches[i])
|
|
601
|
+
skip--
|
|
602
|
+
}
|
|
603
|
+
}
|
|
604
|
+
}
|
|
605
|
+
|
|
606
|
+
return computeShortestPathFromNodes(num_bytes, nodes)
|
|
607
|
+
}
|
|
608
|
+
|
|
609
|
+
/* Computes the shortest path of commands from position to at most
|
|
610
|
+
position + num_bytes.
|
|
611
|
+
|
|
612
|
+
On return, path->size() is the number of commands found and path[i] is the
|
|
613
|
+
length of the i-th command (copy length plus insert length).
|
|
614
|
+
Note that the sum of the lengths of all commands can be less than num_bytes.
|
|
615
|
+
|
|
616
|
+
On return, the nodes[0..num_bytes] array will have the following
|
|
617
|
+
"ZopfliNode array invariant":
|
|
618
|
+
For each i in [1..num_bytes], if nodes[i].cost < kInfinity, then
|
|
619
|
+
(1) nodes[i].copy_length() >= 2
|
|
620
|
+
(2) nodes[i].command_length() <= i and
|
|
621
|
+
(3) nodes[i - nodes[i].command_length()].cost < kInfinity
|
|
622
|
+
|
|
623
|
+
REQUIRES: nodes != nil and len(nodes) >= num_bytes + 1 */
|
|
624
|
+
func zopfliComputeShortestPath(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, dist_cache []int, hasher *h10, nodes []zopfliNode) uint {
|
|
625
|
+
var max_backward_limit uint = maxBackwardLimit(params.lgwin)
|
|
626
|
+
var max_zopfli_len uint = maxZopfliLen(params)
|
|
627
|
+
var model zopfliCostModel
|
|
628
|
+
var queue startPosQueue
|
|
629
|
+
var matches [2 * (maxNumMatchesH10 + 64)]backwardMatch
|
|
630
|
+
var store_end uint
|
|
631
|
+
if num_bytes >= hasher.StoreLookahead() {
|
|
632
|
+
store_end = position + num_bytes - hasher.StoreLookahead() + 1
|
|
633
|
+
} else {
|
|
634
|
+
store_end = position
|
|
635
|
+
}
|
|
636
|
+
var i uint
|
|
637
|
+
var gap uint = 0
|
|
638
|
+
var lz_matches_offset uint = 0
|
|
639
|
+
nodes[0].length = 0
|
|
640
|
+
nodes[0].u.cost = 0
|
|
641
|
+
initZopfliCostModel(&model, ¶ms.dist, num_bytes)
|
|
642
|
+
zopfliCostModelSetFromLiteralCosts(&model, position, ringbuffer, ringbuffer_mask)
|
|
643
|
+
initStartPosQueue(&queue)
|
|
644
|
+
for i = 0; i+hasher.HashTypeLength()-1 < num_bytes; i++ {
|
|
645
|
+
var pos uint = position + i
|
|
646
|
+
var max_distance uint = brotli_min_size_t(pos, max_backward_limit)
|
|
647
|
+
var skip uint
|
|
648
|
+
var num_matches uint
|
|
649
|
+
num_matches = findAllMatchesH10(hasher, ¶ms.dictionary, ringbuffer, ringbuffer_mask, pos, num_bytes-i, max_distance, gap, params, matches[lz_matches_offset:])
|
|
650
|
+
if num_matches > 0 && backwardMatchLength(&matches[num_matches-1]) > max_zopfli_len {
|
|
651
|
+
matches[0] = matches[num_matches-1]
|
|
652
|
+
num_matches = 1
|
|
653
|
+
}
|
|
654
|
+
|
|
655
|
+
skip = updateNodes(num_bytes, position, i, ringbuffer, ringbuffer_mask, params, max_backward_limit, dist_cache, num_matches, matches[:], &model, &queue, nodes)
|
|
656
|
+
if skip < longCopyQuickStep {
|
|
657
|
+
skip = 0
|
|
658
|
+
}
|
|
659
|
+
if num_matches == 1 && backwardMatchLength(&matches[0]) > max_zopfli_len {
|
|
660
|
+
skip = brotli_max_size_t(backwardMatchLength(&matches[0]), skip)
|
|
661
|
+
}
|
|
662
|
+
|
|
663
|
+
if skip > 1 {
|
|
664
|
+
/* Add the tail of the copy to the hasher. */
|
|
665
|
+
hasher.StoreRange(ringbuffer, ringbuffer_mask, pos+1, brotli_min_size_t(pos+skip, store_end))
|
|
666
|
+
|
|
667
|
+
skip--
|
|
668
|
+
for skip != 0 {
|
|
669
|
+
i++
|
|
670
|
+
if i+hasher.HashTypeLength()-1 >= num_bytes {
|
|
671
|
+
break
|
|
672
|
+
}
|
|
673
|
+
evaluateNode(position, i, max_backward_limit, gap, dist_cache, &model, &queue, nodes)
|
|
674
|
+
skip--
|
|
675
|
+
}
|
|
676
|
+
}
|
|
677
|
+
}
|
|
678
|
+
|
|
679
|
+
cleanupZopfliCostModel(&model)
|
|
680
|
+
return computeShortestPathFromNodes(num_bytes, nodes)
|
|
681
|
+
}
|
|
682
|
+
|
|
683
|
+
func createZopfliBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher *h10, dist_cache []int, last_insert_len *uint, commands *[]command, num_literals *uint) {
|
|
684
|
+
var nodes []zopfliNode
|
|
685
|
+
nodes = make([]zopfliNode, (num_bytes + 1))
|
|
686
|
+
initZopfliNodes(nodes, num_bytes+1)
|
|
687
|
+
zopfliComputeShortestPath(num_bytes, position, ringbuffer, ringbuffer_mask, params, dist_cache, hasher, nodes)
|
|
688
|
+
zopfliCreateCommands(num_bytes, position, nodes, dist_cache, last_insert_len, params, commands, num_literals)
|
|
689
|
+
nodes = nil
|
|
690
|
+
}
|
|
691
|
+
|
|
692
|
+
func createHqZopfliBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher hasherHandle, dist_cache []int, last_insert_len *uint, commands *[]command, num_literals *uint) {
|
|
693
|
+
var max_backward_limit uint = maxBackwardLimit(params.lgwin)
|
|
694
|
+
var num_matches []uint32 = make([]uint32, num_bytes)
|
|
695
|
+
var matches_size uint = 4 * num_bytes
|
|
696
|
+
var store_end uint
|
|
697
|
+
if num_bytes >= hasher.StoreLookahead() {
|
|
698
|
+
store_end = position + num_bytes - hasher.StoreLookahead() + 1
|
|
699
|
+
} else {
|
|
700
|
+
store_end = position
|
|
701
|
+
}
|
|
702
|
+
var cur_match_pos uint = 0
|
|
703
|
+
var i uint
|
|
704
|
+
var orig_num_literals uint
|
|
705
|
+
var orig_last_insert_len uint
|
|
706
|
+
var orig_dist_cache [4]int
|
|
707
|
+
var orig_num_commands int
|
|
708
|
+
var model zopfliCostModel
|
|
709
|
+
var nodes []zopfliNode
|
|
710
|
+
var matches []backwardMatch = make([]backwardMatch, matches_size)
|
|
711
|
+
var gap uint = 0
|
|
712
|
+
var shadow_matches uint = 0
|
|
713
|
+
var new_array []backwardMatch
|
|
714
|
+
for i = 0; i+hasher.HashTypeLength()-1 < num_bytes; i++ {
|
|
715
|
+
var pos uint = position + i
|
|
716
|
+
var max_distance uint = brotli_min_size_t(pos, max_backward_limit)
|
|
717
|
+
var max_length uint = num_bytes - i
|
|
718
|
+
var num_found_matches uint
|
|
719
|
+
var cur_match_end uint
|
|
720
|
+
var j uint
|
|
721
|
+
|
|
722
|
+
/* Ensure that we have enough free slots. */
|
|
723
|
+
if matches_size < cur_match_pos+maxNumMatchesH10+shadow_matches {
|
|
724
|
+
var new_size uint = matches_size
|
|
725
|
+
if new_size == 0 {
|
|
726
|
+
new_size = cur_match_pos + maxNumMatchesH10 + shadow_matches
|
|
727
|
+
}
|
|
728
|
+
|
|
729
|
+
for new_size < cur_match_pos+maxNumMatchesH10+shadow_matches {
|
|
730
|
+
new_size *= 2
|
|
731
|
+
}
|
|
732
|
+
|
|
733
|
+
new_array = make([]backwardMatch, new_size)
|
|
734
|
+
if matches_size != 0 {
|
|
735
|
+
copy(new_array, matches[:matches_size])
|
|
736
|
+
}
|
|
737
|
+
|
|
738
|
+
matches = new_array
|
|
739
|
+
matches_size = new_size
|
|
740
|
+
}
|
|
741
|
+
|
|
742
|
+
num_found_matches = findAllMatchesH10(hasher.(*h10), ¶ms.dictionary, ringbuffer, ringbuffer_mask, pos, max_length, max_distance, gap, params, matches[cur_match_pos+shadow_matches:])
|
|
743
|
+
cur_match_end = cur_match_pos + num_found_matches
|
|
744
|
+
for j = cur_match_pos; j+1 < cur_match_end; j++ {
|
|
745
|
+
assert(backwardMatchLength(&matches[j]) <= backwardMatchLength(&matches[j+1]))
|
|
746
|
+
}
|
|
747
|
+
|
|
748
|
+
num_matches[i] = uint32(num_found_matches)
|
|
749
|
+
if num_found_matches > 0 {
|
|
750
|
+
var match_len uint = backwardMatchLength(&matches[cur_match_end-1])
|
|
751
|
+
if match_len > maxZopfliLenQuality11 {
|
|
752
|
+
var skip uint = match_len - 1
|
|
753
|
+
matches[cur_match_pos] = matches[cur_match_end-1]
|
|
754
|
+
cur_match_pos++
|
|
755
|
+
num_matches[i] = 1
|
|
756
|
+
|
|
757
|
+
/* Add the tail of the copy to the hasher. */
|
|
758
|
+
hasher.StoreRange(ringbuffer, ringbuffer_mask, pos+1, brotli_min_size_t(pos+match_len, store_end))
|
|
759
|
+
var pos uint = i
|
|
760
|
+
for i := 0; i < int(skip); i++ {
|
|
761
|
+
num_matches[pos+1:][i] = 0
|
|
762
|
+
}
|
|
763
|
+
i += skip
|
|
764
|
+
} else {
|
|
765
|
+
cur_match_pos = cur_match_end
|
|
766
|
+
}
|
|
767
|
+
}
|
|
768
|
+
}
|
|
769
|
+
|
|
770
|
+
orig_num_literals = *num_literals
|
|
771
|
+
orig_last_insert_len = *last_insert_len
|
|
772
|
+
copy(orig_dist_cache[:], dist_cache[:4])
|
|
773
|
+
orig_num_commands = len(*commands)
|
|
774
|
+
nodes = make([]zopfliNode, (num_bytes + 1))
|
|
775
|
+
initZopfliCostModel(&model, ¶ms.dist, num_bytes)
|
|
776
|
+
for i = 0; i < 2; i++ {
|
|
777
|
+
initZopfliNodes(nodes, num_bytes+1)
|
|
778
|
+
if i == 0 {
|
|
779
|
+
zopfliCostModelSetFromLiteralCosts(&model, position, ringbuffer, ringbuffer_mask)
|
|
780
|
+
} else {
|
|
781
|
+
zopfliCostModelSetFromCommands(&model, position, ringbuffer, ringbuffer_mask, (*commands)[orig_num_commands:], orig_last_insert_len)
|
|
782
|
+
}
|
|
783
|
+
|
|
784
|
+
*commands = (*commands)[:orig_num_commands]
|
|
785
|
+
*num_literals = orig_num_literals
|
|
786
|
+
*last_insert_len = orig_last_insert_len
|
|
787
|
+
copy(dist_cache, orig_dist_cache[:4])
|
|
788
|
+
zopfliIterate(num_bytes, position, ringbuffer, ringbuffer_mask, params, gap, dist_cache, &model, num_matches, matches, nodes)
|
|
789
|
+
zopfliCreateCommands(num_bytes, position, nodes, dist_cache, last_insert_len, params, commands, num_literals)
|
|
790
|
+
}
|
|
791
|
+
|
|
792
|
+
cleanupZopfliCostModel(&model)
|
|
793
|
+
nodes = nil
|
|
794
|
+
matches = nil
|
|
795
|
+
num_matches = nil
|
|
796
|
+
}
|