typenative 0.3.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +57 -0
- package/README.md +17 -1
- package/bin/cli.js +25 -11
- package/bin/go/convert.js +11 -1
- package/bin/index.js +221 -47
- package/bin/main.js +35 -14
- package/bin/node/assert.js +2 -2
- package/bin/node/buffer.js +9 -0
- package/bin/node/dns.js +1 -1
- package/bin/node/events.js +1 -1
- package/bin/node/fs.js +1 -1
- package/bin/node/fs_watch.js +407 -0
- package/bin/node/http2.js +33 -6
- package/bin/node/readline.js +1 -1
- package/bin/node/registry.js +2 -1
- package/bin/node/stream.js +6 -3
- package/bin/node/timers.js +1 -1
- package/bin/node/typed_arrays.js +608 -0
- package/bin/node/util.js +3 -0
- package/bin/node/zlib.js +131 -16
- package/bin/transpiler.js +635 -44
- package/bin/version.js +1 -1
- package/conformance.md +16 -4
- package/examples/README.md +54 -0
- package/examples/fibonacci/README.md +89 -0
- package/examples/fibonacci/bench.mjs +87 -0
- package/examples/fibonacci/main.ts +21 -0
- package/examples/hello-world/README.md +62 -0
- package/examples/hello-world/bench.mjs +66 -0
- package/examples/hello-world/main.ts +7 -0
- package/examples/json-server/README.md +125 -0
- package/examples/json-server/bench.mjs +171 -0
- package/examples/json-server/main.ts +67 -0
- package/govendor/github.com/andybalholm/brotli/LICENSE +19 -0
- package/govendor/github.com/andybalholm/brotli/README.md +19 -0
- package/govendor/github.com/andybalholm/brotli/backward_references.go +185 -0
- package/govendor/github.com/andybalholm/brotli/backward_references_hq.go +796 -0
- package/govendor/github.com/andybalholm/brotli/bit_cost.go +436 -0
- package/govendor/github.com/andybalholm/brotli/bit_reader.go +266 -0
- package/govendor/github.com/andybalholm/brotli/bitwriter.go +56 -0
- package/govendor/github.com/andybalholm/brotli/block_splitter.go +144 -0
- package/govendor/github.com/andybalholm/brotli/block_splitter_command.go +434 -0
- package/govendor/github.com/andybalholm/brotli/block_splitter_distance.go +433 -0
- package/govendor/github.com/andybalholm/brotli/block_splitter_literal.go +433 -0
- package/govendor/github.com/andybalholm/brotli/brotli_bit_stream.go +1435 -0
- package/govendor/github.com/andybalholm/brotli/cluster.go +30 -0
- package/govendor/github.com/andybalholm/brotli/cluster_command.go +164 -0
- package/govendor/github.com/andybalholm/brotli/cluster_distance.go +326 -0
- package/govendor/github.com/andybalholm/brotli/cluster_literal.go +326 -0
- package/govendor/github.com/andybalholm/brotli/command.go +254 -0
- package/govendor/github.com/andybalholm/brotli/compress_fragment.go +834 -0
- package/govendor/github.com/andybalholm/brotli/compress_fragment_two_pass.go +775 -0
- package/govendor/github.com/andybalholm/brotli/constants.go +77 -0
- package/govendor/github.com/andybalholm/brotli/context.go +2176 -0
- package/govendor/github.com/andybalholm/brotli/decode.go +2570 -0
- package/govendor/github.com/andybalholm/brotli/dictionary.go +122890 -0
- package/govendor/github.com/andybalholm/brotli/dictionary_hash.go +32779 -0
- package/govendor/github.com/andybalholm/brotli/encode.go +1220 -0
- package/govendor/github.com/andybalholm/brotli/encoder.go +177 -0
- package/govendor/github.com/andybalholm/brotli/encoder_dict.go +22 -0
- package/govendor/github.com/andybalholm/brotli/encoder_fast.go +201 -0
- package/govendor/github.com/andybalholm/brotli/entropy_encode.go +592 -0
- package/govendor/github.com/andybalholm/brotli/entropy_encode_static.go +4399 -0
- package/govendor/github.com/andybalholm/brotli/fast_log.go +290 -0
- package/govendor/github.com/andybalholm/brotli/find_match_length.go +45 -0
- package/govendor/github.com/andybalholm/brotli/go.mod +3 -0
- package/govendor/github.com/andybalholm/brotli/h10.go +287 -0
- package/govendor/github.com/andybalholm/brotli/h5.go +214 -0
- package/govendor/github.com/andybalholm/brotli/h6.go +216 -0
- package/govendor/github.com/andybalholm/brotli/hash.go +342 -0
- package/govendor/github.com/andybalholm/brotli/hash_composite.go +93 -0
- package/govendor/github.com/andybalholm/brotli/hash_forgetful_chain.go +252 -0
- package/govendor/github.com/andybalholm/brotli/hash_longest_match_quickly.go +214 -0
- package/govendor/github.com/andybalholm/brotli/hash_rolling.go +168 -0
- package/govendor/github.com/andybalholm/brotli/histogram.go +226 -0
- package/govendor/github.com/andybalholm/brotli/http.go +201 -0
- package/govendor/github.com/andybalholm/brotli/huffman.go +653 -0
- package/govendor/github.com/andybalholm/brotli/literal_cost.go +182 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/bargain1.go +212 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/bargain2.go +248 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/bargain3.go +278 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/emitter.go +34 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/m0.go +175 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/m4.go +339 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/matchfinder.go +103 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/pathfinder.go +328 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/textencoder.go +53 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/trio.go +327 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/zdfast.go +258 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/zfast.go +206 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/zm.go +302 -0
- package/govendor/github.com/andybalholm/brotli/memory.go +66 -0
- package/govendor/github.com/andybalholm/brotli/metablock.go +574 -0
- package/govendor/github.com/andybalholm/brotli/metablock_command.go +165 -0
- package/govendor/github.com/andybalholm/brotli/metablock_distance.go +165 -0
- package/govendor/github.com/andybalholm/brotli/metablock_literal.go +165 -0
- package/govendor/github.com/andybalholm/brotli/params.go +37 -0
- package/govendor/github.com/andybalholm/brotli/platform.go +103 -0
- package/govendor/github.com/andybalholm/brotli/prefix.go +30 -0
- package/govendor/github.com/andybalholm/brotli/prefix_dec.go +723 -0
- package/govendor/github.com/andybalholm/brotli/quality.go +196 -0
- package/govendor/github.com/andybalholm/brotli/reader.go +127 -0
- package/govendor/github.com/andybalholm/brotli/ringbuffer.go +134 -0
- package/govendor/github.com/andybalholm/brotli/state.go +313 -0
- package/govendor/github.com/andybalholm/brotli/static_dict.go +662 -0
- package/govendor/github.com/andybalholm/brotli/static_dict_lut.go +75094 -0
- package/govendor/github.com/andybalholm/brotli/symbol_list.go +22 -0
- package/govendor/github.com/andybalholm/brotli/transform.go +641 -0
- package/govendor/github.com/andybalholm/brotli/utf8_util.go +70 -0
- package/govendor/github.com/andybalholm/brotli/util.go +7 -0
- package/govendor/github.com/andybalholm/brotli/write_bits.go +52 -0
- package/govendor/github.com/andybalholm/brotli/writer.go +162 -0
- package/govendor/github.com/fsnotify/fsnotify/.mailmap +2 -0
- package/govendor/github.com/fsnotify/fsnotify/CHANGELOG.md +651 -0
- package/govendor/github.com/fsnotify/fsnotify/CONTRIBUTING.md +147 -0
- package/govendor/github.com/fsnotify/fsnotify/LICENSE +25 -0
- package/govendor/github.com/fsnotify/fsnotify/README.md +214 -0
- package/govendor/github.com/fsnotify/fsnotify/backend_fen.go +469 -0
- package/govendor/github.com/fsnotify/fsnotify/backend_inotify.go +597 -0
- package/govendor/github.com/fsnotify/fsnotify/backend_kqueue.go +735 -0
- package/govendor/github.com/fsnotify/fsnotify/backend_other.go +22 -0
- package/govendor/github.com/fsnotify/fsnotify/backend_windows.go +702 -0
- package/govendor/github.com/fsnotify/fsnotify/fsnotify.go +526 -0
- package/govendor/github.com/fsnotify/fsnotify/go.mod +3 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/darwin.go +20 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_darwin.go +15 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_dragonfly.go +15 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_freebsd.go +20 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_kqueue.go +32 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_linux.go +56 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_netbsd.go +15 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_openbsd.go +16 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_solaris.go +45 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_windows.go +40 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/freebsd.go +20 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/internal.go +2 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/unix.go +20 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/unix2.go +25 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/windows.go +40 -0
- package/govendor/github.com/fsnotify/fsnotify/shared.go +64 -0
- package/govendor/github.com/fsnotify/fsnotify/staticcheck.conf +3 -0
- package/govendor/github.com/fsnotify/fsnotify/system_bsd.go +7 -0
- package/govendor/github.com/fsnotify/fsnotify/system_darwin.go +8 -0
- package/govendor/manifest.txt +3 -1
- package/package.json +1 -1
- package/types/typenative-npm.d.ts +72 -0
- package/types/typenative.d.ts +117 -1
|
@@ -0,0 +1,342 @@
|
|
|
1
|
+
package brotli
|
|
2
|
+
|
|
3
|
+
import (
|
|
4
|
+
"encoding/binary"
|
|
5
|
+
"fmt"
|
|
6
|
+
)
|
|
7
|
+
|
|
8
|
+
type hasherCommon struct {
|
|
9
|
+
params hasherParams
|
|
10
|
+
is_prepared_ bool
|
|
11
|
+
dict_num_lookups uint
|
|
12
|
+
dict_num_matches uint
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
func (h *hasherCommon) Common() *hasherCommon {
|
|
16
|
+
return h
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
type hasherHandle interface {
|
|
20
|
+
Common() *hasherCommon
|
|
21
|
+
Initialize(params *encoderParams)
|
|
22
|
+
Prepare(one_shot bool, input_size uint, data []byte)
|
|
23
|
+
StitchToPreviousBlock(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint)
|
|
24
|
+
HashTypeLength() uint
|
|
25
|
+
StoreLookahead() uint
|
|
26
|
+
PrepareDistanceCache(distance_cache []int)
|
|
27
|
+
FindLongestMatch(dictionary *encoderDictionary, data []byte, ring_buffer_mask uint, distance_cache []int, cur_ix uint, max_length uint, max_backward uint, gap uint, max_distance uint, out *hasherSearchResult)
|
|
28
|
+
StoreRange(data []byte, mask uint, ix_start uint, ix_end uint)
|
|
29
|
+
Store(data []byte, mask uint, ix uint)
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
const kCutoffTransformsCount uint32 = 10
|
|
33
|
+
|
|
34
|
+
/* 0, 12, 27, 23, 42, 63, 56, 48, 59, 64 */
|
|
35
|
+
/* 0+0, 4+8, 8+19, 12+11, 16+26, 20+43, 24+32, 28+20, 32+27, 36+28 */
|
|
36
|
+
const kCutoffTransforms uint64 = 0x071B520ADA2D3200
|
|
37
|
+
|
|
38
|
+
type hasherSearchResult struct {
|
|
39
|
+
len uint
|
|
40
|
+
distance uint
|
|
41
|
+
score uint
|
|
42
|
+
len_code_delta int
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/* kHashMul32 multiplier has these properties:
|
|
46
|
+
* The multiplier must be odd. Otherwise we may lose the highest bit.
|
|
47
|
+
* No long streaks of ones or zeros.
|
|
48
|
+
* There is no effort to ensure that it is a prime, the oddity is enough
|
|
49
|
+
for this use.
|
|
50
|
+
* The number has been tuned heuristically against compression benchmarks. */
|
|
51
|
+
const kHashMul32 uint32 = 0x1E35A7BD
|
|
52
|
+
|
|
53
|
+
const kHashMul64 uint64 = 0x1E35A7BD1E35A7BD
|
|
54
|
+
|
|
55
|
+
const kHashMul64Long uint64 = 0x1FE35A7BD3579BD3
|
|
56
|
+
|
|
57
|
+
func hash14(data []byte) uint32 {
|
|
58
|
+
var h uint32 = binary.LittleEndian.Uint32(data) * kHashMul32
|
|
59
|
+
|
|
60
|
+
/* The higher bits contain more mixture from the multiplication,
|
|
61
|
+
so we take our results from there. */
|
|
62
|
+
return h >> (32 - 14)
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
func prepareDistanceCache(distance_cache []int, num_distances int) {
|
|
66
|
+
if num_distances > 4 {
|
|
67
|
+
var last_distance int = distance_cache[0]
|
|
68
|
+
distance_cache[4] = last_distance - 1
|
|
69
|
+
distance_cache[5] = last_distance + 1
|
|
70
|
+
distance_cache[6] = last_distance - 2
|
|
71
|
+
distance_cache[7] = last_distance + 2
|
|
72
|
+
distance_cache[8] = last_distance - 3
|
|
73
|
+
distance_cache[9] = last_distance + 3
|
|
74
|
+
if num_distances > 10 {
|
|
75
|
+
var next_last_distance int = distance_cache[1]
|
|
76
|
+
distance_cache[10] = next_last_distance - 1
|
|
77
|
+
distance_cache[11] = next_last_distance + 1
|
|
78
|
+
distance_cache[12] = next_last_distance - 2
|
|
79
|
+
distance_cache[13] = next_last_distance + 2
|
|
80
|
+
distance_cache[14] = next_last_distance - 3
|
|
81
|
+
distance_cache[15] = next_last_distance + 3
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
const literalByteScore = 135
|
|
87
|
+
|
|
88
|
+
const distanceBitPenalty = 30
|
|
89
|
+
|
|
90
|
+
/* Score must be positive after applying maximal penalty. */
|
|
91
|
+
const scoreBase = (distanceBitPenalty * 8 * 8)
|
|
92
|
+
|
|
93
|
+
/* Usually, we always choose the longest backward reference. This function
|
|
94
|
+
allows for the exception of that rule.
|
|
95
|
+
|
|
96
|
+
If we choose a backward reference that is further away, it will
|
|
97
|
+
usually be coded with more bits. We approximate this by assuming
|
|
98
|
+
log2(distance). If the distance can be expressed in terms of the
|
|
99
|
+
last four distances, we use some heuristic constants to estimate
|
|
100
|
+
the bits cost. For the first up to four literals we use the bit
|
|
101
|
+
cost of the literals from the literal cost model, after that we
|
|
102
|
+
use the average bit cost of the cost model.
|
|
103
|
+
|
|
104
|
+
This function is used to sometimes discard a longer backward reference
|
|
105
|
+
when it is not much longer and the bit cost for encoding it is more
|
|
106
|
+
than the saved literals.
|
|
107
|
+
|
|
108
|
+
backward_reference_offset MUST be positive. */
|
|
109
|
+
func backwardReferenceScore(copy_length uint, backward_reference_offset uint) uint {
|
|
110
|
+
return scoreBase + literalByteScore*uint(copy_length) - distanceBitPenalty*uint(log2FloorNonZero(backward_reference_offset))
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
func backwardReferenceScoreUsingLastDistance(copy_length uint) uint {
|
|
114
|
+
return literalByteScore*uint(copy_length) + scoreBase + 15
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
func backwardReferencePenaltyUsingLastDistance(distance_short_code uint) uint {
|
|
118
|
+
return uint(39) + ((0x1CA10 >> (distance_short_code & 0xE)) & 0xE)
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
func testStaticDictionaryItem(dictionary *encoderDictionary, item uint, data []byte, max_length uint, max_backward uint, max_distance uint, out *hasherSearchResult) bool {
|
|
122
|
+
var len uint
|
|
123
|
+
var word_idx uint
|
|
124
|
+
var offset uint
|
|
125
|
+
var matchlen uint
|
|
126
|
+
var backward uint
|
|
127
|
+
var score uint
|
|
128
|
+
len = item & 0x1F
|
|
129
|
+
word_idx = item >> 5
|
|
130
|
+
offset = uint(dictionary.words.offsets_by_length[len]) + len*word_idx
|
|
131
|
+
if len > max_length {
|
|
132
|
+
return false
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
matchlen = findMatchLengthWithLimit(data, dictionary.words.data[offset:], uint(len))
|
|
136
|
+
if matchlen+uint(dictionary.cutoffTransformsCount) <= len || matchlen == 0 {
|
|
137
|
+
return false
|
|
138
|
+
}
|
|
139
|
+
{
|
|
140
|
+
var cut uint = len - matchlen
|
|
141
|
+
var transform_id uint = (cut << 2) + uint((dictionary.cutoffTransforms>>(cut*6))&0x3F)
|
|
142
|
+
backward = max_backward + 1 + word_idx + (transform_id << dictionary.words.size_bits_by_length[len])
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
if backward > max_distance {
|
|
146
|
+
return false
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
score = backwardReferenceScore(matchlen, backward)
|
|
150
|
+
if score < out.score {
|
|
151
|
+
return false
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
out.len = matchlen
|
|
155
|
+
out.len_code_delta = int(len) - int(matchlen)
|
|
156
|
+
out.distance = backward
|
|
157
|
+
out.score = score
|
|
158
|
+
return true
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
func searchInStaticDictionary(dictionary *encoderDictionary, handle hasherHandle, data []byte, max_length uint, max_backward uint, max_distance uint, out *hasherSearchResult, shallow bool) {
|
|
162
|
+
var key uint
|
|
163
|
+
var i uint
|
|
164
|
+
var self *hasherCommon = handle.Common()
|
|
165
|
+
if self.dict_num_matches < self.dict_num_lookups>>7 {
|
|
166
|
+
return
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
key = uint(hash14(data) << 1)
|
|
170
|
+
for i = 0; ; (func() { i++; key++ })() {
|
|
171
|
+
var tmp uint
|
|
172
|
+
if shallow {
|
|
173
|
+
tmp = 1
|
|
174
|
+
} else {
|
|
175
|
+
tmp = 2
|
|
176
|
+
}
|
|
177
|
+
if i >= tmp {
|
|
178
|
+
break
|
|
179
|
+
}
|
|
180
|
+
var item uint = uint(dictionary.hash_table[key])
|
|
181
|
+
self.dict_num_lookups++
|
|
182
|
+
if item != 0 {
|
|
183
|
+
var item_matches bool = testStaticDictionaryItem(dictionary, item, data, max_length, max_backward, max_distance, out)
|
|
184
|
+
if item_matches {
|
|
185
|
+
self.dict_num_matches++
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
type backwardMatch struct {
|
|
192
|
+
distance uint32
|
|
193
|
+
length_and_code uint32
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
func initBackwardMatch(self *backwardMatch, dist uint, len uint) {
|
|
197
|
+
self.distance = uint32(dist)
|
|
198
|
+
self.length_and_code = uint32(len << 5)
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
func initDictionaryBackwardMatch(self *backwardMatch, dist uint, len uint, len_code uint) {
|
|
202
|
+
self.distance = uint32(dist)
|
|
203
|
+
var tmp uint
|
|
204
|
+
if len == len_code {
|
|
205
|
+
tmp = 0
|
|
206
|
+
} else {
|
|
207
|
+
tmp = len_code
|
|
208
|
+
}
|
|
209
|
+
self.length_and_code = uint32(len<<5 | tmp)
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
func backwardMatchLength(self *backwardMatch) uint {
|
|
213
|
+
return uint(self.length_and_code >> 5)
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
func backwardMatchLengthCode(self *backwardMatch) uint {
|
|
217
|
+
var code uint = uint(self.length_and_code) & 31
|
|
218
|
+
if code != 0 {
|
|
219
|
+
return code
|
|
220
|
+
} else {
|
|
221
|
+
return backwardMatchLength(self)
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
func hasherReset(handle hasherHandle) {
|
|
226
|
+
if handle == nil {
|
|
227
|
+
return
|
|
228
|
+
}
|
|
229
|
+
handle.Common().is_prepared_ = false
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
func newHasher(typ int) hasherHandle {
|
|
233
|
+
switch typ {
|
|
234
|
+
case 2:
|
|
235
|
+
return &hashLongestMatchQuickly{
|
|
236
|
+
bucketBits: 16,
|
|
237
|
+
bucketSweep: 1,
|
|
238
|
+
hashLen: 5,
|
|
239
|
+
useDictionary: true,
|
|
240
|
+
}
|
|
241
|
+
case 3:
|
|
242
|
+
return &hashLongestMatchQuickly{
|
|
243
|
+
bucketBits: 16,
|
|
244
|
+
bucketSweep: 2,
|
|
245
|
+
hashLen: 5,
|
|
246
|
+
useDictionary: false,
|
|
247
|
+
}
|
|
248
|
+
case 4:
|
|
249
|
+
return &hashLongestMatchQuickly{
|
|
250
|
+
bucketBits: 17,
|
|
251
|
+
bucketSweep: 4,
|
|
252
|
+
hashLen: 5,
|
|
253
|
+
useDictionary: true,
|
|
254
|
+
}
|
|
255
|
+
case 5:
|
|
256
|
+
return new(h5)
|
|
257
|
+
case 6:
|
|
258
|
+
return new(h6)
|
|
259
|
+
case 10:
|
|
260
|
+
return new(h10)
|
|
261
|
+
case 35:
|
|
262
|
+
return &hashComposite{
|
|
263
|
+
ha: newHasher(3),
|
|
264
|
+
hb: &hashRolling{jump: 4},
|
|
265
|
+
}
|
|
266
|
+
case 40:
|
|
267
|
+
return &hashForgetfulChain{
|
|
268
|
+
bucketBits: 15,
|
|
269
|
+
numBanks: 1,
|
|
270
|
+
bankBits: 16,
|
|
271
|
+
numLastDistancesToCheck: 4,
|
|
272
|
+
}
|
|
273
|
+
case 41:
|
|
274
|
+
return &hashForgetfulChain{
|
|
275
|
+
bucketBits: 15,
|
|
276
|
+
numBanks: 1,
|
|
277
|
+
bankBits: 16,
|
|
278
|
+
numLastDistancesToCheck: 10,
|
|
279
|
+
}
|
|
280
|
+
case 42:
|
|
281
|
+
return &hashForgetfulChain{
|
|
282
|
+
bucketBits: 15,
|
|
283
|
+
numBanks: 512,
|
|
284
|
+
bankBits: 9,
|
|
285
|
+
numLastDistancesToCheck: 16,
|
|
286
|
+
}
|
|
287
|
+
case 54:
|
|
288
|
+
return &hashLongestMatchQuickly{
|
|
289
|
+
bucketBits: 20,
|
|
290
|
+
bucketSweep: 4,
|
|
291
|
+
hashLen: 7,
|
|
292
|
+
useDictionary: false,
|
|
293
|
+
}
|
|
294
|
+
case 55:
|
|
295
|
+
return &hashComposite{
|
|
296
|
+
ha: newHasher(54),
|
|
297
|
+
hb: &hashRolling{jump: 4},
|
|
298
|
+
}
|
|
299
|
+
case 65:
|
|
300
|
+
return &hashComposite{
|
|
301
|
+
ha: newHasher(6),
|
|
302
|
+
hb: &hashRolling{jump: 1},
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
panic(fmt.Sprintf("unknown hasher type: %d", typ))
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
func hasherSetup(handle *hasherHandle, params *encoderParams, data []byte, position uint, input_size uint, is_last bool) {
|
|
310
|
+
var self hasherHandle = nil
|
|
311
|
+
var common *hasherCommon = nil
|
|
312
|
+
var one_shot bool = (position == 0 && is_last)
|
|
313
|
+
if *handle == nil {
|
|
314
|
+
chooseHasher(params, ¶ms.hasher)
|
|
315
|
+
self = newHasher(params.hasher.type_)
|
|
316
|
+
|
|
317
|
+
*handle = self
|
|
318
|
+
common = self.Common()
|
|
319
|
+
common.params = params.hasher
|
|
320
|
+
self.Initialize(params)
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
self = *handle
|
|
324
|
+
common = self.Common()
|
|
325
|
+
if !common.is_prepared_ {
|
|
326
|
+
self.Prepare(one_shot, input_size, data)
|
|
327
|
+
|
|
328
|
+
if position == 0 {
|
|
329
|
+
common.dict_num_lookups = 0
|
|
330
|
+
common.dict_num_matches = 0
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
common.is_prepared_ = true
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
func initOrStitchToPreviousBlock(handle *hasherHandle, data []byte, mask uint, params *encoderParams, position uint, input_size uint, is_last bool) {
|
|
338
|
+
var self hasherHandle
|
|
339
|
+
hasherSetup(handle, params, data, position, input_size, is_last)
|
|
340
|
+
self = *handle
|
|
341
|
+
self.StitchToPreviousBlock(input_size, position, data, mask)
|
|
342
|
+
}
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
package brotli
|
|
2
|
+
|
|
3
|
+
/* Copyright 2018 Google Inc. All Rights Reserved.
|
|
4
|
+
|
|
5
|
+
Distributed under MIT license.
|
|
6
|
+
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
func (h *hashComposite) HashTypeLength() uint {
|
|
10
|
+
var a uint = h.ha.HashTypeLength()
|
|
11
|
+
var b uint = h.hb.HashTypeLength()
|
|
12
|
+
if a > b {
|
|
13
|
+
return a
|
|
14
|
+
} else {
|
|
15
|
+
return b
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
func (h *hashComposite) StoreLookahead() uint {
|
|
20
|
+
var a uint = h.ha.StoreLookahead()
|
|
21
|
+
var b uint = h.hb.StoreLookahead()
|
|
22
|
+
if a > b {
|
|
23
|
+
return a
|
|
24
|
+
} else {
|
|
25
|
+
return b
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/* Composite hasher: This hasher allows to combine two other hashers, HASHER_A
|
|
30
|
+
and HASHER_B. */
|
|
31
|
+
type hashComposite struct {
|
|
32
|
+
hasherCommon
|
|
33
|
+
ha hasherHandle
|
|
34
|
+
hb hasherHandle
|
|
35
|
+
params *encoderParams
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
func (h *hashComposite) Initialize(params *encoderParams) {
|
|
39
|
+
h.params = params
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/* TODO: Initialize of the hashers is defered to Prepare (and params
|
|
43
|
+
remembered here) because we don't get the one_shot and input_size params
|
|
44
|
+
here that are needed to know the memory size of them. Instead provide
|
|
45
|
+
those params to all hashers InitializehashComposite */
|
|
46
|
+
func (h *hashComposite) Prepare(one_shot bool, input_size uint, data []byte) {
|
|
47
|
+
if h.ha == nil {
|
|
48
|
+
var common_a *hasherCommon
|
|
49
|
+
var common_b *hasherCommon
|
|
50
|
+
|
|
51
|
+
common_a = h.ha.Common()
|
|
52
|
+
common_a.params = h.params.hasher
|
|
53
|
+
common_a.is_prepared_ = false
|
|
54
|
+
common_a.dict_num_lookups = 0
|
|
55
|
+
common_a.dict_num_matches = 0
|
|
56
|
+
h.ha.Initialize(h.params)
|
|
57
|
+
|
|
58
|
+
common_b = h.hb.Common()
|
|
59
|
+
common_b.params = h.params.hasher
|
|
60
|
+
common_b.is_prepared_ = false
|
|
61
|
+
common_b.dict_num_lookups = 0
|
|
62
|
+
common_b.dict_num_matches = 0
|
|
63
|
+
h.hb.Initialize(h.params)
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
h.ha.Prepare(one_shot, input_size, data)
|
|
67
|
+
h.hb.Prepare(one_shot, input_size, data)
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
func (h *hashComposite) Store(data []byte, mask uint, ix uint) {
|
|
71
|
+
h.ha.Store(data, mask, ix)
|
|
72
|
+
h.hb.Store(data, mask, ix)
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
func (h *hashComposite) StoreRange(data []byte, mask uint, ix_start uint, ix_end uint) {
|
|
76
|
+
h.ha.StoreRange(data, mask, ix_start, ix_end)
|
|
77
|
+
h.hb.StoreRange(data, mask, ix_start, ix_end)
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
func (h *hashComposite) StitchToPreviousBlock(num_bytes uint, position uint, ringbuffer []byte, ring_buffer_mask uint) {
|
|
81
|
+
h.ha.StitchToPreviousBlock(num_bytes, position, ringbuffer, ring_buffer_mask)
|
|
82
|
+
h.hb.StitchToPreviousBlock(num_bytes, position, ringbuffer, ring_buffer_mask)
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
func (h *hashComposite) PrepareDistanceCache(distance_cache []int) {
|
|
86
|
+
h.ha.PrepareDistanceCache(distance_cache)
|
|
87
|
+
h.hb.PrepareDistanceCache(distance_cache)
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
func (h *hashComposite) FindLongestMatch(dictionary *encoderDictionary, data []byte, ring_buffer_mask uint, distance_cache []int, cur_ix uint, max_length uint, max_backward uint, gap uint, max_distance uint, out *hasherSearchResult) {
|
|
91
|
+
h.ha.FindLongestMatch(dictionary, data, ring_buffer_mask, distance_cache, cur_ix, max_length, max_backward, gap, max_distance, out)
|
|
92
|
+
h.hb.FindLongestMatch(dictionary, data, ring_buffer_mask, distance_cache, cur_ix, max_length, max_backward, gap, max_distance, out)
|
|
93
|
+
}
|
|
@@ -0,0 +1,252 @@
|
|
|
1
|
+
package brotli
|
|
2
|
+
|
|
3
|
+
import "encoding/binary"
|
|
4
|
+
|
|
5
|
+
/* Copyright 2016 Google Inc. All Rights Reserved.
|
|
6
|
+
|
|
7
|
+
Distributed under MIT license.
|
|
8
|
+
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
func (*hashForgetfulChain) HashTypeLength() uint {
|
|
12
|
+
return 4
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
func (*hashForgetfulChain) StoreLookahead() uint {
|
|
16
|
+
return 4
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
/* HashBytes is the function that chooses the bucket to place the address in.*/
|
|
20
|
+
func (h *hashForgetfulChain) HashBytes(data []byte) uint {
|
|
21
|
+
var hash uint32 = binary.LittleEndian.Uint32(data) * kHashMul32
|
|
22
|
+
|
|
23
|
+
/* The higher bits contain more mixture from the multiplication,
|
|
24
|
+
so we take our results from there. */
|
|
25
|
+
return uint(hash >> (32 - h.bucketBits))
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
type slot struct {
|
|
29
|
+
delta uint16
|
|
30
|
+
next uint16
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/* A (forgetful) hash table to the data seen by the compressor, to
|
|
34
|
+
help create backward references to previous data.
|
|
35
|
+
|
|
36
|
+
Hashes are stored in chains which are bucketed to groups. Group of chains
|
|
37
|
+
share a storage "bank". When more than "bank size" chain nodes are added,
|
|
38
|
+
oldest nodes are replaced; this way several chains may share a tail. */
|
|
39
|
+
type hashForgetfulChain struct {
|
|
40
|
+
hasherCommon
|
|
41
|
+
|
|
42
|
+
bucketBits uint
|
|
43
|
+
numBanks uint
|
|
44
|
+
bankBits uint
|
|
45
|
+
numLastDistancesToCheck int
|
|
46
|
+
|
|
47
|
+
addr []uint32
|
|
48
|
+
head []uint16
|
|
49
|
+
tiny_hash [65536]byte
|
|
50
|
+
banks [][]slot
|
|
51
|
+
free_slot_idx []uint16
|
|
52
|
+
max_hops uint
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
func (h *hashForgetfulChain) Initialize(params *encoderParams) {
|
|
56
|
+
var q uint
|
|
57
|
+
if params.quality > 6 {
|
|
58
|
+
q = 7
|
|
59
|
+
} else {
|
|
60
|
+
q = 8
|
|
61
|
+
}
|
|
62
|
+
h.max_hops = q << uint(params.quality-4)
|
|
63
|
+
|
|
64
|
+
bankSize := 1 << h.bankBits
|
|
65
|
+
bucketSize := 1 << h.bucketBits
|
|
66
|
+
|
|
67
|
+
h.addr = make([]uint32, bucketSize)
|
|
68
|
+
h.head = make([]uint16, bucketSize)
|
|
69
|
+
h.banks = make([][]slot, h.numBanks)
|
|
70
|
+
for i := range h.banks {
|
|
71
|
+
h.banks[i] = make([]slot, bankSize)
|
|
72
|
+
}
|
|
73
|
+
h.free_slot_idx = make([]uint16, h.numBanks)
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
func (h *hashForgetfulChain) Prepare(one_shot bool, input_size uint, data []byte) {
|
|
77
|
+
var partial_prepare_threshold uint = (1 << h.bucketBits) >> 6
|
|
78
|
+
/* Partial preparation is 100 times slower (per socket). */
|
|
79
|
+
if one_shot && input_size <= partial_prepare_threshold {
|
|
80
|
+
var i uint
|
|
81
|
+
for i = 0; i < input_size; i++ {
|
|
82
|
+
var bucket uint = h.HashBytes(data[i:])
|
|
83
|
+
|
|
84
|
+
/* See InitEmpty comment. */
|
|
85
|
+
h.addr[bucket] = 0xCCCCCCCC
|
|
86
|
+
|
|
87
|
+
h.head[bucket] = 0xCCCC
|
|
88
|
+
}
|
|
89
|
+
} else {
|
|
90
|
+
/* Fill |addr| array with 0xCCCCCCCC value. Because of wrapping, position
|
|
91
|
+
processed by hasher never reaches 3GB + 64M; this makes all new chains
|
|
92
|
+
to be terminated after the first node. */
|
|
93
|
+
for i := range h.addr {
|
|
94
|
+
h.addr[i] = 0xCCCCCCCC
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
for i := range h.head {
|
|
98
|
+
h.head[i] = 0
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
h.tiny_hash = [65536]byte{}
|
|
103
|
+
for i := range h.free_slot_idx {
|
|
104
|
+
h.free_slot_idx[i] = 0
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/* Look at 4 bytes at &data[ix & mask]. Compute a hash from these, and prepend
|
|
109
|
+
node to corresponding chain; also update tiny_hash for current position. */
|
|
110
|
+
func (h *hashForgetfulChain) Store(data []byte, mask uint, ix uint) {
|
|
111
|
+
var key uint = h.HashBytes(data[ix&mask:])
|
|
112
|
+
var bank uint = key & (h.numBanks - 1)
|
|
113
|
+
idx := uint(h.free_slot_idx[bank]) & ((1 << h.bankBits) - 1)
|
|
114
|
+
h.free_slot_idx[bank]++
|
|
115
|
+
var delta uint = ix - uint(h.addr[key])
|
|
116
|
+
h.tiny_hash[uint16(ix)] = byte(key)
|
|
117
|
+
if delta > 0xFFFF {
|
|
118
|
+
delta = 0xFFFF
|
|
119
|
+
}
|
|
120
|
+
h.banks[bank][idx].delta = uint16(delta)
|
|
121
|
+
h.banks[bank][idx].next = h.head[key]
|
|
122
|
+
h.addr[key] = uint32(ix)
|
|
123
|
+
h.head[key] = uint16(idx)
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
func (h *hashForgetfulChain) StoreRange(data []byte, mask uint, ix_start uint, ix_end uint) {
|
|
127
|
+
var i uint
|
|
128
|
+
for i = ix_start; i < ix_end; i++ {
|
|
129
|
+
h.Store(data, mask, i)
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
func (h *hashForgetfulChain) StitchToPreviousBlock(num_bytes uint, position uint, ringbuffer []byte, ring_buffer_mask uint) {
|
|
134
|
+
if num_bytes >= h.HashTypeLength()-1 && position >= 3 {
|
|
135
|
+
/* Prepare the hashes for three last bytes of the last write.
|
|
136
|
+
These could not be calculated before, since they require knowledge
|
|
137
|
+
of both the previous and the current block. */
|
|
138
|
+
h.Store(ringbuffer, ring_buffer_mask, position-3)
|
|
139
|
+
h.Store(ringbuffer, ring_buffer_mask, position-2)
|
|
140
|
+
h.Store(ringbuffer, ring_buffer_mask, position-1)
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
func (h *hashForgetfulChain) PrepareDistanceCache(distance_cache []int) {
|
|
145
|
+
prepareDistanceCache(distance_cache, h.numLastDistancesToCheck)
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/* Find a longest backward match of &data[cur_ix] up to the length of
|
|
149
|
+
max_length and stores the position cur_ix in the hash table.
|
|
150
|
+
|
|
151
|
+
REQUIRES: PrepareDistanceCachehashForgetfulChain must be invoked for current distance cache
|
|
152
|
+
values; if this method is invoked repeatedly with the same distance
|
|
153
|
+
cache values, it is enough to invoke PrepareDistanceCachehashForgetfulChain once.
|
|
154
|
+
|
|
155
|
+
Does not look for matches longer than max_length.
|
|
156
|
+
Does not look for matches further away than max_backward.
|
|
157
|
+
Writes the best match into |out|.
|
|
158
|
+
|out|->score is updated only if a better match is found. */
|
|
159
|
+
func (h *hashForgetfulChain) FindLongestMatch(dictionary *encoderDictionary, data []byte, ring_buffer_mask uint, distance_cache []int, cur_ix uint, max_length uint, max_backward uint, gap uint, max_distance uint, out *hasherSearchResult) {
|
|
160
|
+
var cur_ix_masked uint = cur_ix & ring_buffer_mask
|
|
161
|
+
var min_score uint = out.score
|
|
162
|
+
var best_score uint = out.score
|
|
163
|
+
var best_len uint = out.len
|
|
164
|
+
var key uint = h.HashBytes(data[cur_ix_masked:])
|
|
165
|
+
var tiny_hash byte = byte(key)
|
|
166
|
+
/* Don't accept a short copy from far away. */
|
|
167
|
+
out.len = 0
|
|
168
|
+
|
|
169
|
+
out.len_code_delta = 0
|
|
170
|
+
|
|
171
|
+
/* Try last distance first. */
|
|
172
|
+
for i := 0; i < h.numLastDistancesToCheck; i++ {
|
|
173
|
+
var backward uint = uint(distance_cache[i])
|
|
174
|
+
var prev_ix uint = (cur_ix - backward)
|
|
175
|
+
|
|
176
|
+
/* For distance code 0 we want to consider 2-byte matches. */
|
|
177
|
+
if i > 0 && h.tiny_hash[uint16(prev_ix)] != tiny_hash {
|
|
178
|
+
continue
|
|
179
|
+
}
|
|
180
|
+
if prev_ix >= cur_ix || backward > max_backward {
|
|
181
|
+
continue
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
prev_ix &= ring_buffer_mask
|
|
185
|
+
{
|
|
186
|
+
var len uint = findMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length)
|
|
187
|
+
if len >= 2 {
|
|
188
|
+
var score uint = backwardReferenceScoreUsingLastDistance(uint(len))
|
|
189
|
+
if best_score < score {
|
|
190
|
+
if i != 0 {
|
|
191
|
+
score -= backwardReferencePenaltyUsingLastDistance(uint(i))
|
|
192
|
+
}
|
|
193
|
+
if best_score < score {
|
|
194
|
+
best_score = score
|
|
195
|
+
best_len = uint(len)
|
|
196
|
+
out.len = best_len
|
|
197
|
+
out.distance = backward
|
|
198
|
+
out.score = best_score
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
{
|
|
205
|
+
var bank uint = key & (h.numBanks - 1)
|
|
206
|
+
var backward uint = 0
|
|
207
|
+
var hops uint = h.max_hops
|
|
208
|
+
var delta uint = cur_ix - uint(h.addr[key])
|
|
209
|
+
var slot uint = uint(h.head[key])
|
|
210
|
+
for {
|
|
211
|
+
tmp6 := hops
|
|
212
|
+
hops--
|
|
213
|
+
if tmp6 == 0 {
|
|
214
|
+
break
|
|
215
|
+
}
|
|
216
|
+
var prev_ix uint
|
|
217
|
+
var last uint = slot
|
|
218
|
+
backward += delta
|
|
219
|
+
if backward > max_backward {
|
|
220
|
+
break
|
|
221
|
+
}
|
|
222
|
+
prev_ix = (cur_ix - backward) & ring_buffer_mask
|
|
223
|
+
slot = uint(h.banks[bank][last].next)
|
|
224
|
+
delta = uint(h.banks[bank][last].delta)
|
|
225
|
+
if cur_ix_masked+best_len > ring_buffer_mask || prev_ix+best_len > ring_buffer_mask || data[cur_ix_masked+best_len] != data[prev_ix+best_len] {
|
|
226
|
+
continue
|
|
227
|
+
}
|
|
228
|
+
{
|
|
229
|
+
var len uint = findMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length)
|
|
230
|
+
if len >= 4 {
|
|
231
|
+
/* Comparing for >= 3 does not change the semantics, but just saves
|
|
232
|
+
for a few unnecessary binary logarithms in backward reference
|
|
233
|
+
score, since we are not interested in such short matches. */
|
|
234
|
+
var score uint = backwardReferenceScore(uint(len), backward)
|
|
235
|
+
if best_score < score {
|
|
236
|
+
best_score = score
|
|
237
|
+
best_len = uint(len)
|
|
238
|
+
out.len = best_len
|
|
239
|
+
out.distance = backward
|
|
240
|
+
out.score = best_score
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
h.Store(data, ring_buffer_mask, cur_ix)
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
if out.score == min_score {
|
|
250
|
+
searchInStaticDictionary(dictionary, h, data[cur_ix_masked:], max_length, max_backward+gap, max_distance, out, false)
|
|
251
|
+
}
|
|
252
|
+
}
|