typenative 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +71 -0
- package/README.md +17 -1
- package/bin/cli.js +25 -11
- package/bin/go/convert.js +11 -1
- package/bin/index.js +221 -44
- package/bin/main.js +35 -14
- package/bin/node/assert.js +2 -2
- package/bin/node/buffer.js +9 -0
- package/bin/node/dns.js +1 -1
- package/bin/node/events.js +1 -1
- package/bin/node/fs.js +1 -1
- package/bin/node/fs_watch.js +407 -0
- package/bin/node/http2.js +33 -6
- package/bin/node/readline.js +1 -1
- package/bin/node/registry.js +2 -1
- package/bin/node/stream.js +6 -3
- package/bin/node/timers.js +1 -1
- package/bin/node/tls.js +3 -1
- package/bin/node/typed_arrays.js +608 -0
- package/bin/node/util.js +3 -0
- package/bin/node/zlib.js +131 -16
- package/bin/transpiler.js +684 -62
- package/bin/version.js +1 -1
- package/conformance.md +16 -4
- package/examples/README.md +54 -0
- package/examples/fibonacci/README.md +89 -0
- package/examples/fibonacci/bench.mjs +87 -0
- package/examples/fibonacci/main.ts +21 -0
- package/examples/hello-world/README.md +62 -0
- package/examples/hello-world/bench.mjs +66 -0
- package/examples/hello-world/main.ts +7 -0
- package/examples/json-server/README.md +125 -0
- package/examples/json-server/bench.mjs +171 -0
- package/examples/json-server/main.ts +67 -0
- package/govendor/github.com/andybalholm/brotli/LICENSE +19 -0
- package/govendor/github.com/andybalholm/brotli/README.md +19 -0
- package/govendor/github.com/andybalholm/brotli/backward_references.go +185 -0
- package/govendor/github.com/andybalholm/brotli/backward_references_hq.go +796 -0
- package/govendor/github.com/andybalholm/brotli/bit_cost.go +436 -0
- package/govendor/github.com/andybalholm/brotli/bit_reader.go +266 -0
- package/govendor/github.com/andybalholm/brotli/bitwriter.go +56 -0
- package/govendor/github.com/andybalholm/brotli/block_splitter.go +144 -0
- package/govendor/github.com/andybalholm/brotli/block_splitter_command.go +434 -0
- package/govendor/github.com/andybalholm/brotli/block_splitter_distance.go +433 -0
- package/govendor/github.com/andybalholm/brotli/block_splitter_literal.go +433 -0
- package/govendor/github.com/andybalholm/brotli/brotli_bit_stream.go +1435 -0
- package/govendor/github.com/andybalholm/brotli/cluster.go +30 -0
- package/govendor/github.com/andybalholm/brotli/cluster_command.go +164 -0
- package/govendor/github.com/andybalholm/brotli/cluster_distance.go +326 -0
- package/govendor/github.com/andybalholm/brotli/cluster_literal.go +326 -0
- package/govendor/github.com/andybalholm/brotli/command.go +254 -0
- package/govendor/github.com/andybalholm/brotli/compress_fragment.go +834 -0
- package/govendor/github.com/andybalholm/brotli/compress_fragment_two_pass.go +775 -0
- package/govendor/github.com/andybalholm/brotli/constants.go +77 -0
- package/govendor/github.com/andybalholm/brotli/context.go +2176 -0
- package/govendor/github.com/andybalholm/brotli/decode.go +2570 -0
- package/govendor/github.com/andybalholm/brotli/dictionary.go +122890 -0
- package/govendor/github.com/andybalholm/brotli/dictionary_hash.go +32779 -0
- package/govendor/github.com/andybalholm/brotli/encode.go +1220 -0
- package/govendor/github.com/andybalholm/brotli/encoder.go +177 -0
- package/govendor/github.com/andybalholm/brotli/encoder_dict.go +22 -0
- package/govendor/github.com/andybalholm/brotli/encoder_fast.go +201 -0
- package/govendor/github.com/andybalholm/brotli/entropy_encode.go +592 -0
- package/govendor/github.com/andybalholm/brotli/entropy_encode_static.go +4399 -0
- package/govendor/github.com/andybalholm/brotli/fast_log.go +290 -0
- package/govendor/github.com/andybalholm/brotli/find_match_length.go +45 -0
- package/govendor/github.com/andybalholm/brotli/go.mod +3 -0
- package/govendor/github.com/andybalholm/brotli/h10.go +287 -0
- package/govendor/github.com/andybalholm/brotli/h5.go +214 -0
- package/govendor/github.com/andybalholm/brotli/h6.go +216 -0
- package/govendor/github.com/andybalholm/brotli/hash.go +342 -0
- package/govendor/github.com/andybalholm/brotli/hash_composite.go +93 -0
- package/govendor/github.com/andybalholm/brotli/hash_forgetful_chain.go +252 -0
- package/govendor/github.com/andybalholm/brotli/hash_longest_match_quickly.go +214 -0
- package/govendor/github.com/andybalholm/brotli/hash_rolling.go +168 -0
- package/govendor/github.com/andybalholm/brotli/histogram.go +226 -0
- package/govendor/github.com/andybalholm/brotli/http.go +201 -0
- package/govendor/github.com/andybalholm/brotli/huffman.go +653 -0
- package/govendor/github.com/andybalholm/brotli/literal_cost.go +182 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/bargain1.go +212 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/bargain2.go +248 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/bargain3.go +278 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/emitter.go +34 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/m0.go +175 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/m4.go +339 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/matchfinder.go +103 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/pathfinder.go +328 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/textencoder.go +53 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/trio.go +327 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/zdfast.go +258 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/zfast.go +206 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/zm.go +302 -0
- package/govendor/github.com/andybalholm/brotli/memory.go +66 -0
- package/govendor/github.com/andybalholm/brotli/metablock.go +574 -0
- package/govendor/github.com/andybalholm/brotli/metablock_command.go +165 -0
- package/govendor/github.com/andybalholm/brotli/metablock_distance.go +165 -0
- package/govendor/github.com/andybalholm/brotli/metablock_literal.go +165 -0
- package/govendor/github.com/andybalholm/brotli/params.go +37 -0
- package/govendor/github.com/andybalholm/brotli/platform.go +103 -0
- package/govendor/github.com/andybalholm/brotli/prefix.go +30 -0
- package/govendor/github.com/andybalholm/brotli/prefix_dec.go +723 -0
- package/govendor/github.com/andybalholm/brotli/quality.go +196 -0
- package/govendor/github.com/andybalholm/brotli/reader.go +127 -0
- package/govendor/github.com/andybalholm/brotli/ringbuffer.go +134 -0
- package/govendor/github.com/andybalholm/brotli/state.go +313 -0
- package/govendor/github.com/andybalholm/brotli/static_dict.go +662 -0
- package/govendor/github.com/andybalholm/brotli/static_dict_lut.go +75094 -0
- package/govendor/github.com/andybalholm/brotli/symbol_list.go +22 -0
- package/govendor/github.com/andybalholm/brotli/transform.go +641 -0
- package/govendor/github.com/andybalholm/brotli/utf8_util.go +70 -0
- package/govendor/github.com/andybalholm/brotli/util.go +7 -0
- package/govendor/github.com/andybalholm/brotli/write_bits.go +52 -0
- package/govendor/github.com/andybalholm/brotli/writer.go +162 -0
- package/govendor/github.com/fsnotify/fsnotify/.mailmap +2 -0
- package/govendor/github.com/fsnotify/fsnotify/CHANGELOG.md +651 -0
- package/govendor/github.com/fsnotify/fsnotify/CONTRIBUTING.md +147 -0
- package/govendor/github.com/fsnotify/fsnotify/LICENSE +25 -0
- package/govendor/github.com/fsnotify/fsnotify/README.md +214 -0
- package/govendor/github.com/fsnotify/fsnotify/backend_fen.go +469 -0
- package/govendor/github.com/fsnotify/fsnotify/backend_inotify.go +597 -0
- package/govendor/github.com/fsnotify/fsnotify/backend_kqueue.go +735 -0
- package/govendor/github.com/fsnotify/fsnotify/backend_other.go +22 -0
- package/govendor/github.com/fsnotify/fsnotify/backend_windows.go +702 -0
- package/govendor/github.com/fsnotify/fsnotify/fsnotify.go +526 -0
- package/govendor/github.com/fsnotify/fsnotify/go.mod +3 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/darwin.go +20 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_darwin.go +15 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_dragonfly.go +15 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_freebsd.go +20 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_kqueue.go +32 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_linux.go +56 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_netbsd.go +15 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_openbsd.go +16 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_solaris.go +45 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_windows.go +40 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/freebsd.go +20 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/internal.go +2 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/unix.go +20 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/unix2.go +25 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/windows.go +40 -0
- package/govendor/github.com/fsnotify/fsnotify/shared.go +64 -0
- package/govendor/github.com/fsnotify/fsnotify/staticcheck.conf +3 -0
- package/govendor/github.com/fsnotify/fsnotify/system_bsd.go +7 -0
- package/govendor/github.com/fsnotify/fsnotify/system_darwin.go +8 -0
- package/govendor/manifest.txt +3 -1
- package/package.json +1 -1
- package/types/typenative-npm.d.ts +72 -0
- package/types/typenative.d.ts +117 -1
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
package brotli
|
|
2
|
+
|
|
3
|
+
import "encoding/binary"
|
|
4
|
+
|
|
5
|
+
/* Copyright 2010 Google Inc. All Rights Reserved.
|
|
6
|
+
|
|
7
|
+
Distributed under MIT license.
|
|
8
|
+
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
/* A (forgetful) hash table to the data seen by the compressor, to
|
|
12
|
+
help create backward references to previous data.
|
|
13
|
+
|
|
14
|
+
This is a hash map of fixed size (bucket_size_) to a ring buffer of
|
|
15
|
+
fixed size (block_size_). The ring buffer contains the last block_size_
|
|
16
|
+
index positions of the given hash key in the compressed data. */
|
|
17
|
+
func (*h5) HashTypeLength() uint {
|
|
18
|
+
return 4
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
func (*h5) StoreLookahead() uint {
|
|
22
|
+
return 4
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/* HashBytes is the function that chooses the bucket to place the address in. */
|
|
26
|
+
func hashBytesH5(data []byte, shift int) uint32 {
|
|
27
|
+
var h uint32 = binary.LittleEndian.Uint32(data) * kHashMul32
|
|
28
|
+
|
|
29
|
+
/* The higher bits contain more mixture from the multiplication,
|
|
30
|
+
so we take our results from there. */
|
|
31
|
+
return uint32(h >> uint(shift))
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
type h5 struct {
|
|
35
|
+
hasherCommon
|
|
36
|
+
bucket_size_ uint
|
|
37
|
+
block_size_ uint
|
|
38
|
+
hash_shift_ int
|
|
39
|
+
block_mask_ uint32
|
|
40
|
+
num []uint16
|
|
41
|
+
buckets []uint32
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
func (h *h5) Initialize(params *encoderParams) {
|
|
45
|
+
h.hash_shift_ = 32 - h.params.bucket_bits
|
|
46
|
+
h.bucket_size_ = uint(1) << uint(h.params.bucket_bits)
|
|
47
|
+
h.block_size_ = uint(1) << uint(h.params.block_bits)
|
|
48
|
+
h.block_mask_ = uint32(h.block_size_ - 1)
|
|
49
|
+
h.num = make([]uint16, h.bucket_size_)
|
|
50
|
+
h.buckets = make([]uint32, h.block_size_*h.bucket_size_)
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
func (h *h5) Prepare(one_shot bool, input_size uint, data []byte) {
|
|
54
|
+
var num []uint16 = h.num
|
|
55
|
+
var partial_prepare_threshold uint = h.bucket_size_ >> 6
|
|
56
|
+
/* Partial preparation is 100 times slower (per socket). */
|
|
57
|
+
if one_shot && input_size <= partial_prepare_threshold {
|
|
58
|
+
var i uint
|
|
59
|
+
for i = 0; i < input_size; i++ {
|
|
60
|
+
var key uint32 = hashBytesH5(data[i:], h.hash_shift_)
|
|
61
|
+
num[key] = 0
|
|
62
|
+
}
|
|
63
|
+
} else {
|
|
64
|
+
for i := 0; i < int(h.bucket_size_); i++ {
|
|
65
|
+
num[i] = 0
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/* Look at 4 bytes at &data[ix & mask].
|
|
71
|
+
Compute a hash from these, and store the value of ix at that position. */
|
|
72
|
+
func (h *h5) Store(data []byte, mask uint, ix uint) {
|
|
73
|
+
var num []uint16 = h.num
|
|
74
|
+
var key uint32 = hashBytesH5(data[ix&mask:], h.hash_shift_)
|
|
75
|
+
var minor_ix uint = uint(num[key]) & uint(h.block_mask_)
|
|
76
|
+
var offset uint = minor_ix + uint(key<<uint(h.params.block_bits))
|
|
77
|
+
h.buckets[offset] = uint32(ix)
|
|
78
|
+
num[key]++
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
func (h *h5) StoreRange(data []byte, mask uint, ix_start uint, ix_end uint) {
|
|
82
|
+
var i uint
|
|
83
|
+
for i = ix_start; i < ix_end; i++ {
|
|
84
|
+
h.Store(data, mask, i)
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
func (h *h5) StitchToPreviousBlock(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint) {
|
|
89
|
+
if num_bytes >= h.HashTypeLength()-1 && position >= 3 {
|
|
90
|
+
/* Prepare the hashes for three last bytes of the last write.
|
|
91
|
+
These could not be calculated before, since they require knowledge
|
|
92
|
+
of both the previous and the current block. */
|
|
93
|
+
h.Store(ringbuffer, ringbuffer_mask, position-3)
|
|
94
|
+
h.Store(ringbuffer, ringbuffer_mask, position-2)
|
|
95
|
+
h.Store(ringbuffer, ringbuffer_mask, position-1)
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
func (h *h5) PrepareDistanceCache(distance_cache []int) {
|
|
100
|
+
prepareDistanceCache(distance_cache, h.params.num_last_distances_to_check)
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/* Find a longest backward match of &data[cur_ix] up to the length of
|
|
104
|
+
max_length and stores the position cur_ix in the hash table.
|
|
105
|
+
|
|
106
|
+
REQUIRES: PrepareDistanceCacheH5 must be invoked for current distance cache
|
|
107
|
+
values; if this method is invoked repeatedly with the same distance
|
|
108
|
+
cache values, it is enough to invoke PrepareDistanceCacheH5 once.
|
|
109
|
+
|
|
110
|
+
Does not look for matches longer than max_length.
|
|
111
|
+
Does not look for matches further away than max_backward.
|
|
112
|
+
Writes the best match into |out|.
|
|
113
|
+
|out|->score is updated only if a better match is found. */
|
|
114
|
+
func (h *h5) FindLongestMatch(dictionary *encoderDictionary, data []byte, ring_buffer_mask uint, distance_cache []int, cur_ix uint, max_length uint, max_backward uint, gap uint, max_distance uint, out *hasherSearchResult) {
|
|
115
|
+
var num []uint16 = h.num
|
|
116
|
+
var buckets []uint32 = h.buckets
|
|
117
|
+
var cur_ix_masked uint = cur_ix & ring_buffer_mask
|
|
118
|
+
var min_score uint = out.score
|
|
119
|
+
var best_score uint = out.score
|
|
120
|
+
var best_len uint = out.len
|
|
121
|
+
var i uint
|
|
122
|
+
var bucket []uint32
|
|
123
|
+
/* Don't accept a short copy from far away. */
|
|
124
|
+
out.len = 0
|
|
125
|
+
|
|
126
|
+
out.len_code_delta = 0
|
|
127
|
+
|
|
128
|
+
/* Try last distance first. */
|
|
129
|
+
for i = 0; i < uint(h.params.num_last_distances_to_check); i++ {
|
|
130
|
+
var backward uint = uint(distance_cache[i])
|
|
131
|
+
var prev_ix uint = uint(cur_ix - backward)
|
|
132
|
+
if prev_ix >= cur_ix {
|
|
133
|
+
continue
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
if backward > max_backward {
|
|
137
|
+
continue
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
prev_ix &= ring_buffer_mask
|
|
141
|
+
|
|
142
|
+
if cur_ix_masked+best_len > ring_buffer_mask || prev_ix+best_len > ring_buffer_mask || data[cur_ix_masked+best_len] != data[prev_ix+best_len] {
|
|
143
|
+
continue
|
|
144
|
+
}
|
|
145
|
+
{
|
|
146
|
+
var len uint = findMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length)
|
|
147
|
+
if len >= 3 || (len == 2 && i < 2) {
|
|
148
|
+
/* Comparing for >= 2 does not change the semantics, but just saves for
|
|
149
|
+
a few unnecessary binary logarithms in backward reference score,
|
|
150
|
+
since we are not interested in such short matches. */
|
|
151
|
+
var score uint = backwardReferenceScoreUsingLastDistance(uint(len))
|
|
152
|
+
if best_score < score {
|
|
153
|
+
if i != 0 {
|
|
154
|
+
score -= backwardReferencePenaltyUsingLastDistance(i)
|
|
155
|
+
}
|
|
156
|
+
if best_score < score {
|
|
157
|
+
best_score = score
|
|
158
|
+
best_len = uint(len)
|
|
159
|
+
out.len = best_len
|
|
160
|
+
out.distance = backward
|
|
161
|
+
out.score = best_score
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
{
|
|
168
|
+
var key uint32 = hashBytesH5(data[cur_ix_masked:], h.hash_shift_)
|
|
169
|
+
bucket = buckets[key<<uint(h.params.block_bits):]
|
|
170
|
+
var down uint
|
|
171
|
+
if uint(num[key]) > h.block_size_ {
|
|
172
|
+
down = uint(num[key]) - h.block_size_
|
|
173
|
+
} else {
|
|
174
|
+
down = 0
|
|
175
|
+
}
|
|
176
|
+
for i = uint(num[key]); i > down; {
|
|
177
|
+
var prev_ix uint
|
|
178
|
+
i--
|
|
179
|
+
prev_ix = uint(bucket[uint32(i)&h.block_mask_])
|
|
180
|
+
var backward uint = cur_ix - prev_ix
|
|
181
|
+
if backward > max_backward {
|
|
182
|
+
break
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
prev_ix &= ring_buffer_mask
|
|
186
|
+
if cur_ix_masked+best_len > ring_buffer_mask || prev_ix+best_len > ring_buffer_mask || data[cur_ix_masked+best_len] != data[prev_ix+best_len] {
|
|
187
|
+
continue
|
|
188
|
+
}
|
|
189
|
+
{
|
|
190
|
+
var len uint = findMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length)
|
|
191
|
+
if len >= 4 {
|
|
192
|
+
/* Comparing for >= 3 does not change the semantics, but just saves
|
|
193
|
+
for a few unnecessary binary logarithms in backward reference
|
|
194
|
+
score, since we are not interested in such short matches. */
|
|
195
|
+
var score uint = backwardReferenceScore(uint(len), backward)
|
|
196
|
+
if best_score < score {
|
|
197
|
+
best_score = score
|
|
198
|
+
best_len = uint(len)
|
|
199
|
+
out.len = best_len
|
|
200
|
+
out.distance = backward
|
|
201
|
+
out.score = best_score
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
bucket[uint32(num[key])&h.block_mask_] = uint32(cur_ix)
|
|
208
|
+
num[key]++
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
if min_score == out.score {
|
|
212
|
+
searchInStaticDictionary(dictionary, h, data[cur_ix_masked:], max_length, max_backward+gap, max_distance, out, false)
|
|
213
|
+
}
|
|
214
|
+
}
|
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
package brotli
|
|
2
|
+
|
|
3
|
+
import "encoding/binary"
|
|
4
|
+
|
|
5
|
+
/* Copyright 2010 Google Inc. All Rights Reserved.
|
|
6
|
+
|
|
7
|
+
Distributed under MIT license.
|
|
8
|
+
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
/* A (forgetful) hash table to the data seen by the compressor, to
|
|
12
|
+
help create backward references to previous data.
|
|
13
|
+
|
|
14
|
+
This is a hash map of fixed size (bucket_size_) to a ring buffer of
|
|
15
|
+
fixed size (block_size_). The ring buffer contains the last block_size_
|
|
16
|
+
index positions of the given hash key in the compressed data. */
|
|
17
|
+
func (*h6) HashTypeLength() uint {
|
|
18
|
+
return 8
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
func (*h6) StoreLookahead() uint {
|
|
22
|
+
return 8
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/* HashBytes is the function that chooses the bucket to place the address in. */
|
|
26
|
+
func hashBytesH6(data []byte, mask uint64, shift int) uint32 {
|
|
27
|
+
var h uint64 = (binary.LittleEndian.Uint64(data) & mask) * kHashMul64Long
|
|
28
|
+
|
|
29
|
+
/* The higher bits contain more mixture from the multiplication,
|
|
30
|
+
so we take our results from there. */
|
|
31
|
+
return uint32(h >> uint(shift))
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
type h6 struct {
|
|
35
|
+
hasherCommon
|
|
36
|
+
bucket_size_ uint
|
|
37
|
+
block_size_ uint
|
|
38
|
+
hash_shift_ int
|
|
39
|
+
hash_mask_ uint64
|
|
40
|
+
block_mask_ uint32
|
|
41
|
+
num []uint16
|
|
42
|
+
buckets []uint32
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
func (h *h6) Initialize(params *encoderParams) {
|
|
46
|
+
h.hash_shift_ = 64 - h.params.bucket_bits
|
|
47
|
+
h.hash_mask_ = (^(uint64(0))) >> uint(64-8*h.params.hash_len)
|
|
48
|
+
h.bucket_size_ = uint(1) << uint(h.params.bucket_bits)
|
|
49
|
+
h.block_size_ = uint(1) << uint(h.params.block_bits)
|
|
50
|
+
h.block_mask_ = uint32(h.block_size_ - 1)
|
|
51
|
+
h.num = make([]uint16, h.bucket_size_)
|
|
52
|
+
h.buckets = make([]uint32, h.block_size_*h.bucket_size_)
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
func (h *h6) Prepare(one_shot bool, input_size uint, data []byte) {
|
|
56
|
+
var num []uint16 = h.num
|
|
57
|
+
var partial_prepare_threshold uint = h.bucket_size_ >> 6
|
|
58
|
+
/* Partial preparation is 100 times slower (per socket). */
|
|
59
|
+
if one_shot && input_size <= partial_prepare_threshold {
|
|
60
|
+
var i uint
|
|
61
|
+
for i = 0; i < input_size; i++ {
|
|
62
|
+
var key uint32 = hashBytesH6(data[i:], h.hash_mask_, h.hash_shift_)
|
|
63
|
+
num[key] = 0
|
|
64
|
+
}
|
|
65
|
+
} else {
|
|
66
|
+
for i := 0; i < int(h.bucket_size_); i++ {
|
|
67
|
+
num[i] = 0
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/* Look at 4 bytes at &data[ix & mask].
|
|
73
|
+
Compute a hash from these, and store the value of ix at that position. */
|
|
74
|
+
func (h *h6) Store(data []byte, mask uint, ix uint) {
|
|
75
|
+
var num []uint16 = h.num
|
|
76
|
+
var key uint32 = hashBytesH6(data[ix&mask:], h.hash_mask_, h.hash_shift_)
|
|
77
|
+
var minor_ix uint = uint(num[key]) & uint(h.block_mask_)
|
|
78
|
+
var offset uint = minor_ix + uint(key<<uint(h.params.block_bits))
|
|
79
|
+
h.buckets[offset] = uint32(ix)
|
|
80
|
+
num[key]++
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
func (h *h6) StoreRange(data []byte, mask uint, ix_start uint, ix_end uint) {
|
|
84
|
+
var i uint
|
|
85
|
+
for i = ix_start; i < ix_end; i++ {
|
|
86
|
+
h.Store(data, mask, i)
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
func (h *h6) StitchToPreviousBlock(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint) {
|
|
91
|
+
if num_bytes >= h.HashTypeLength()-1 && position >= 3 {
|
|
92
|
+
/* Prepare the hashes for three last bytes of the last write.
|
|
93
|
+
These could not be calculated before, since they require knowledge
|
|
94
|
+
of both the previous and the current block. */
|
|
95
|
+
h.Store(ringbuffer, ringbuffer_mask, position-3)
|
|
96
|
+
h.Store(ringbuffer, ringbuffer_mask, position-2)
|
|
97
|
+
h.Store(ringbuffer, ringbuffer_mask, position-1)
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
func (h *h6) PrepareDistanceCache(distance_cache []int) {
|
|
102
|
+
prepareDistanceCache(distance_cache, h.params.num_last_distances_to_check)
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/* Find a longest backward match of &data[cur_ix] up to the length of
|
|
106
|
+
max_length and stores the position cur_ix in the hash table.
|
|
107
|
+
|
|
108
|
+
REQUIRES: PrepareDistanceCacheH6 must be invoked for current distance cache
|
|
109
|
+
values; if this method is invoked repeatedly with the same distance
|
|
110
|
+
cache values, it is enough to invoke PrepareDistanceCacheH6 once.
|
|
111
|
+
|
|
112
|
+
Does not look for matches longer than max_length.
|
|
113
|
+
Does not look for matches further away than max_backward.
|
|
114
|
+
Writes the best match into |out|.
|
|
115
|
+
|out|->score is updated only if a better match is found. */
|
|
116
|
+
func (h *h6) FindLongestMatch(dictionary *encoderDictionary, data []byte, ring_buffer_mask uint, distance_cache []int, cur_ix uint, max_length uint, max_backward uint, gap uint, max_distance uint, out *hasherSearchResult) {
|
|
117
|
+
var num []uint16 = h.num
|
|
118
|
+
var buckets []uint32 = h.buckets
|
|
119
|
+
var cur_ix_masked uint = cur_ix & ring_buffer_mask
|
|
120
|
+
var min_score uint = out.score
|
|
121
|
+
var best_score uint = out.score
|
|
122
|
+
var best_len uint = out.len
|
|
123
|
+
var i uint
|
|
124
|
+
var bucket []uint32
|
|
125
|
+
/* Don't accept a short copy from far away. */
|
|
126
|
+
out.len = 0
|
|
127
|
+
|
|
128
|
+
out.len_code_delta = 0
|
|
129
|
+
|
|
130
|
+
/* Try last distance first. */
|
|
131
|
+
for i = 0; i < uint(h.params.num_last_distances_to_check); i++ {
|
|
132
|
+
var backward uint = uint(distance_cache[i])
|
|
133
|
+
var prev_ix uint = uint(cur_ix - backward)
|
|
134
|
+
if prev_ix >= cur_ix {
|
|
135
|
+
continue
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
if backward > max_backward {
|
|
139
|
+
continue
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
prev_ix &= ring_buffer_mask
|
|
143
|
+
|
|
144
|
+
if cur_ix_masked+best_len > ring_buffer_mask || prev_ix+best_len > ring_buffer_mask || data[cur_ix_masked+best_len] != data[prev_ix+best_len] {
|
|
145
|
+
continue
|
|
146
|
+
}
|
|
147
|
+
{
|
|
148
|
+
var len uint = findMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length)
|
|
149
|
+
if len >= 3 || (len == 2 && i < 2) {
|
|
150
|
+
/* Comparing for >= 2 does not change the semantics, but just saves for
|
|
151
|
+
a few unnecessary binary logarithms in backward reference score,
|
|
152
|
+
since we are not interested in such short matches. */
|
|
153
|
+
var score uint = backwardReferenceScoreUsingLastDistance(uint(len))
|
|
154
|
+
if best_score < score {
|
|
155
|
+
if i != 0 {
|
|
156
|
+
score -= backwardReferencePenaltyUsingLastDistance(i)
|
|
157
|
+
}
|
|
158
|
+
if best_score < score {
|
|
159
|
+
best_score = score
|
|
160
|
+
best_len = uint(len)
|
|
161
|
+
out.len = best_len
|
|
162
|
+
out.distance = backward
|
|
163
|
+
out.score = best_score
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
{
|
|
170
|
+
var key uint32 = hashBytesH6(data[cur_ix_masked:], h.hash_mask_, h.hash_shift_)
|
|
171
|
+
bucket = buckets[key<<uint(h.params.block_bits):]
|
|
172
|
+
var down uint
|
|
173
|
+
if uint(num[key]) > h.block_size_ {
|
|
174
|
+
down = uint(num[key]) - h.block_size_
|
|
175
|
+
} else {
|
|
176
|
+
down = 0
|
|
177
|
+
}
|
|
178
|
+
for i = uint(num[key]); i > down; {
|
|
179
|
+
var prev_ix uint
|
|
180
|
+
i--
|
|
181
|
+
prev_ix = uint(bucket[uint32(i)&h.block_mask_])
|
|
182
|
+
var backward uint = cur_ix - prev_ix
|
|
183
|
+
if backward > max_backward {
|
|
184
|
+
break
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
prev_ix &= ring_buffer_mask
|
|
188
|
+
if cur_ix_masked+best_len > ring_buffer_mask || prev_ix+best_len > ring_buffer_mask || data[cur_ix_masked+best_len] != data[prev_ix+best_len] {
|
|
189
|
+
continue
|
|
190
|
+
}
|
|
191
|
+
{
|
|
192
|
+
var len uint = findMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length)
|
|
193
|
+
if len >= 4 {
|
|
194
|
+
/* Comparing for >= 3 does not change the semantics, but just saves
|
|
195
|
+
for a few unnecessary binary logarithms in backward reference
|
|
196
|
+
score, since we are not interested in such short matches. */
|
|
197
|
+
var score uint = backwardReferenceScore(uint(len), backward)
|
|
198
|
+
if best_score < score {
|
|
199
|
+
best_score = score
|
|
200
|
+
best_len = uint(len)
|
|
201
|
+
out.len = best_len
|
|
202
|
+
out.distance = backward
|
|
203
|
+
out.score = best_score
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
bucket[uint32(num[key])&h.block_mask_] = uint32(cur_ix)
|
|
210
|
+
num[key]++
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
if min_score == out.score {
|
|
214
|
+
searchInStaticDictionary(dictionary, h, data[cur_ix_masked:], max_length, max_backward+gap, max_distance, out, false)
|
|
215
|
+
}
|
|
216
|
+
}
|