typenative 0.3.1 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/ci.yml +2 -0
- package/CHANGELOG.md +258 -0
- package/CONTRIBUTION.md +1 -1
- package/README.md +17 -1
- package/bin/cli.js +26 -11
- package/bin/go/bridge.js +74 -14
- package/bin/go/convert.js +59 -1
- package/bin/go/metadata.js +13 -4
- package/bin/go/overrides.js +4 -6
- package/bin/go/stdlib.generated.js +2 -0
- package/bin/index.js +338 -47
- package/bin/main.js +36 -14
- package/bin/node/assert.js +2 -2
- package/bin/node/buffer.js +9 -0
- package/bin/node/child_process.js +51 -3
- package/bin/node/cluster.js +41 -3
- package/bin/node/core.js +1 -1
- package/bin/node/crypto.js +205 -4
- package/bin/node/dgram.js +230 -6
- package/bin/node/dns.js +471 -8
- package/bin/node/errno_tables.js +445 -0
- package/bin/node/events.js +1 -1
- package/bin/node/fs.js +22 -15
- package/bin/node/fs_watch.js +407 -0
- package/bin/node/http.js +128 -30
- package/bin/node/http2.js +37 -7
- package/bin/node/module_v8.js +142 -0
- package/bin/node/net.js +9 -6
- package/bin/node/process.js +242 -0
- package/bin/node/readline.js +272 -1
- package/bin/node/registry.js +3 -1
- package/bin/node/sqlite.js +60 -12
- package/bin/node/stream.js +145 -10
- package/bin/node/test.js +311 -19
- package/bin/node/timers.js +1 -1
- package/bin/node/typed_arrays.js +608 -0
- package/bin/node/url.js +23 -5
- package/bin/node/util.js +21 -1
- package/bin/node/util_extra.js +6 -30
- package/bin/node/worker_threads.js +79 -19
- package/bin/node/zlib.js +131 -16
- package/bin/transpiler.js +4729 -616
- package/bin/typecheck-node.js +1 -0
- package/bin/version.js +1 -1
- package/conformance.md +1030 -877
- package/examples/README.md +54 -0
- package/examples/fibonacci/README.md +89 -0
- package/examples/fibonacci/bench.mjs +87 -0
- package/examples/fibonacci/main.ts +21 -0
- package/examples/hello-world/README.md +62 -0
- package/examples/hello-world/bench.mjs +66 -0
- package/examples/hello-world/main.ts +7 -0
- package/examples/json-server/README.md +127 -0
- package/examples/json-server/bench.mjs +263 -0
- package/examples/json-server/main.ts +67 -0
- package/govendor/github.com/andybalholm/brotli/LICENSE +19 -0
- package/govendor/github.com/andybalholm/brotli/README.md +19 -0
- package/govendor/github.com/andybalholm/brotli/backward_references.go +185 -0
- package/govendor/github.com/andybalholm/brotli/backward_references_hq.go +796 -0
- package/govendor/github.com/andybalholm/brotli/bit_cost.go +436 -0
- package/govendor/github.com/andybalholm/brotli/bit_reader.go +266 -0
- package/govendor/github.com/andybalholm/brotli/bitwriter.go +56 -0
- package/govendor/github.com/andybalholm/brotli/block_splitter.go +144 -0
- package/govendor/github.com/andybalholm/brotli/block_splitter_command.go +434 -0
- package/govendor/github.com/andybalholm/brotli/block_splitter_distance.go +433 -0
- package/govendor/github.com/andybalholm/brotli/block_splitter_literal.go +433 -0
- package/govendor/github.com/andybalholm/brotli/brotli_bit_stream.go +1435 -0
- package/govendor/github.com/andybalholm/brotli/cluster.go +30 -0
- package/govendor/github.com/andybalholm/brotli/cluster_command.go +164 -0
- package/govendor/github.com/andybalholm/brotli/cluster_distance.go +326 -0
- package/govendor/github.com/andybalholm/brotli/cluster_literal.go +326 -0
- package/govendor/github.com/andybalholm/brotli/command.go +254 -0
- package/govendor/github.com/andybalholm/brotli/compress_fragment.go +834 -0
- package/govendor/github.com/andybalholm/brotli/compress_fragment_two_pass.go +775 -0
- package/govendor/github.com/andybalholm/brotli/constants.go +77 -0
- package/govendor/github.com/andybalholm/brotli/context.go +2176 -0
- package/govendor/github.com/andybalholm/brotli/decode.go +2570 -0
- package/govendor/github.com/andybalholm/brotli/dictionary.go +122890 -0
- package/govendor/github.com/andybalholm/brotli/dictionary_hash.go +32779 -0
- package/govendor/github.com/andybalholm/brotli/encode.go +1220 -0
- package/govendor/github.com/andybalholm/brotli/encoder.go +177 -0
- package/govendor/github.com/andybalholm/brotli/encoder_dict.go +22 -0
- package/govendor/github.com/andybalholm/brotli/encoder_fast.go +201 -0
- package/govendor/github.com/andybalholm/brotli/entropy_encode.go +592 -0
- package/govendor/github.com/andybalholm/brotli/entropy_encode_static.go +4399 -0
- package/govendor/github.com/andybalholm/brotli/fast_log.go +290 -0
- package/govendor/github.com/andybalholm/brotli/find_match_length.go +45 -0
- package/govendor/github.com/andybalholm/brotli/go.mod +3 -0
- package/govendor/github.com/andybalholm/brotli/h10.go +287 -0
- package/govendor/github.com/andybalholm/brotli/h5.go +214 -0
- package/govendor/github.com/andybalholm/brotli/h6.go +216 -0
- package/govendor/github.com/andybalholm/brotli/hash.go +342 -0
- package/govendor/github.com/andybalholm/brotli/hash_composite.go +93 -0
- package/govendor/github.com/andybalholm/brotli/hash_forgetful_chain.go +252 -0
- package/govendor/github.com/andybalholm/brotli/hash_longest_match_quickly.go +214 -0
- package/govendor/github.com/andybalholm/brotli/hash_rolling.go +168 -0
- package/govendor/github.com/andybalholm/brotli/histogram.go +226 -0
- package/govendor/github.com/andybalholm/brotli/http.go +201 -0
- package/govendor/github.com/andybalholm/brotli/huffman.go +653 -0
- package/govendor/github.com/andybalholm/brotli/literal_cost.go +182 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/bargain1.go +212 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/bargain2.go +248 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/bargain3.go +278 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/emitter.go +34 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/m0.go +175 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/m4.go +339 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/matchfinder.go +103 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/pathfinder.go +328 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/textencoder.go +53 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/trio.go +327 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/zdfast.go +258 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/zfast.go +206 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/zm.go +302 -0
- package/govendor/github.com/andybalholm/brotli/memory.go +66 -0
- package/govendor/github.com/andybalholm/brotli/metablock.go +574 -0
- package/govendor/github.com/andybalholm/brotli/metablock_command.go +165 -0
- package/govendor/github.com/andybalholm/brotli/metablock_distance.go +165 -0
- package/govendor/github.com/andybalholm/brotli/metablock_literal.go +165 -0
- package/govendor/github.com/andybalholm/brotli/params.go +37 -0
- package/govendor/github.com/andybalholm/brotli/platform.go +103 -0
- package/govendor/github.com/andybalholm/brotli/prefix.go +30 -0
- package/govendor/github.com/andybalholm/brotli/prefix_dec.go +723 -0
- package/govendor/github.com/andybalholm/brotli/quality.go +196 -0
- package/govendor/github.com/andybalholm/brotli/reader.go +127 -0
- package/govendor/github.com/andybalholm/brotli/ringbuffer.go +134 -0
- package/govendor/github.com/andybalholm/brotli/state.go +313 -0
- package/govendor/github.com/andybalholm/brotli/static_dict.go +662 -0
- package/govendor/github.com/andybalholm/brotli/static_dict_lut.go +75094 -0
- package/govendor/github.com/andybalholm/brotli/symbol_list.go +22 -0
- package/govendor/github.com/andybalholm/brotli/transform.go +641 -0
- package/govendor/github.com/andybalholm/brotli/utf8_util.go +70 -0
- package/govendor/github.com/andybalholm/brotli/util.go +7 -0
- package/govendor/github.com/andybalholm/brotli/write_bits.go +52 -0
- package/govendor/github.com/andybalholm/brotli/writer.go +162 -0
- package/govendor/github.com/fsnotify/fsnotify/.mailmap +2 -0
- package/govendor/github.com/fsnotify/fsnotify/CHANGELOG.md +651 -0
- package/govendor/github.com/fsnotify/fsnotify/CONTRIBUTING.md +147 -0
- package/govendor/github.com/fsnotify/fsnotify/LICENSE +25 -0
- package/govendor/github.com/fsnotify/fsnotify/README.md +214 -0
- package/govendor/github.com/fsnotify/fsnotify/backend_fen.go +469 -0
- package/govendor/github.com/fsnotify/fsnotify/backend_inotify.go +597 -0
- package/govendor/github.com/fsnotify/fsnotify/backend_kqueue.go +735 -0
- package/govendor/github.com/fsnotify/fsnotify/backend_other.go +22 -0
- package/govendor/github.com/fsnotify/fsnotify/backend_windows.go +702 -0
- package/govendor/github.com/fsnotify/fsnotify/fsnotify.go +526 -0
- package/govendor/github.com/fsnotify/fsnotify/go.mod +3 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/darwin.go +20 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_darwin.go +15 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_dragonfly.go +15 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_freebsd.go +20 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_kqueue.go +32 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_linux.go +56 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_netbsd.go +15 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_openbsd.go +16 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_solaris.go +45 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_windows.go +40 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/freebsd.go +20 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/internal.go +2 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/unix.go +20 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/unix2.go +25 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/windows.go +40 -0
- package/govendor/github.com/fsnotify/fsnotify/shared.go +64 -0
- package/govendor/github.com/fsnotify/fsnotify/staticcheck.conf +3 -0
- package/govendor/github.com/fsnotify/fsnotify/system_bsd.go +7 -0
- package/govendor/github.com/fsnotify/fsnotify/system_darwin.go +8 -0
- package/govendor/manifest.txt +3 -1
- package/package.json +1 -1
- package/types/typenative-go.d.ts +23 -0
- package/types/typenative-npm.d.ts +241 -5
- package/types/typenative.d.ts +302 -12
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
package brotli
|
|
2
|
+
|
|
3
|
+
import "encoding/binary"
|
|
4
|
+
|
|
5
|
+
/* Copyright 2010 Google Inc. All Rights Reserved.
|
|
6
|
+
|
|
7
|
+
Distributed under MIT license.
|
|
8
|
+
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
/* For BUCKET_SWEEP == 1, enabling the dictionary lookup makes compression
|
|
12
|
+
a little faster (0.5% - 1%) and it compresses 0.15% better on small text
|
|
13
|
+
and HTML inputs. */
|
|
14
|
+
|
|
15
|
+
func (*hashLongestMatchQuickly) HashTypeLength() uint {
|
|
16
|
+
return 8
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
func (*hashLongestMatchQuickly) StoreLookahead() uint {
|
|
20
|
+
return 8
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/* HashBytes is the function that chooses the bucket to place
|
|
24
|
+
the address in. The HashLongestMatch and hashLongestMatchQuickly
|
|
25
|
+
classes have separate, different implementations of hashing. */
|
|
26
|
+
func (h *hashLongestMatchQuickly) HashBytes(data []byte) uint32 {
|
|
27
|
+
var hash uint64 = ((binary.LittleEndian.Uint64(data) << (64 - 8*h.hashLen)) * kHashMul64)
|
|
28
|
+
|
|
29
|
+
/* The higher bits contain more mixture from the multiplication,
|
|
30
|
+
so we take our results from there. */
|
|
31
|
+
return uint32(hash >> (64 - h.bucketBits))
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/* A (forgetful) hash table to the data seen by the compressor, to
|
|
35
|
+
help create backward references to previous data.
|
|
36
|
+
|
|
37
|
+
This is a hash map of fixed size (1 << 16). Starting from the
|
|
38
|
+
given index, 1 buckets are used to store values of a key. */
|
|
39
|
+
type hashLongestMatchQuickly struct {
|
|
40
|
+
hasherCommon
|
|
41
|
+
|
|
42
|
+
bucketBits uint
|
|
43
|
+
bucketSweep int
|
|
44
|
+
hashLen uint
|
|
45
|
+
useDictionary bool
|
|
46
|
+
|
|
47
|
+
buckets []uint32
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
func (h *hashLongestMatchQuickly) Initialize(params *encoderParams) {
|
|
51
|
+
h.buckets = make([]uint32, 1<<h.bucketBits+h.bucketSweep)
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
func (h *hashLongestMatchQuickly) Prepare(one_shot bool, input_size uint, data []byte) {
|
|
55
|
+
var partial_prepare_threshold uint = (4 << h.bucketBits) >> 7
|
|
56
|
+
/* Partial preparation is 100 times slower (per socket). */
|
|
57
|
+
if one_shot && input_size <= partial_prepare_threshold {
|
|
58
|
+
var i uint
|
|
59
|
+
for i = 0; i < input_size; i++ {
|
|
60
|
+
var key uint32 = h.HashBytes(data[i:])
|
|
61
|
+
for j := 0; j < h.bucketSweep; j++ {
|
|
62
|
+
h.buckets[key+uint32(j)] = 0
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
} else {
|
|
66
|
+
/* It is not strictly necessary to fill this buffer here, but
|
|
67
|
+
not filling will make the results of the compression stochastic
|
|
68
|
+
(but correct). This is because random data would cause the
|
|
69
|
+
system to find accidentally good backward references here and there. */
|
|
70
|
+
for i := range h.buckets {
|
|
71
|
+
h.buckets[i] = 0
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/* Look at 5 bytes at &data[ix & mask].
|
|
77
|
+
Compute a hash from these, and store the value somewhere within
|
|
78
|
+
[ix .. ix+3]. */
|
|
79
|
+
func (h *hashLongestMatchQuickly) Store(data []byte, mask uint, ix uint) {
|
|
80
|
+
var key uint32 = h.HashBytes(data[ix&mask:])
|
|
81
|
+
var off uint32 = uint32(ix>>3) % uint32(h.bucketSweep)
|
|
82
|
+
/* Wiggle the value with the bucket sweep range. */
|
|
83
|
+
h.buckets[key+off] = uint32(ix)
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
func (h *hashLongestMatchQuickly) StoreRange(data []byte, mask uint, ix_start uint, ix_end uint) {
|
|
87
|
+
var i uint
|
|
88
|
+
for i = ix_start; i < ix_end; i++ {
|
|
89
|
+
h.Store(data, mask, i)
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
func (h *hashLongestMatchQuickly) StitchToPreviousBlock(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint) {
|
|
94
|
+
if num_bytes >= h.HashTypeLength()-1 && position >= 3 {
|
|
95
|
+
/* Prepare the hashes for three last bytes of the last write.
|
|
96
|
+
These could not be calculated before, since they require knowledge
|
|
97
|
+
of both the previous and the current block. */
|
|
98
|
+
h.Store(ringbuffer, ringbuffer_mask, position-3)
|
|
99
|
+
h.Store(ringbuffer, ringbuffer_mask, position-2)
|
|
100
|
+
h.Store(ringbuffer, ringbuffer_mask, position-1)
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
func (*hashLongestMatchQuickly) PrepareDistanceCache(distance_cache []int) {
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/* Find a longest backward match of &data[cur_ix & ring_buffer_mask]
|
|
108
|
+
up to the length of max_length and stores the position cur_ix in the
|
|
109
|
+
hash table.
|
|
110
|
+
|
|
111
|
+
Does not look for matches longer than max_length.
|
|
112
|
+
Does not look for matches further away than max_backward.
|
|
113
|
+
Writes the best match into |out|.
|
|
114
|
+
|out|->score is updated only if a better match is found. */
|
|
115
|
+
func (h *hashLongestMatchQuickly) FindLongestMatch(dictionary *encoderDictionary, data []byte, ring_buffer_mask uint, distance_cache []int, cur_ix uint, max_length uint, max_backward uint, gap uint, max_distance uint, out *hasherSearchResult) {
|
|
116
|
+
var best_len_in uint = out.len
|
|
117
|
+
var cur_ix_masked uint = cur_ix & ring_buffer_mask
|
|
118
|
+
var key uint32 = h.HashBytes(data[cur_ix_masked:])
|
|
119
|
+
var compare_char int = int(data[cur_ix_masked+best_len_in])
|
|
120
|
+
var min_score uint = out.score
|
|
121
|
+
var best_score uint = out.score
|
|
122
|
+
var best_len uint = best_len_in
|
|
123
|
+
var cached_backward uint = uint(distance_cache[0])
|
|
124
|
+
var prev_ix uint = cur_ix - cached_backward
|
|
125
|
+
var bucket []uint32
|
|
126
|
+
out.len_code_delta = 0
|
|
127
|
+
if prev_ix < cur_ix {
|
|
128
|
+
prev_ix &= uint(uint32(ring_buffer_mask))
|
|
129
|
+
if compare_char == int(data[prev_ix+best_len]) {
|
|
130
|
+
var len uint = findMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length)
|
|
131
|
+
if len >= 4 {
|
|
132
|
+
var score uint = backwardReferenceScoreUsingLastDistance(uint(len))
|
|
133
|
+
if best_score < score {
|
|
134
|
+
best_score = score
|
|
135
|
+
best_len = uint(len)
|
|
136
|
+
out.len = uint(len)
|
|
137
|
+
out.distance = cached_backward
|
|
138
|
+
out.score = best_score
|
|
139
|
+
compare_char = int(data[cur_ix_masked+best_len])
|
|
140
|
+
if h.bucketSweep == 1 {
|
|
141
|
+
h.buckets[key] = uint32(cur_ix)
|
|
142
|
+
return
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
if h.bucketSweep == 1 {
|
|
150
|
+
var backward uint
|
|
151
|
+
var len uint
|
|
152
|
+
|
|
153
|
+
/* Only one to look for, don't bother to prepare for a loop. */
|
|
154
|
+
prev_ix = uint(h.buckets[key])
|
|
155
|
+
|
|
156
|
+
h.buckets[key] = uint32(cur_ix)
|
|
157
|
+
backward = cur_ix - prev_ix
|
|
158
|
+
prev_ix &= uint(uint32(ring_buffer_mask))
|
|
159
|
+
if compare_char != int(data[prev_ix+best_len_in]) {
|
|
160
|
+
return
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
if backward == 0 || backward > max_backward {
|
|
164
|
+
return
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
len = findMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length)
|
|
168
|
+
if len >= 4 {
|
|
169
|
+
var score uint = backwardReferenceScore(uint(len), backward)
|
|
170
|
+
if best_score < score {
|
|
171
|
+
out.len = uint(len)
|
|
172
|
+
out.distance = backward
|
|
173
|
+
out.score = score
|
|
174
|
+
return
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
} else {
|
|
178
|
+
bucket = h.buckets[key:]
|
|
179
|
+
var i int
|
|
180
|
+
prev_ix = uint(bucket[0])
|
|
181
|
+
bucket = bucket[1:]
|
|
182
|
+
for i = 0; i < h.bucketSweep; (func() { i++; tmp3 := bucket; bucket = bucket[1:]; prev_ix = uint(tmp3[0]) })() {
|
|
183
|
+
var backward uint = cur_ix - prev_ix
|
|
184
|
+
var len uint
|
|
185
|
+
prev_ix &= uint(uint32(ring_buffer_mask))
|
|
186
|
+
if compare_char != int(data[prev_ix+best_len]) {
|
|
187
|
+
continue
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
if backward == 0 || backward > max_backward {
|
|
191
|
+
continue
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
len = findMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length)
|
|
195
|
+
if len >= 4 {
|
|
196
|
+
var score uint = backwardReferenceScore(uint(len), backward)
|
|
197
|
+
if best_score < score {
|
|
198
|
+
best_score = score
|
|
199
|
+
best_len = uint(len)
|
|
200
|
+
out.len = best_len
|
|
201
|
+
out.distance = backward
|
|
202
|
+
out.score = score
|
|
203
|
+
compare_char = int(data[cur_ix_masked+best_len])
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
if h.useDictionary && min_score == out.score {
|
|
210
|
+
searchInStaticDictionary(dictionary, h, data[cur_ix_masked:], max_length, max_backward+gap, max_distance, out, true)
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
h.buckets[key+uint32((cur_ix>>3)%uint(h.bucketSweep))] = uint32(cur_ix)
|
|
214
|
+
}
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
package brotli
|
|
2
|
+
|
|
3
|
+
/* Copyright 2018 Google Inc. All Rights Reserved.
|
|
4
|
+
|
|
5
|
+
Distributed under MIT license.
|
|
6
|
+
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
/* NOTE: this hasher does not search in the dictionary. It is used as
|
|
10
|
+
backup-hasher, the main hasher already searches in it. */
|
|
11
|
+
|
|
12
|
+
const kRollingHashMul32 uint32 = 69069
|
|
13
|
+
|
|
14
|
+
const kInvalidPosHashRolling uint32 = 0xffffffff
|
|
15
|
+
|
|
16
|
+
/* This hasher uses a longer forward length, but returning a higher value here
|
|
17
|
+
will hurt compression by the main hasher when combined with a composite
|
|
18
|
+
hasher. The hasher tests for forward itself instead. */
|
|
19
|
+
func (*hashRolling) HashTypeLength() uint {
|
|
20
|
+
return 4
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
func (*hashRolling) StoreLookahead() uint {
|
|
24
|
+
return 4
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/* Computes a code from a single byte. A lookup table of 256 values could be
|
|
28
|
+
used, but simply adding 1 works about as good. */
|
|
29
|
+
func (*hashRolling) HashByte(b byte) uint32 {
|
|
30
|
+
return uint32(b) + 1
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
func (h *hashRolling) HashRollingFunctionInitial(state uint32, add byte, factor uint32) uint32 {
|
|
34
|
+
return uint32(factor*state + h.HashByte(add))
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
func (h *hashRolling) HashRollingFunction(state uint32, add byte, rem byte, factor uint32, factor_remove uint32) uint32 {
|
|
38
|
+
return uint32(factor*state + h.HashByte(add) - factor_remove*h.HashByte(rem))
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/* Rolling hash for long distance long string matches. Stores one position
|
|
42
|
+
per bucket, bucket key is computed over a long region. */
|
|
43
|
+
type hashRolling struct {
|
|
44
|
+
hasherCommon
|
|
45
|
+
|
|
46
|
+
jump int
|
|
47
|
+
|
|
48
|
+
state uint32
|
|
49
|
+
table []uint32
|
|
50
|
+
next_ix uint
|
|
51
|
+
factor uint32
|
|
52
|
+
factor_remove uint32
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
func (h *hashRolling) Initialize(params *encoderParams) {
|
|
56
|
+
h.state = 0
|
|
57
|
+
h.next_ix = 0
|
|
58
|
+
|
|
59
|
+
h.factor = kRollingHashMul32
|
|
60
|
+
|
|
61
|
+
/* Compute the factor of the oldest byte to remove: factor**steps modulo
|
|
62
|
+
0xffffffff (the multiplications rely on 32-bit overflow) */
|
|
63
|
+
h.factor_remove = 1
|
|
64
|
+
|
|
65
|
+
for i := 0; i < 32; i += h.jump {
|
|
66
|
+
h.factor_remove *= h.factor
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
h.table = make([]uint32, 16777216)
|
|
70
|
+
for i := 0; i < 16777216; i++ {
|
|
71
|
+
h.table[i] = kInvalidPosHashRolling
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
func (h *hashRolling) Prepare(one_shot bool, input_size uint, data []byte) {
|
|
76
|
+
/* Too small size, cannot use this hasher. */
|
|
77
|
+
if input_size < 32 {
|
|
78
|
+
return
|
|
79
|
+
}
|
|
80
|
+
h.state = 0
|
|
81
|
+
for i := 0; i < 32; i += h.jump {
|
|
82
|
+
h.state = h.HashRollingFunctionInitial(h.state, data[i], h.factor)
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
func (*hashRolling) Store(data []byte, mask uint, ix uint) {
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
func (*hashRolling) StoreRange(data []byte, mask uint, ix_start uint, ix_end uint) {
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
func (h *hashRolling) StitchToPreviousBlock(num_bytes uint, position uint, ringbuffer []byte, ring_buffer_mask uint) {
|
|
93
|
+
var position_masked uint
|
|
94
|
+
/* In this case we must re-initialize the hasher from scratch from the
|
|
95
|
+
current position. */
|
|
96
|
+
|
|
97
|
+
var available uint = num_bytes
|
|
98
|
+
if position&uint(h.jump-1) != 0 {
|
|
99
|
+
var diff uint = uint(h.jump) - (position & uint(h.jump-1))
|
|
100
|
+
if diff > available {
|
|
101
|
+
available = 0
|
|
102
|
+
} else {
|
|
103
|
+
available = available - diff
|
|
104
|
+
}
|
|
105
|
+
position += diff
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
position_masked = position & ring_buffer_mask
|
|
109
|
+
|
|
110
|
+
/* wrapping around ringbuffer not handled. */
|
|
111
|
+
if available > ring_buffer_mask-position_masked {
|
|
112
|
+
available = ring_buffer_mask - position_masked
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
h.Prepare(false, available, ringbuffer[position&ring_buffer_mask:])
|
|
116
|
+
h.next_ix = position
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
func (*hashRolling) PrepareDistanceCache(distance_cache []int) {
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
func (h *hashRolling) FindLongestMatch(dictionary *encoderDictionary, data []byte, ring_buffer_mask uint, distance_cache []int, cur_ix uint, max_length uint, max_backward uint, gap uint, max_distance uint, out *hasherSearchResult) {
|
|
123
|
+
var cur_ix_masked uint = cur_ix & ring_buffer_mask
|
|
124
|
+
var pos uint = h.next_ix
|
|
125
|
+
|
|
126
|
+
if cur_ix&uint(h.jump-1) != 0 {
|
|
127
|
+
return
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/* Not enough lookahead */
|
|
131
|
+
if max_length < 32 {
|
|
132
|
+
return
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
for pos = h.next_ix; pos <= cur_ix; pos += uint(h.jump) {
|
|
136
|
+
var code uint32 = h.state & ((16777216 * 64) - 1)
|
|
137
|
+
var rem byte = data[pos&ring_buffer_mask]
|
|
138
|
+
var add byte = data[(pos+32)&ring_buffer_mask]
|
|
139
|
+
var found_ix uint = uint(kInvalidPosHashRolling)
|
|
140
|
+
|
|
141
|
+
h.state = h.HashRollingFunction(h.state, add, rem, h.factor, h.factor_remove)
|
|
142
|
+
|
|
143
|
+
if code < 16777216 {
|
|
144
|
+
found_ix = uint(h.table[code])
|
|
145
|
+
h.table[code] = uint32(pos)
|
|
146
|
+
if pos == cur_ix && uint32(found_ix) != kInvalidPosHashRolling {
|
|
147
|
+
/* The cast to 32-bit makes backward distances up to 4GB work even
|
|
148
|
+
if cur_ix is above 4GB, despite using 32-bit values in the table. */
|
|
149
|
+
var backward uint = uint(uint32(cur_ix - found_ix))
|
|
150
|
+
if backward <= max_backward {
|
|
151
|
+
var found_ix_masked uint = found_ix & ring_buffer_mask
|
|
152
|
+
var len uint = findMatchLengthWithLimit(data[found_ix_masked:], data[cur_ix_masked:], max_length)
|
|
153
|
+
if len >= 4 && len > out.len {
|
|
154
|
+
var score uint = backwardReferenceScore(uint(len), backward)
|
|
155
|
+
if score > out.score {
|
|
156
|
+
out.len = uint(len)
|
|
157
|
+
out.distance = backward
|
|
158
|
+
out.score = score
|
|
159
|
+
out.len_code_delta = 0
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
h.next_ix = cur_ix + uint(h.jump)
|
|
168
|
+
}
|
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
package brotli
|
|
2
|
+
|
|
3
|
+
import "math"
|
|
4
|
+
|
|
5
|
+
/* The distance symbols effectively used by "Large Window Brotli" (32-bit). */
|
|
6
|
+
const numHistogramDistanceSymbols = 544
|
|
7
|
+
|
|
8
|
+
type histogramLiteral struct {
|
|
9
|
+
data_ [numLiteralSymbols]uint32
|
|
10
|
+
total_count_ uint
|
|
11
|
+
bit_cost_ float64
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
func histogramClearLiteral(self *histogramLiteral) {
|
|
15
|
+
self.data_ = [numLiteralSymbols]uint32{}
|
|
16
|
+
self.total_count_ = 0
|
|
17
|
+
self.bit_cost_ = math.MaxFloat64
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
func clearHistogramsLiteral(array []histogramLiteral, length uint) {
|
|
21
|
+
var i uint
|
|
22
|
+
for i = 0; i < length; i++ {
|
|
23
|
+
histogramClearLiteral(&array[i:][0])
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
func histogramAddLiteral(self *histogramLiteral, val uint) {
|
|
28
|
+
self.data_[val]++
|
|
29
|
+
self.total_count_++
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
func histogramAddVectorLiteral(self *histogramLiteral, p []byte, n uint) {
|
|
33
|
+
self.total_count_ += n
|
|
34
|
+
n += 1
|
|
35
|
+
for {
|
|
36
|
+
n--
|
|
37
|
+
if n == 0 {
|
|
38
|
+
break
|
|
39
|
+
}
|
|
40
|
+
self.data_[p[0]]++
|
|
41
|
+
p = p[1:]
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
func histogramAddHistogramLiteral(self *histogramLiteral, v *histogramLiteral) {
|
|
46
|
+
var i uint
|
|
47
|
+
self.total_count_ += v.total_count_
|
|
48
|
+
for i = 0; i < numLiteralSymbols; i++ {
|
|
49
|
+
self.data_[i] += v.data_[i]
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
func histogramDataSizeLiteral() uint {
|
|
54
|
+
return numLiteralSymbols
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
type histogramCommand struct {
|
|
58
|
+
data_ [numCommandSymbols]uint32
|
|
59
|
+
total_count_ uint
|
|
60
|
+
bit_cost_ float64
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
func histogramClearCommand(self *histogramCommand) {
|
|
64
|
+
self.data_ = [numCommandSymbols]uint32{}
|
|
65
|
+
self.total_count_ = 0
|
|
66
|
+
self.bit_cost_ = math.MaxFloat64
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
func clearHistogramsCommand(array []histogramCommand, length uint) {
|
|
70
|
+
var i uint
|
|
71
|
+
for i = 0; i < length; i++ {
|
|
72
|
+
histogramClearCommand(&array[i:][0])
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
func histogramAddCommand(self *histogramCommand, val uint) {
|
|
77
|
+
self.data_[val]++
|
|
78
|
+
self.total_count_++
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
func histogramAddVectorCommand(self *histogramCommand, p []uint16, n uint) {
|
|
82
|
+
self.total_count_ += n
|
|
83
|
+
n += 1
|
|
84
|
+
for {
|
|
85
|
+
n--
|
|
86
|
+
if n == 0 {
|
|
87
|
+
break
|
|
88
|
+
}
|
|
89
|
+
self.data_[p[0]]++
|
|
90
|
+
p = p[1:]
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
func histogramAddHistogramCommand(self *histogramCommand, v *histogramCommand) {
|
|
95
|
+
var i uint
|
|
96
|
+
self.total_count_ += v.total_count_
|
|
97
|
+
for i = 0; i < numCommandSymbols; i++ {
|
|
98
|
+
self.data_[i] += v.data_[i]
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
func histogramDataSizeCommand() uint {
|
|
103
|
+
return numCommandSymbols
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
type histogramDistance struct {
|
|
107
|
+
data_ [numDistanceSymbols]uint32
|
|
108
|
+
total_count_ uint
|
|
109
|
+
bit_cost_ float64
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
func histogramClearDistance(self *histogramDistance) {
|
|
113
|
+
self.data_ = [numDistanceSymbols]uint32{}
|
|
114
|
+
self.total_count_ = 0
|
|
115
|
+
self.bit_cost_ = math.MaxFloat64
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
func clearHistogramsDistance(array []histogramDistance, length uint) {
|
|
119
|
+
var i uint
|
|
120
|
+
for i = 0; i < length; i++ {
|
|
121
|
+
histogramClearDistance(&array[i:][0])
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
func histogramAddDistance(self *histogramDistance, val uint) {
|
|
126
|
+
self.data_[val]++
|
|
127
|
+
self.total_count_++
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
func histogramAddVectorDistance(self *histogramDistance, p []uint16, n uint) {
|
|
131
|
+
self.total_count_ += n
|
|
132
|
+
n += 1
|
|
133
|
+
for {
|
|
134
|
+
n--
|
|
135
|
+
if n == 0 {
|
|
136
|
+
break
|
|
137
|
+
}
|
|
138
|
+
self.data_[p[0]]++
|
|
139
|
+
p = p[1:]
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
func histogramAddHistogramDistance(self *histogramDistance, v *histogramDistance) {
|
|
144
|
+
var i uint
|
|
145
|
+
self.total_count_ += v.total_count_
|
|
146
|
+
for i = 0; i < numDistanceSymbols; i++ {
|
|
147
|
+
self.data_[i] += v.data_[i]
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
func histogramDataSizeDistance() uint {
|
|
152
|
+
return numDistanceSymbols
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
type blockSplitIterator struct {
|
|
156
|
+
split_ *blockSplit
|
|
157
|
+
idx_ uint
|
|
158
|
+
type_ uint
|
|
159
|
+
length_ uint
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
func initBlockSplitIterator(self *blockSplitIterator, split *blockSplit) {
|
|
163
|
+
self.split_ = split
|
|
164
|
+
self.idx_ = 0
|
|
165
|
+
self.type_ = 0
|
|
166
|
+
if len(split.lengths) > 0 {
|
|
167
|
+
self.length_ = uint(split.lengths[0])
|
|
168
|
+
} else {
|
|
169
|
+
self.length_ = 0
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
func blockSplitIteratorNext(self *blockSplitIterator) {
|
|
174
|
+
if self.length_ == 0 {
|
|
175
|
+
self.idx_++
|
|
176
|
+
self.type_ = uint(self.split_.types[self.idx_])
|
|
177
|
+
self.length_ = uint(self.split_.lengths[self.idx_])
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
self.length_--
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
func buildHistogramsWithContext(cmds []command, literal_split *blockSplit, insert_and_copy_split *blockSplit, dist_split *blockSplit, ringbuffer []byte, start_pos uint, mask uint, prev_byte byte, prev_byte2 byte, context_modes []int, literal_histograms []histogramLiteral, insert_and_copy_histograms []histogramCommand, copy_dist_histograms []histogramDistance) {
|
|
184
|
+
var pos uint = start_pos
|
|
185
|
+
var literal_it blockSplitIterator
|
|
186
|
+
var insert_and_copy_it blockSplitIterator
|
|
187
|
+
var dist_it blockSplitIterator
|
|
188
|
+
|
|
189
|
+
initBlockSplitIterator(&literal_it, literal_split)
|
|
190
|
+
initBlockSplitIterator(&insert_and_copy_it, insert_and_copy_split)
|
|
191
|
+
initBlockSplitIterator(&dist_it, dist_split)
|
|
192
|
+
for i := range cmds {
|
|
193
|
+
var cmd *command = &cmds[i]
|
|
194
|
+
var j uint
|
|
195
|
+
blockSplitIteratorNext(&insert_and_copy_it)
|
|
196
|
+
histogramAddCommand(&insert_and_copy_histograms[insert_and_copy_it.type_], uint(cmd.cmd_prefix_))
|
|
197
|
+
|
|
198
|
+
/* TODO: unwrap iterator blocks. */
|
|
199
|
+
for j = uint(cmd.insert_len_); j != 0; j-- {
|
|
200
|
+
var context uint
|
|
201
|
+
blockSplitIteratorNext(&literal_it)
|
|
202
|
+
context = literal_it.type_
|
|
203
|
+
if context_modes != nil {
|
|
204
|
+
var lut contextLUT = getContextLUT(context_modes[context])
|
|
205
|
+
context = (context << literalContextBits) + uint(getContext(prev_byte, prev_byte2, lut))
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
histogramAddLiteral(&literal_histograms[context], uint(ringbuffer[pos&mask]))
|
|
209
|
+
prev_byte2 = prev_byte
|
|
210
|
+
prev_byte = ringbuffer[pos&mask]
|
|
211
|
+
pos++
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
pos += uint(commandCopyLen(cmd))
|
|
215
|
+
if commandCopyLen(cmd) != 0 {
|
|
216
|
+
prev_byte2 = ringbuffer[(pos-2)&mask]
|
|
217
|
+
prev_byte = ringbuffer[(pos-1)&mask]
|
|
218
|
+
if cmd.cmd_prefix_ >= 128 {
|
|
219
|
+
var context uint
|
|
220
|
+
blockSplitIteratorNext(&dist_it)
|
|
221
|
+
context = uint(uint32(dist_it.type_<<distanceContextBits) + commandDistanceContext(cmd))
|
|
222
|
+
histogramAddDistance(©_dist_histograms[context], uint(cmd.dist_prefix_)&0x3FF)
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
}
|