typenative 0.3.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +57 -0
- package/README.md +17 -1
- package/bin/cli.js +25 -11
- package/bin/go/convert.js +11 -1
- package/bin/index.js +221 -47
- package/bin/main.js +35 -14
- package/bin/node/assert.js +2 -2
- package/bin/node/buffer.js +9 -0
- package/bin/node/dns.js +1 -1
- package/bin/node/events.js +1 -1
- package/bin/node/fs.js +1 -1
- package/bin/node/fs_watch.js +407 -0
- package/bin/node/http2.js +33 -6
- package/bin/node/readline.js +1 -1
- package/bin/node/registry.js +2 -1
- package/bin/node/stream.js +6 -3
- package/bin/node/timers.js +1 -1
- package/bin/node/typed_arrays.js +608 -0
- package/bin/node/util.js +3 -0
- package/bin/node/zlib.js +131 -16
- package/bin/transpiler.js +635 -44
- package/bin/version.js +1 -1
- package/conformance.md +16 -4
- package/examples/README.md +54 -0
- package/examples/fibonacci/README.md +89 -0
- package/examples/fibonacci/bench.mjs +87 -0
- package/examples/fibonacci/main.ts +21 -0
- package/examples/hello-world/README.md +62 -0
- package/examples/hello-world/bench.mjs +66 -0
- package/examples/hello-world/main.ts +7 -0
- package/examples/json-server/README.md +125 -0
- package/examples/json-server/bench.mjs +171 -0
- package/examples/json-server/main.ts +67 -0
- package/govendor/github.com/andybalholm/brotli/LICENSE +19 -0
- package/govendor/github.com/andybalholm/brotli/README.md +19 -0
- package/govendor/github.com/andybalholm/brotli/backward_references.go +185 -0
- package/govendor/github.com/andybalholm/brotli/backward_references_hq.go +796 -0
- package/govendor/github.com/andybalholm/brotli/bit_cost.go +436 -0
- package/govendor/github.com/andybalholm/brotli/bit_reader.go +266 -0
- package/govendor/github.com/andybalholm/brotli/bitwriter.go +56 -0
- package/govendor/github.com/andybalholm/brotli/block_splitter.go +144 -0
- package/govendor/github.com/andybalholm/brotli/block_splitter_command.go +434 -0
- package/govendor/github.com/andybalholm/brotli/block_splitter_distance.go +433 -0
- package/govendor/github.com/andybalholm/brotli/block_splitter_literal.go +433 -0
- package/govendor/github.com/andybalholm/brotli/brotli_bit_stream.go +1435 -0
- package/govendor/github.com/andybalholm/brotli/cluster.go +30 -0
- package/govendor/github.com/andybalholm/brotli/cluster_command.go +164 -0
- package/govendor/github.com/andybalholm/brotli/cluster_distance.go +326 -0
- package/govendor/github.com/andybalholm/brotli/cluster_literal.go +326 -0
- package/govendor/github.com/andybalholm/brotli/command.go +254 -0
- package/govendor/github.com/andybalholm/brotli/compress_fragment.go +834 -0
- package/govendor/github.com/andybalholm/brotli/compress_fragment_two_pass.go +775 -0
- package/govendor/github.com/andybalholm/brotli/constants.go +77 -0
- package/govendor/github.com/andybalholm/brotli/context.go +2176 -0
- package/govendor/github.com/andybalholm/brotli/decode.go +2570 -0
- package/govendor/github.com/andybalholm/brotli/dictionary.go +122890 -0
- package/govendor/github.com/andybalholm/brotli/dictionary_hash.go +32779 -0
- package/govendor/github.com/andybalholm/brotli/encode.go +1220 -0
- package/govendor/github.com/andybalholm/brotli/encoder.go +177 -0
- package/govendor/github.com/andybalholm/brotli/encoder_dict.go +22 -0
- package/govendor/github.com/andybalholm/brotli/encoder_fast.go +201 -0
- package/govendor/github.com/andybalholm/brotli/entropy_encode.go +592 -0
- package/govendor/github.com/andybalholm/brotli/entropy_encode_static.go +4399 -0
- package/govendor/github.com/andybalholm/brotli/fast_log.go +290 -0
- package/govendor/github.com/andybalholm/brotli/find_match_length.go +45 -0
- package/govendor/github.com/andybalholm/brotli/go.mod +3 -0
- package/govendor/github.com/andybalholm/brotli/h10.go +287 -0
- package/govendor/github.com/andybalholm/brotli/h5.go +214 -0
- package/govendor/github.com/andybalholm/brotli/h6.go +216 -0
- package/govendor/github.com/andybalholm/brotli/hash.go +342 -0
- package/govendor/github.com/andybalholm/brotli/hash_composite.go +93 -0
- package/govendor/github.com/andybalholm/brotli/hash_forgetful_chain.go +252 -0
- package/govendor/github.com/andybalholm/brotli/hash_longest_match_quickly.go +214 -0
- package/govendor/github.com/andybalholm/brotli/hash_rolling.go +168 -0
- package/govendor/github.com/andybalholm/brotli/histogram.go +226 -0
- package/govendor/github.com/andybalholm/brotli/http.go +201 -0
- package/govendor/github.com/andybalholm/brotli/huffman.go +653 -0
- package/govendor/github.com/andybalholm/brotli/literal_cost.go +182 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/bargain1.go +212 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/bargain2.go +248 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/bargain3.go +278 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/emitter.go +34 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/m0.go +175 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/m4.go +339 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/matchfinder.go +103 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/pathfinder.go +328 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/textencoder.go +53 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/trio.go +327 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/zdfast.go +258 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/zfast.go +206 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/zm.go +302 -0
- package/govendor/github.com/andybalholm/brotli/memory.go +66 -0
- package/govendor/github.com/andybalholm/brotli/metablock.go +574 -0
- package/govendor/github.com/andybalholm/brotli/metablock_command.go +165 -0
- package/govendor/github.com/andybalholm/brotli/metablock_distance.go +165 -0
- package/govendor/github.com/andybalholm/brotli/metablock_literal.go +165 -0
- package/govendor/github.com/andybalholm/brotli/params.go +37 -0
- package/govendor/github.com/andybalholm/brotli/platform.go +103 -0
- package/govendor/github.com/andybalholm/brotli/prefix.go +30 -0
- package/govendor/github.com/andybalholm/brotli/prefix_dec.go +723 -0
- package/govendor/github.com/andybalholm/brotli/quality.go +196 -0
- package/govendor/github.com/andybalholm/brotli/reader.go +127 -0
- package/govendor/github.com/andybalholm/brotli/ringbuffer.go +134 -0
- package/govendor/github.com/andybalholm/brotli/state.go +313 -0
- package/govendor/github.com/andybalholm/brotli/static_dict.go +662 -0
- package/govendor/github.com/andybalholm/brotli/static_dict_lut.go +75094 -0
- package/govendor/github.com/andybalholm/brotli/symbol_list.go +22 -0
- package/govendor/github.com/andybalholm/brotli/transform.go +641 -0
- package/govendor/github.com/andybalholm/brotli/utf8_util.go +70 -0
- package/govendor/github.com/andybalholm/brotli/util.go +7 -0
- package/govendor/github.com/andybalholm/brotli/write_bits.go +52 -0
- package/govendor/github.com/andybalholm/brotli/writer.go +162 -0
- package/govendor/github.com/fsnotify/fsnotify/.mailmap +2 -0
- package/govendor/github.com/fsnotify/fsnotify/CHANGELOG.md +651 -0
- package/govendor/github.com/fsnotify/fsnotify/CONTRIBUTING.md +147 -0
- package/govendor/github.com/fsnotify/fsnotify/LICENSE +25 -0
- package/govendor/github.com/fsnotify/fsnotify/README.md +214 -0
- package/govendor/github.com/fsnotify/fsnotify/backend_fen.go +469 -0
- package/govendor/github.com/fsnotify/fsnotify/backend_inotify.go +597 -0
- package/govendor/github.com/fsnotify/fsnotify/backend_kqueue.go +735 -0
- package/govendor/github.com/fsnotify/fsnotify/backend_other.go +22 -0
- package/govendor/github.com/fsnotify/fsnotify/backend_windows.go +702 -0
- package/govendor/github.com/fsnotify/fsnotify/fsnotify.go +526 -0
- package/govendor/github.com/fsnotify/fsnotify/go.mod +3 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/darwin.go +20 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_darwin.go +15 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_dragonfly.go +15 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_freebsd.go +20 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_kqueue.go +32 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_linux.go +56 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_netbsd.go +15 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_openbsd.go +16 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_solaris.go +45 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_windows.go +40 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/freebsd.go +20 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/internal.go +2 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/unix.go +20 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/unix2.go +25 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/windows.go +40 -0
- package/govendor/github.com/fsnotify/fsnotify/shared.go +64 -0
- package/govendor/github.com/fsnotify/fsnotify/staticcheck.conf +3 -0
- package/govendor/github.com/fsnotify/fsnotify/system_bsd.go +7 -0
- package/govendor/github.com/fsnotify/fsnotify/system_darwin.go +8 -0
- package/govendor/manifest.txt +3 -1
- package/package.json +1 -1
- package/types/typenative-npm.d.ts +72 -0
- package/types/typenative.d.ts +117 -1
|
@@ -0,0 +1,278 @@
|
|
|
1
|
+
package matchfinder
|
|
2
|
+
|
|
3
|
+
import (
|
|
4
|
+
"encoding/binary"
|
|
5
|
+
"math"
|
|
6
|
+
"math/bits"
|
|
7
|
+
"slices"
|
|
8
|
+
)
|
|
9
|
+
|
|
10
|
+
// Bargain3 is a MatchFinder that attempts to find the encoding with the lowest
|
|
11
|
+
// "bit cost", using 3 hash lengths (5, 8, and 12).
|
|
12
|
+
type Bargain3 struct {
|
|
13
|
+
MaxDistance int
|
|
14
|
+
|
|
15
|
+
// Skip is whether to look for matches at every other byte instead of every
|
|
16
|
+
// byte (to increase speed but decrease compression).
|
|
17
|
+
Skip bool
|
|
18
|
+
|
|
19
|
+
history []byte
|
|
20
|
+
table5 [1 << 17]tableEntry
|
|
21
|
+
table8 [1 << 18]tableEntry
|
|
22
|
+
table12 [1 << 19]tableEntry
|
|
23
|
+
|
|
24
|
+
// holding onto buffers to reduce allocations:
|
|
25
|
+
|
|
26
|
+
arrivals []arrival
|
|
27
|
+
matches []Match
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
func (z *Bargain3) Reset() {
|
|
31
|
+
z.table5 = [len(z.table5)]tableEntry{}
|
|
32
|
+
z.table8 = [len(z.table8)]tableEntry{}
|
|
33
|
+
z.table12 = [len(z.table12)]tableEntry{}
|
|
34
|
+
z.history = z.history[:0]
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
func (z *Bargain3) FindMatches(dst []Match, src []byte) []Match {
|
|
38
|
+
if z.MaxDistance == 0 {
|
|
39
|
+
z.MaxDistance = 1 << 16
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
var histogram [256]uint32
|
|
43
|
+
for _, b := range src {
|
|
44
|
+
histogram[b]++
|
|
45
|
+
}
|
|
46
|
+
var byteCost [256]float32
|
|
47
|
+
for b, n := range histogram {
|
|
48
|
+
cost := max(math.Log2(float64(len(src))/float64(n)), 1)
|
|
49
|
+
byteCost[b] = float32(cost)
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
// Each element in arrivals corresponds to the position just after
|
|
53
|
+
// the corresponding byte in src.
|
|
54
|
+
arrivals := z.arrivals
|
|
55
|
+
if len(arrivals) < len(src) {
|
|
56
|
+
arrivals = make([]arrival, len(src))
|
|
57
|
+
z.arrivals = arrivals
|
|
58
|
+
} else {
|
|
59
|
+
arrivals = arrivals[:len(src)]
|
|
60
|
+
for i := range arrivals {
|
|
61
|
+
arrivals[i] = arrival{}
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
if len(z.history) > z.MaxDistance*2 {
|
|
66
|
+
delta := len(z.history) - z.MaxDistance
|
|
67
|
+
copy(z.history, z.history[delta:])
|
|
68
|
+
z.history = z.history[:z.MaxDistance]
|
|
69
|
+
|
|
70
|
+
for i := range z.table5 {
|
|
71
|
+
v := z.table5[i].offset
|
|
72
|
+
v -= int32(delta)
|
|
73
|
+
if v < 0 {
|
|
74
|
+
z.table5[i] = tableEntry{}
|
|
75
|
+
} else {
|
|
76
|
+
z.table5[i].offset = v
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
for i := range z.table8 {
|
|
80
|
+
v := z.table8[i].offset
|
|
81
|
+
v -= int32(delta)
|
|
82
|
+
if v < 0 {
|
|
83
|
+
z.table8[i] = tableEntry{}
|
|
84
|
+
} else {
|
|
85
|
+
z.table8[i].offset = v
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
for i := range z.table12 {
|
|
89
|
+
v := z.table12[i].offset
|
|
90
|
+
v -= int32(delta)
|
|
91
|
+
if v < 0 {
|
|
92
|
+
z.table12[i] = tableEntry{}
|
|
93
|
+
} else {
|
|
94
|
+
z.table12[i].offset = v
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
historyLen := len(z.history)
|
|
100
|
+
z.history = append(z.history, src...)
|
|
101
|
+
src = z.history
|
|
102
|
+
|
|
103
|
+
addMatch := func(m absoluteMatch, unmatched int, repeat bool) {
|
|
104
|
+
var startCost float32
|
|
105
|
+
if m.Start > historyLen {
|
|
106
|
+
startCost = arrivals[m.Start-historyLen-1].cost
|
|
107
|
+
}
|
|
108
|
+
insertCost := float32(bits.Len(uint(unmatched)))
|
|
109
|
+
var distanceCost float32
|
|
110
|
+
if !repeat {
|
|
111
|
+
distanceCost = float32(bits.Len(uint(m.Start - m.Match)))
|
|
112
|
+
}
|
|
113
|
+
cost := startCost + baseMatchCost + insertCost + distanceCost
|
|
114
|
+
for j := m.End; j >= m.Start+3; j-- {
|
|
115
|
+
a := &arrivals[j-historyLen-1]
|
|
116
|
+
if a.cost > 0 && a.cost <= cost {
|
|
117
|
+
break
|
|
118
|
+
}
|
|
119
|
+
*a = arrival{
|
|
120
|
+
length: uint32(j - m.Start),
|
|
121
|
+
distance: uint32(m.Start - m.Match),
|
|
122
|
+
cost: cost,
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
var nextOverlapSearch int
|
|
128
|
+
|
|
129
|
+
for i := historyLen; i < len(src); i++ {
|
|
130
|
+
var arrivedHere arrival
|
|
131
|
+
if i > historyLen {
|
|
132
|
+
arrivedHere = arrivals[i-historyLen-1]
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
unmatched := 0
|
|
136
|
+
if arrivedHere.distance == 0 {
|
|
137
|
+
unmatched = int(arrivedHere.length)
|
|
138
|
+
}
|
|
139
|
+
prevDistance := 0
|
|
140
|
+
if unmatched != 0 && i-unmatched > historyLen {
|
|
141
|
+
prevDistance = int(arrivals[i-historyLen-1-unmatched].distance)
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
literalCost := byteCost[src[i]]
|
|
145
|
+
nextArrival := &arrivals[i-historyLen]
|
|
146
|
+
if nextArrival.cost == 0 || arrivedHere.cost+literalCost < nextArrival.cost {
|
|
147
|
+
*nextArrival = arrival{
|
|
148
|
+
cost: arrivedHere.cost + literalCost,
|
|
149
|
+
length: uint32(unmatched + 1),
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
if i > len(src)-12 {
|
|
154
|
+
// There's no room to check hashes.
|
|
155
|
+
continue
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
cv := binary.LittleEndian.Uint64(src[i:])
|
|
159
|
+
extra := binary.LittleEndian.Uint32(src[i+8:])
|
|
160
|
+
nextHash5 := z.hash5(cv)
|
|
161
|
+
nextHash8 := z.hash8(cv)
|
|
162
|
+
nextHash12 := z.hash12(cv, extra)
|
|
163
|
+
candidate5 := z.table5[nextHash5]
|
|
164
|
+
candidate8 := z.table8[nextHash8]
|
|
165
|
+
candidate12 := z.table12[nextHash12]
|
|
166
|
+
|
|
167
|
+
entry := tableEntry{offset: int32(i), val: uint32(cv)}
|
|
168
|
+
z.table5[nextHash5] = entry
|
|
169
|
+
z.table8[nextHash8] = entry
|
|
170
|
+
z.table12[nextHash12] = entry
|
|
171
|
+
|
|
172
|
+
// Look for a repeat match, unless there is no previous distance, or a match at
|
|
173
|
+
// that distance has already been found.
|
|
174
|
+
if prevDistance != 0 && prevDistance != int(arrivals[i-historyLen-1+4].distance) {
|
|
175
|
+
repIndex := i - prevDistance
|
|
176
|
+
if repIndex >= 0 && binary.LittleEndian.Uint32(src[repIndex:]) == uint32(cv) {
|
|
177
|
+
// We have a repeat of the previous match distance.
|
|
178
|
+
m := extendMatch2(src, i, repIndex, i)
|
|
179
|
+
addMatch(m, unmatched, true)
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
if z.Skip {
|
|
184
|
+
if i%2 != 0 {
|
|
185
|
+
continue
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
nextByteIsUnmatched := arrivals[i-historyLen-1+1].distance == 0
|
|
190
|
+
|
|
191
|
+
if unmatched > 0 || i >= nextOverlapSearch || nextByteIsUnmatched {
|
|
192
|
+
if int(candidate5.offset) < i && i-int(candidate5.offset) < z.MaxDistance && uint32(cv) == candidate5.val &&
|
|
193
|
+
binary.LittleEndian.Uint32(src[candidate5.offset:]) == uint32(cv) {
|
|
194
|
+
m := extendMatch2(src, i, int(candidate5.offset), historyLen)
|
|
195
|
+
delta := i - m.Start
|
|
196
|
+
if delta == 0 {
|
|
197
|
+
addMatch(m, unmatched, false)
|
|
198
|
+
} else {
|
|
199
|
+
// The match was extended backwards. Add it with and without the extra.
|
|
200
|
+
addMatch(m, max(unmatched-delta, 0), false)
|
|
201
|
+
m.Start += delta
|
|
202
|
+
m.Match += delta
|
|
203
|
+
addMatch(m, unmatched, false)
|
|
204
|
+
}
|
|
205
|
+
nextOverlapSearch = max(nextOverlapSearch, m.Start+1, m.End-6)
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
if int(candidate8.offset) < i && i-int(candidate8.offset) < z.MaxDistance && uint32(cv) == candidate8.val &&
|
|
209
|
+
binary.LittleEndian.Uint32(src[candidate8.offset:]) == uint32(cv) {
|
|
210
|
+
m := extendMatch2(src, i, int(candidate8.offset), historyLen)
|
|
211
|
+
delta := i - m.Start
|
|
212
|
+
if delta == 0 {
|
|
213
|
+
addMatch(m, unmatched, false)
|
|
214
|
+
} else {
|
|
215
|
+
// The match was extended backwards. Add it with and without the extra.
|
|
216
|
+
addMatch(m, max(unmatched-delta, 0), false)
|
|
217
|
+
m.Start += delta
|
|
218
|
+
m.Match += delta
|
|
219
|
+
addMatch(m, unmatched, false)
|
|
220
|
+
}
|
|
221
|
+
nextOverlapSearch = max(nextOverlapSearch, m.Start+1, m.End-6)
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
if int(candidate12.offset) < i && i-int(candidate12.offset) < z.MaxDistance && uint32(cv) == candidate12.val &&
|
|
225
|
+
binary.LittleEndian.Uint32(src[candidate12.offset:]) == uint32(cv) {
|
|
226
|
+
m := extendMatch2(src, i, int(candidate12.offset), historyLen)
|
|
227
|
+
delta := i - m.Start
|
|
228
|
+
if delta == 0 {
|
|
229
|
+
addMatch(m, unmatched, false)
|
|
230
|
+
} else {
|
|
231
|
+
// The match was extended backwards. Add it with and without the extra.
|
|
232
|
+
addMatch(m, max(unmatched-delta, 0), false)
|
|
233
|
+
m.Start += delta
|
|
234
|
+
m.Match += delta
|
|
235
|
+
addMatch(m, unmatched, false)
|
|
236
|
+
}
|
|
237
|
+
nextOverlapSearch = max(nextOverlapSearch, m.Start+1, m.End-6)
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
// We've found the shortest path; now walk it backward and store the matches.
|
|
243
|
+
matches := z.matches[:0]
|
|
244
|
+
i := len(arrivals) - 1
|
|
245
|
+
for i >= 0 {
|
|
246
|
+
a := arrivals[i]
|
|
247
|
+
if a.distance > 0 {
|
|
248
|
+
matches = append(matches, Match{
|
|
249
|
+
Length: int(a.length),
|
|
250
|
+
Distance: int(a.distance),
|
|
251
|
+
})
|
|
252
|
+
i -= int(a.length)
|
|
253
|
+
} else {
|
|
254
|
+
if len(matches) == 0 {
|
|
255
|
+
matches = append(matches, Match{})
|
|
256
|
+
}
|
|
257
|
+
matches[len(matches)-1].Unmatched = int(a.length)
|
|
258
|
+
i -= int(a.length)
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
z.matches = matches
|
|
262
|
+
|
|
263
|
+
slices.Reverse(matches)
|
|
264
|
+
|
|
265
|
+
return append(dst, matches...)
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
func (z *Bargain3) hash5(u uint64) uint32 {
|
|
269
|
+
return uint32(((u << 24) * 889523592379) >> (64 - 17))
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
func (z *Bargain3) hash8(u uint64) uint32 {
|
|
273
|
+
return uint32((u * 0xcf1bbcdcb7a56463) >> (64 - 18))
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
func (z *Bargain3) hash12(u uint64, e uint32) uint32 {
|
|
277
|
+
return uint32((u*0xcf1bbcdcb7a56463 + uint64(e)*(2654435761<<32)) >> (64 - 19))
|
|
278
|
+
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
package matchfinder
|
|
2
|
+
|
|
3
|
+
// An absoluteMatch is like a Match, but it stores indexes into the byte
|
|
4
|
+
// stream instead of lengths.
|
|
5
|
+
type absoluteMatch struct {
|
|
6
|
+
// Start is the index of the first byte.
|
|
7
|
+
Start int
|
|
8
|
+
|
|
9
|
+
// End is the index of the byte after the last byte
|
|
10
|
+
// (so that End - Start = Length).
|
|
11
|
+
End int
|
|
12
|
+
|
|
13
|
+
// Match is the index of the previous data that matches
|
|
14
|
+
// (Start - Match = Distance).
|
|
15
|
+
Match int
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
// A matchEmitter manages the output of matches for a MatchFinder.
|
|
19
|
+
type matchEmitter struct {
|
|
20
|
+
// Dst is the destination slice that Matches are added to.
|
|
21
|
+
Dst []Match
|
|
22
|
+
|
|
23
|
+
// NextEmit is the index of the next byte to emit.
|
|
24
|
+
NextEmit int
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
func (e *matchEmitter) emit(m absoluteMatch) {
|
|
28
|
+
e.Dst = append(e.Dst, Match{
|
|
29
|
+
Unmatched: m.Start - e.NextEmit,
|
|
30
|
+
Length: m.End - m.Start,
|
|
31
|
+
Distance: m.Start - m.Match,
|
|
32
|
+
})
|
|
33
|
+
e.NextEmit = m.End
|
|
34
|
+
}
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
package matchfinder
|
|
2
|
+
|
|
3
|
+
import (
|
|
4
|
+
"encoding/binary"
|
|
5
|
+
)
|
|
6
|
+
|
|
7
|
+
// M0 is an implementation of the MatchFinder interface based
|
|
8
|
+
// on the algorithm used by snappy, but modified to be more like the algorithm
|
|
9
|
+
// used by compression level 0 of the brotli reference implementation.
|
|
10
|
+
//
|
|
11
|
+
// It has a maximum block size of 65536 bytes.
|
|
12
|
+
type M0 struct {
|
|
13
|
+
// Lazy turns on "lazy matching," for higher compression but less speed.
|
|
14
|
+
Lazy bool
|
|
15
|
+
|
|
16
|
+
MaxDistance int
|
|
17
|
+
MaxLength int
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
func (M0) Reset() {}
|
|
21
|
+
|
|
22
|
+
const (
|
|
23
|
+
m0HashLen = 5
|
|
24
|
+
|
|
25
|
+
m0TableBits = 16
|
|
26
|
+
m0TableSize = 1 << m0TableBits
|
|
27
|
+
m0Shift = 32 - m0TableBits
|
|
28
|
+
// m0TableMask is redundant, but helps the compiler eliminate bounds
|
|
29
|
+
// checks.
|
|
30
|
+
m0TableMask = m0TableSize - 1
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
func (m M0) hash(data uint64) uint64 {
|
|
34
|
+
hash := (data << (64 - 8*m0HashLen)) * hashMul64
|
|
35
|
+
return hash >> (64 - m0TableBits)
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
// FindMatches looks for matches in src, appends them to dst, and returns dst.
|
|
39
|
+
// src must not be longer than 65536 bytes.
|
|
40
|
+
func (m M0) FindMatches(dst []Match, src []byte) []Match {
|
|
41
|
+
const inputMargin = 16 - 1
|
|
42
|
+
const minNonLiteralBlockSize = 1 + 1 + inputMargin
|
|
43
|
+
|
|
44
|
+
if len(src) < minNonLiteralBlockSize {
|
|
45
|
+
dst = append(dst, Match{
|
|
46
|
+
Unmatched: len(src),
|
|
47
|
+
})
|
|
48
|
+
return dst
|
|
49
|
+
}
|
|
50
|
+
if len(src) > 65536 {
|
|
51
|
+
panic("block too long")
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
var table [m0TableSize]uint16
|
|
55
|
+
|
|
56
|
+
// sLimit is when to stop looking for offset/length copies. The inputMargin
|
|
57
|
+
// lets us use a fast path for emitLiteral in the main loop, while we are
|
|
58
|
+
// looking for copies.
|
|
59
|
+
sLimit := len(src) - inputMargin
|
|
60
|
+
|
|
61
|
+
// nextEmit is where in src the next emitLiteral should start from.
|
|
62
|
+
nextEmit := 0
|
|
63
|
+
|
|
64
|
+
// The encoded form must start with a literal, as there are no previous
|
|
65
|
+
// bytes to copy, so we start looking for hash matches at s == 1.
|
|
66
|
+
s := 1
|
|
67
|
+
nextHash := m.hash(binary.LittleEndian.Uint64(src[s:]))
|
|
68
|
+
|
|
69
|
+
for {
|
|
70
|
+
// Copied from the C++ snappy implementation:
|
|
71
|
+
//
|
|
72
|
+
// Heuristic match skipping: If 32 bytes are scanned with no matches
|
|
73
|
+
// found, start looking only at every other byte. If 32 more bytes are
|
|
74
|
+
// scanned (or skipped), look at every third byte, etc.. When a match
|
|
75
|
+
// is found, immediately go back to looking at every byte. This is a
|
|
76
|
+
// small loss (~5% performance, ~0.1% density) for compressible data
|
|
77
|
+
// due to more bookkeeping, but for non-compressible data (such as
|
|
78
|
+
// JPEG) it's a huge win since the compressor quickly "realizes" the
|
|
79
|
+
// data is incompressible and doesn't bother looking for matches
|
|
80
|
+
// everywhere.
|
|
81
|
+
//
|
|
82
|
+
// The "skip" variable keeps track of how many bytes there are since
|
|
83
|
+
// the last match; dividing it by 32 (ie. right-shifting by five) gives
|
|
84
|
+
// the number of bytes to move ahead for each iteration.
|
|
85
|
+
skip := 32
|
|
86
|
+
|
|
87
|
+
nextS := s
|
|
88
|
+
candidate := 0
|
|
89
|
+
for {
|
|
90
|
+
s = nextS
|
|
91
|
+
bytesBetweenHashLookups := skip >> 5
|
|
92
|
+
nextS = s + bytesBetweenHashLookups
|
|
93
|
+
skip += bytesBetweenHashLookups
|
|
94
|
+
if nextS > sLimit {
|
|
95
|
+
goto emitRemainder
|
|
96
|
+
}
|
|
97
|
+
candidate = int(table[nextHash&m0TableMask])
|
|
98
|
+
table[nextHash&m0TableMask] = uint16(s)
|
|
99
|
+
nextHash = m.hash(binary.LittleEndian.Uint64(src[nextS:]))
|
|
100
|
+
if m.MaxDistance != 0 && s-candidate > m.MaxDistance {
|
|
101
|
+
continue
|
|
102
|
+
}
|
|
103
|
+
if binary.LittleEndian.Uint32(src[s:]) == binary.LittleEndian.Uint32(src[candidate:]) {
|
|
104
|
+
break
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
// Invariant: we have a 4-byte match at s.
|
|
109
|
+
base := s
|
|
110
|
+
s = extendMatch(src, candidate+4, s+4)
|
|
111
|
+
|
|
112
|
+
origBase := base
|
|
113
|
+
if m.Lazy && base+1 < sLimit {
|
|
114
|
+
newBase := base + 1
|
|
115
|
+
h := m.hash(binary.LittleEndian.Uint64(src[newBase:]))
|
|
116
|
+
newCandidate := int(table[h&m0TableMask])
|
|
117
|
+
table[h&m0TableMask] = uint16(newBase)
|
|
118
|
+
okDistance := true
|
|
119
|
+
if m.MaxDistance != 0 && newBase-newCandidate > m.MaxDistance {
|
|
120
|
+
okDistance = false
|
|
121
|
+
}
|
|
122
|
+
if okDistance && binary.LittleEndian.Uint32(src[newBase:]) == binary.LittleEndian.Uint32(src[newCandidate:]) {
|
|
123
|
+
newS := extendMatch(src, newCandidate+4, newBase+4)
|
|
124
|
+
if newS-newBase > s-base+1 {
|
|
125
|
+
s = newS
|
|
126
|
+
base = newBase
|
|
127
|
+
candidate = newCandidate
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
if m.MaxLength != 0 && s-base > m.MaxLength {
|
|
133
|
+
s = base + m.MaxLength
|
|
134
|
+
}
|
|
135
|
+
dst = append(dst, Match{
|
|
136
|
+
Unmatched: base - nextEmit,
|
|
137
|
+
Length: s - base,
|
|
138
|
+
Distance: base - candidate,
|
|
139
|
+
})
|
|
140
|
+
nextEmit = s
|
|
141
|
+
if s >= sLimit {
|
|
142
|
+
goto emitRemainder
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
if m.Lazy {
|
|
146
|
+
// If lazy matching is enabled, we update the hash table for
|
|
147
|
+
// every byte in the match.
|
|
148
|
+
for i := origBase + 2; i < s-3; i++ {
|
|
149
|
+
x := binary.LittleEndian.Uint64(src[i:])
|
|
150
|
+
table[m.hash(x)&m0TableMask] = uint16(i)
|
|
151
|
+
}
|
|
152
|
+
} else {
|
|
153
|
+
// We update the hash table only at base+1
|
|
154
|
+
x := binary.LittleEndian.Uint64(src[base+1:])
|
|
155
|
+
table[m.hash(x)&m0TableMask] = uint16(base + 1)
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
// We could immediately start working at s now, but to improve
|
|
159
|
+
// compression we first update the hash table.
|
|
160
|
+
x := binary.LittleEndian.Uint64(src[s-3:])
|
|
161
|
+
table[m.hash(x)&m0TableMask] = uint16(s - 3)
|
|
162
|
+
table[m.hash(x>>8)&m0TableMask] = uint16(s - 2)
|
|
163
|
+
prevHash := m.hash(x >> 16)
|
|
164
|
+
table[prevHash&m0TableMask] = uint16(s - 1)
|
|
165
|
+
nextHash = m.hash(x >> 24)
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
emitRemainder:
|
|
169
|
+
if nextEmit < len(src) {
|
|
170
|
+
dst = append(dst, Match{
|
|
171
|
+
Unmatched: len(src) - nextEmit,
|
|
172
|
+
})
|
|
173
|
+
}
|
|
174
|
+
return dst
|
|
175
|
+
}
|