typenative 0.3.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +57 -0
- package/README.md +17 -1
- package/bin/cli.js +25 -11
- package/bin/go/convert.js +11 -1
- package/bin/index.js +221 -47
- package/bin/main.js +35 -14
- package/bin/node/assert.js +2 -2
- package/bin/node/buffer.js +9 -0
- package/bin/node/dns.js +1 -1
- package/bin/node/events.js +1 -1
- package/bin/node/fs.js +1 -1
- package/bin/node/fs_watch.js +407 -0
- package/bin/node/http2.js +33 -6
- package/bin/node/readline.js +1 -1
- package/bin/node/registry.js +2 -1
- package/bin/node/stream.js +6 -3
- package/bin/node/timers.js +1 -1
- package/bin/node/typed_arrays.js +608 -0
- package/bin/node/util.js +3 -0
- package/bin/node/zlib.js +131 -16
- package/bin/transpiler.js +635 -44
- package/bin/version.js +1 -1
- package/conformance.md +16 -4
- package/examples/README.md +54 -0
- package/examples/fibonacci/README.md +89 -0
- package/examples/fibonacci/bench.mjs +87 -0
- package/examples/fibonacci/main.ts +21 -0
- package/examples/hello-world/README.md +62 -0
- package/examples/hello-world/bench.mjs +66 -0
- package/examples/hello-world/main.ts +7 -0
- package/examples/json-server/README.md +125 -0
- package/examples/json-server/bench.mjs +171 -0
- package/examples/json-server/main.ts +67 -0
- package/govendor/github.com/andybalholm/brotli/LICENSE +19 -0
- package/govendor/github.com/andybalholm/brotli/README.md +19 -0
- package/govendor/github.com/andybalholm/brotli/backward_references.go +185 -0
- package/govendor/github.com/andybalholm/brotli/backward_references_hq.go +796 -0
- package/govendor/github.com/andybalholm/brotli/bit_cost.go +436 -0
- package/govendor/github.com/andybalholm/brotli/bit_reader.go +266 -0
- package/govendor/github.com/andybalholm/brotli/bitwriter.go +56 -0
- package/govendor/github.com/andybalholm/brotli/block_splitter.go +144 -0
- package/govendor/github.com/andybalholm/brotli/block_splitter_command.go +434 -0
- package/govendor/github.com/andybalholm/brotli/block_splitter_distance.go +433 -0
- package/govendor/github.com/andybalholm/brotli/block_splitter_literal.go +433 -0
- package/govendor/github.com/andybalholm/brotli/brotli_bit_stream.go +1435 -0
- package/govendor/github.com/andybalholm/brotli/cluster.go +30 -0
- package/govendor/github.com/andybalholm/brotli/cluster_command.go +164 -0
- package/govendor/github.com/andybalholm/brotli/cluster_distance.go +326 -0
- package/govendor/github.com/andybalholm/brotli/cluster_literal.go +326 -0
- package/govendor/github.com/andybalholm/brotli/command.go +254 -0
- package/govendor/github.com/andybalholm/brotli/compress_fragment.go +834 -0
- package/govendor/github.com/andybalholm/brotli/compress_fragment_two_pass.go +775 -0
- package/govendor/github.com/andybalholm/brotli/constants.go +77 -0
- package/govendor/github.com/andybalholm/brotli/context.go +2176 -0
- package/govendor/github.com/andybalholm/brotli/decode.go +2570 -0
- package/govendor/github.com/andybalholm/brotli/dictionary.go +122890 -0
- package/govendor/github.com/andybalholm/brotli/dictionary_hash.go +32779 -0
- package/govendor/github.com/andybalholm/brotli/encode.go +1220 -0
- package/govendor/github.com/andybalholm/brotli/encoder.go +177 -0
- package/govendor/github.com/andybalholm/brotli/encoder_dict.go +22 -0
- package/govendor/github.com/andybalholm/brotli/encoder_fast.go +201 -0
- package/govendor/github.com/andybalholm/brotli/entropy_encode.go +592 -0
- package/govendor/github.com/andybalholm/brotli/entropy_encode_static.go +4399 -0
- package/govendor/github.com/andybalholm/brotli/fast_log.go +290 -0
- package/govendor/github.com/andybalholm/brotli/find_match_length.go +45 -0
- package/govendor/github.com/andybalholm/brotli/go.mod +3 -0
- package/govendor/github.com/andybalholm/brotli/h10.go +287 -0
- package/govendor/github.com/andybalholm/brotli/h5.go +214 -0
- package/govendor/github.com/andybalholm/brotli/h6.go +216 -0
- package/govendor/github.com/andybalholm/brotli/hash.go +342 -0
- package/govendor/github.com/andybalholm/brotli/hash_composite.go +93 -0
- package/govendor/github.com/andybalholm/brotli/hash_forgetful_chain.go +252 -0
- package/govendor/github.com/andybalholm/brotli/hash_longest_match_quickly.go +214 -0
- package/govendor/github.com/andybalholm/brotli/hash_rolling.go +168 -0
- package/govendor/github.com/andybalholm/brotli/histogram.go +226 -0
- package/govendor/github.com/andybalholm/brotli/http.go +201 -0
- package/govendor/github.com/andybalholm/brotli/huffman.go +653 -0
- package/govendor/github.com/andybalholm/brotli/literal_cost.go +182 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/bargain1.go +212 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/bargain2.go +248 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/bargain3.go +278 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/emitter.go +34 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/m0.go +175 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/m4.go +339 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/matchfinder.go +103 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/pathfinder.go +328 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/textencoder.go +53 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/trio.go +327 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/zdfast.go +258 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/zfast.go +206 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/zm.go +302 -0
- package/govendor/github.com/andybalholm/brotli/memory.go +66 -0
- package/govendor/github.com/andybalholm/brotli/metablock.go +574 -0
- package/govendor/github.com/andybalholm/brotli/metablock_command.go +165 -0
- package/govendor/github.com/andybalholm/brotli/metablock_distance.go +165 -0
- package/govendor/github.com/andybalholm/brotli/metablock_literal.go +165 -0
- package/govendor/github.com/andybalholm/brotli/params.go +37 -0
- package/govendor/github.com/andybalholm/brotli/platform.go +103 -0
- package/govendor/github.com/andybalholm/brotli/prefix.go +30 -0
- package/govendor/github.com/andybalholm/brotli/prefix_dec.go +723 -0
- package/govendor/github.com/andybalholm/brotli/quality.go +196 -0
- package/govendor/github.com/andybalholm/brotli/reader.go +127 -0
- package/govendor/github.com/andybalholm/brotli/ringbuffer.go +134 -0
- package/govendor/github.com/andybalholm/brotli/state.go +313 -0
- package/govendor/github.com/andybalholm/brotli/static_dict.go +662 -0
- package/govendor/github.com/andybalholm/brotli/static_dict_lut.go +75094 -0
- package/govendor/github.com/andybalholm/brotli/symbol_list.go +22 -0
- package/govendor/github.com/andybalholm/brotli/transform.go +641 -0
- package/govendor/github.com/andybalholm/brotli/utf8_util.go +70 -0
- package/govendor/github.com/andybalholm/brotli/util.go +7 -0
- package/govendor/github.com/andybalholm/brotli/write_bits.go +52 -0
- package/govendor/github.com/andybalholm/brotli/writer.go +162 -0
- package/govendor/github.com/fsnotify/fsnotify/.mailmap +2 -0
- package/govendor/github.com/fsnotify/fsnotify/CHANGELOG.md +651 -0
- package/govendor/github.com/fsnotify/fsnotify/CONTRIBUTING.md +147 -0
- package/govendor/github.com/fsnotify/fsnotify/LICENSE +25 -0
- package/govendor/github.com/fsnotify/fsnotify/README.md +214 -0
- package/govendor/github.com/fsnotify/fsnotify/backend_fen.go +469 -0
- package/govendor/github.com/fsnotify/fsnotify/backend_inotify.go +597 -0
- package/govendor/github.com/fsnotify/fsnotify/backend_kqueue.go +735 -0
- package/govendor/github.com/fsnotify/fsnotify/backend_other.go +22 -0
- package/govendor/github.com/fsnotify/fsnotify/backend_windows.go +702 -0
- package/govendor/github.com/fsnotify/fsnotify/fsnotify.go +526 -0
- package/govendor/github.com/fsnotify/fsnotify/go.mod +3 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/darwin.go +20 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_darwin.go +15 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_dragonfly.go +15 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_freebsd.go +20 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_kqueue.go +32 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_linux.go +56 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_netbsd.go +15 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_openbsd.go +16 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_solaris.go +45 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_windows.go +40 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/freebsd.go +20 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/internal.go +2 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/unix.go +20 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/unix2.go +25 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/windows.go +40 -0
- package/govendor/github.com/fsnotify/fsnotify/shared.go +64 -0
- package/govendor/github.com/fsnotify/fsnotify/staticcheck.conf +3 -0
- package/govendor/github.com/fsnotify/fsnotify/system_bsd.go +7 -0
- package/govendor/github.com/fsnotify/fsnotify/system_darwin.go +8 -0
- package/govendor/manifest.txt +3 -1
- package/package.json +1 -1
- package/types/typenative-npm.d.ts +72 -0
- package/types/typenative.d.ts +117 -1
|
@@ -0,0 +1,328 @@
|
|
|
1
|
+
package matchfinder
|
|
2
|
+
|
|
3
|
+
import (
|
|
4
|
+
"encoding/binary"
|
|
5
|
+
"math"
|
|
6
|
+
"math/bits"
|
|
7
|
+
"slices"
|
|
8
|
+
)
|
|
9
|
+
|
|
10
|
+
// Pathfinder is a MatchFinder that uses hash chains to find matches, and a
|
|
11
|
+
// shortest-path optimizer to choose which matches to use.
|
|
12
|
+
type Pathfinder struct {
|
|
13
|
+
// MaxDistance is the maximum distance (in bytes) to look back for
|
|
14
|
+
// a match. The default is 65535.
|
|
15
|
+
MaxDistance int
|
|
16
|
+
|
|
17
|
+
// MinLength is the length of the shortest match to return.
|
|
18
|
+
// The default is 4.
|
|
19
|
+
MinLength int
|
|
20
|
+
|
|
21
|
+
// HashLen is the number of bytes to use to calculate the hashes.
|
|
22
|
+
// The maximum is 8 and the default is 6.
|
|
23
|
+
HashLen int
|
|
24
|
+
|
|
25
|
+
// TableBits is the number of bits in the hash table indexes.
|
|
26
|
+
// The default is 17 (128K entries).
|
|
27
|
+
TableBits int
|
|
28
|
+
|
|
29
|
+
// ChainLength is how many entries to search on the "match chain" of older
|
|
30
|
+
// locations with the same hash as the current location.
|
|
31
|
+
ChainLength int
|
|
32
|
+
|
|
33
|
+
table []uint32
|
|
34
|
+
chain []uint32
|
|
35
|
+
|
|
36
|
+
history []byte
|
|
37
|
+
|
|
38
|
+
// holding onto buffers to reduce allocations:
|
|
39
|
+
|
|
40
|
+
arrivals []arrival
|
|
41
|
+
foundMatches []absoluteMatch
|
|
42
|
+
matches []Match
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
func (q *Pathfinder) Reset() {
|
|
46
|
+
for i := range q.table {
|
|
47
|
+
q.table[i] = 0
|
|
48
|
+
}
|
|
49
|
+
q.history = q.history[:0]
|
|
50
|
+
q.chain = q.chain[:0]
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
// An arrival represents how we got to a certain byte position.
|
|
54
|
+
// The cost is the total cost to get there from the beginning of the block.
|
|
55
|
+
// If distance > 0, the arrival is with a match.
|
|
56
|
+
// If distance == 0, the arrival is with a run of literals.
|
|
57
|
+
type arrival struct {
|
|
58
|
+
length uint32
|
|
59
|
+
distance uint32
|
|
60
|
+
cost float32
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
const (
|
|
64
|
+
baseMatchCost float32 = 4
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
func (q *Pathfinder) FindMatches(dst []Match, src []byte) []Match {
|
|
68
|
+
if q.MaxDistance == 0 {
|
|
69
|
+
q.MaxDistance = 65535
|
|
70
|
+
}
|
|
71
|
+
if q.MinLength == 0 {
|
|
72
|
+
q.MinLength = 4
|
|
73
|
+
}
|
|
74
|
+
if q.HashLen == 0 {
|
|
75
|
+
q.HashLen = 6
|
|
76
|
+
}
|
|
77
|
+
if q.TableBits == 0 {
|
|
78
|
+
q.TableBits = 17
|
|
79
|
+
}
|
|
80
|
+
if len(q.table) < 1<<q.TableBits {
|
|
81
|
+
q.table = make([]uint32, 1<<q.TableBits)
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
var histogram [256]uint32
|
|
85
|
+
for _, b := range src {
|
|
86
|
+
histogram[b]++
|
|
87
|
+
}
|
|
88
|
+
var byteCost [256]float32
|
|
89
|
+
for b, n := range histogram {
|
|
90
|
+
cost := max(math.Log2(float64(len(src))/float64(n)), 1)
|
|
91
|
+
byteCost[b] = float32(cost)
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
// Each element in arrivals corresponds to the position just after
|
|
95
|
+
// the corresponding byte in src.
|
|
96
|
+
arrivals := q.arrivals
|
|
97
|
+
if len(arrivals) < len(src) {
|
|
98
|
+
arrivals = make([]arrival, len(src))
|
|
99
|
+
q.arrivals = arrivals
|
|
100
|
+
} else {
|
|
101
|
+
arrivals = arrivals[:len(src)]
|
|
102
|
+
for i := range arrivals {
|
|
103
|
+
arrivals[i] = arrival{}
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
if len(q.history) > q.MaxDistance*2 {
|
|
108
|
+
// Trim down the history buffer.
|
|
109
|
+
delta := len(q.history) - q.MaxDistance
|
|
110
|
+
copy(q.history, q.history[delta:])
|
|
111
|
+
q.history = q.history[:q.MaxDistance]
|
|
112
|
+
q.chain = q.chain[:q.MaxDistance]
|
|
113
|
+
|
|
114
|
+
for i, v := range q.table {
|
|
115
|
+
newV := max(int(v)-delta, 0)
|
|
116
|
+
q.table[i] = uint32(newV)
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
// Append src to the history buffer.
|
|
121
|
+
historyLen := len(q.history)
|
|
122
|
+
q.history = append(q.history, src...)
|
|
123
|
+
q.chain = append(q.chain, make([]uint32, len(src))...)
|
|
124
|
+
src = q.history
|
|
125
|
+
|
|
126
|
+
// Calculate hashes and build the chain.
|
|
127
|
+
for i := historyLen; i < len(src)-7; i++ {
|
|
128
|
+
h := ((binary.LittleEndian.Uint64(src[i:]) & (1<<(8*q.HashLen) - 1)) * hashMul64) >> (64 - q.TableBits)
|
|
129
|
+
candidate := int(q.table[h])
|
|
130
|
+
q.table[h] = uint32(i)
|
|
131
|
+
if candidate != 0 {
|
|
132
|
+
delta := i - candidate
|
|
133
|
+
q.chain[i] = uint32(delta)
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
// Look for matches, and collect them in foundMatches. Later we'll figure out
|
|
138
|
+
// which ones to actually use.
|
|
139
|
+
foundMatches := q.foundMatches[:0]
|
|
140
|
+
var prevMatch absoluteMatch
|
|
141
|
+
i := historyLen
|
|
142
|
+
for i < len(src)-7 {
|
|
143
|
+
delta := q.chain[i]
|
|
144
|
+
if delta == 0 {
|
|
145
|
+
i++
|
|
146
|
+
continue
|
|
147
|
+
}
|
|
148
|
+
candidate := i - int(delta)
|
|
149
|
+
if candidate <= 0 || i-candidate > q.MaxDistance {
|
|
150
|
+
i++
|
|
151
|
+
continue
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
var currentMatch absoluteMatch
|
|
155
|
+
|
|
156
|
+
if i >= prevMatch.End && prevMatch != (absoluteMatch{}) {
|
|
157
|
+
// Look for a repeat match at i+1.
|
|
158
|
+
prevDistance := prevMatch.Start - prevMatch.Match
|
|
159
|
+
if binary.LittleEndian.Uint32(src[i+1:]) == binary.LittleEndian.Uint32(src[i+1-prevDistance:]) {
|
|
160
|
+
m := extendMatch2(src, i+1, i+1-prevDistance, i+1)
|
|
161
|
+
if m.End-m.Start > q.MinLength {
|
|
162
|
+
currentMatch = m
|
|
163
|
+
foundMatches = append(foundMatches, m)
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
if binary.LittleEndian.Uint32(src[candidate:]) == binary.LittleEndian.Uint32(src[i:]) {
|
|
169
|
+
m := extendMatch2(src, i, candidate, max(historyLen, prevMatch.Start))
|
|
170
|
+
if m.End-m.Start > q.MinLength {
|
|
171
|
+
currentMatch = m
|
|
172
|
+
foundMatches = append(foundMatches, m)
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
for range q.ChainLength {
|
|
177
|
+
delta := q.chain[candidate]
|
|
178
|
+
if delta == 0 {
|
|
179
|
+
break
|
|
180
|
+
}
|
|
181
|
+
candidate -= int(delta)
|
|
182
|
+
if candidate <= 0 || i-candidate > q.MaxDistance {
|
|
183
|
+
break
|
|
184
|
+
}
|
|
185
|
+
if binary.LittleEndian.Uint32(src[candidate:]) == binary.LittleEndian.Uint32(src[i:]) {
|
|
186
|
+
m := extendMatch2(src, i, candidate, max(historyLen, prevMatch.Start))
|
|
187
|
+
if m.End-m.Start > q.MinLength && m.End-m.Start > currentMatch.End-currentMatch.Start {
|
|
188
|
+
currentMatch = m
|
|
189
|
+
foundMatches = append(foundMatches, m)
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
if i < prevMatch.End && currentMatch.End-currentMatch.Start <= prevMatch.End-prevMatch.Start {
|
|
195
|
+
// We were looking for an overlapping match, but we didn't find one longer
|
|
196
|
+
// than the previous match. So we'll go back to sequential search,
|
|
197
|
+
// starting right after the previous match.
|
|
198
|
+
i = prevMatch.End
|
|
199
|
+
continue
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
if currentMatch == (absoluteMatch{}) {
|
|
203
|
+
// No match found. Continue with sequential search.
|
|
204
|
+
i++
|
|
205
|
+
continue
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
// We've found a match; now look for matches overlapping the end of it.
|
|
209
|
+
prevMatch = currentMatch
|
|
210
|
+
i = currentMatch.End + 2 - q.HashLen
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
q.foundMatches = foundMatches
|
|
214
|
+
|
|
215
|
+
slices.SortFunc(foundMatches, func(a, b absoluteMatch) int { return a.Start - b.Start })
|
|
216
|
+
matchIndex := 0
|
|
217
|
+
var pending absoluteMatch
|
|
218
|
+
|
|
219
|
+
for i := historyLen; i < len(src); i++ {
|
|
220
|
+
var arrivedHere arrival
|
|
221
|
+
if i > historyLen {
|
|
222
|
+
arrivedHere = arrivals[i-historyLen-1]
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
unmatched := 0
|
|
226
|
+
if arrivedHere.distance == 0 {
|
|
227
|
+
unmatched = int(arrivedHere.length)
|
|
228
|
+
}
|
|
229
|
+
prevDistance := 0
|
|
230
|
+
if i-unmatched > historyLen {
|
|
231
|
+
prevDistance = int(arrivals[i-historyLen-1-unmatched].distance)
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
literalCost := byteCost[src[i]]
|
|
235
|
+
nextArrival := &arrivals[i-historyLen]
|
|
236
|
+
if nextArrival.cost == 0 || arrivedHere.cost+literalCost < nextArrival.cost {
|
|
237
|
+
*nextArrival = arrival{
|
|
238
|
+
cost: arrivedHere.cost + literalCost,
|
|
239
|
+
length: uint32(unmatched + 1),
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
for matchIndex < len(foundMatches) && foundMatches[matchIndex].Start == i {
|
|
244
|
+
m := foundMatches[matchIndex]
|
|
245
|
+
matchIndex++
|
|
246
|
+
if m.End > pending.End {
|
|
247
|
+
pending = m
|
|
248
|
+
}
|
|
249
|
+
matchCost := baseMatchCost + float32(bits.Len(uint(unmatched)))
|
|
250
|
+
if m.Start-m.Match != prevDistance {
|
|
251
|
+
matchCost += float32(bits.Len(uint(m.Start - m.Match)))
|
|
252
|
+
}
|
|
253
|
+
for j := m.Start + q.MinLength; j <= m.End; j++ {
|
|
254
|
+
adjustedCost := matchCost
|
|
255
|
+
if j-m.Start < 6 {
|
|
256
|
+
// Matches shorter than 6 are comparatively rare, and therefore
|
|
257
|
+
// have longer codes.
|
|
258
|
+
adjustedCost += float32(6-(j-m.Start)) * 2
|
|
259
|
+
}
|
|
260
|
+
a := &arrivals[j-historyLen-1]
|
|
261
|
+
if a.cost == 0 || arrivedHere.cost+adjustedCost < a.cost {
|
|
262
|
+
*a = arrival{
|
|
263
|
+
length: uint32(j - m.Start),
|
|
264
|
+
distance: uint32(m.Start - m.Match),
|
|
265
|
+
cost: arrivedHere.cost + adjustedCost,
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
// If a match from an earlier position extends far enough past the current
|
|
272
|
+
// position, try using the tail of it, starting from here.
|
|
273
|
+
if unmatched == 0 && pending.Start != i && pending.End >= i+q.MinLength &&
|
|
274
|
+
!(arrivedHere.length != 0 && arrivedHere.distance == uint32(pending.Start-pending.Match)) {
|
|
275
|
+
matchCost := baseMatchCost + float32(bits.Len(uint(pending.Start-pending.Match)))
|
|
276
|
+
for j := i + q.MinLength; j <= pending.End; j++ {
|
|
277
|
+
adjustedCost := matchCost
|
|
278
|
+
if j-i < 6 {
|
|
279
|
+
// Matches shorter than 6 are comparatively rare, and therefore
|
|
280
|
+
// have longer codes.
|
|
281
|
+
adjustedCost += float32(6-(j-i)) * 2
|
|
282
|
+
}
|
|
283
|
+
a := &arrivals[j-historyLen-1]
|
|
284
|
+
if a.cost == 0 || arrivedHere.cost+adjustedCost < a.cost {
|
|
285
|
+
*a = arrival{
|
|
286
|
+
length: uint32(j - i),
|
|
287
|
+
distance: uint32(pending.Start - pending.Match),
|
|
288
|
+
cost: arrivedHere.cost + adjustedCost,
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
delta := q.chain[i]
|
|
295
|
+
if delta == 0 {
|
|
296
|
+
continue
|
|
297
|
+
}
|
|
298
|
+
candidate := i - int(delta)
|
|
299
|
+
if candidate <= 0 || i-candidate > q.MaxDistance {
|
|
300
|
+
continue
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
// We've found the shortest path; now walk it backward and store the matches.
|
|
305
|
+
matches := q.matches[:0]
|
|
306
|
+
i = len(arrivals) - 1
|
|
307
|
+
for i >= 0 {
|
|
308
|
+
a := arrivals[i]
|
|
309
|
+
if a.distance > 0 {
|
|
310
|
+
matches = append(matches, Match{
|
|
311
|
+
Length: int(a.length),
|
|
312
|
+
Distance: int(a.distance),
|
|
313
|
+
})
|
|
314
|
+
i -= int(a.length)
|
|
315
|
+
} else {
|
|
316
|
+
if len(matches) == 0 {
|
|
317
|
+
matches = append(matches, Match{})
|
|
318
|
+
}
|
|
319
|
+
matches[len(matches)-1].Unmatched = int(a.length)
|
|
320
|
+
i -= int(a.length)
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
q.matches = matches
|
|
324
|
+
|
|
325
|
+
slices.Reverse(matches)
|
|
326
|
+
|
|
327
|
+
return append(dst, matches...)
|
|
328
|
+
}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
package matchfinder
|
|
2
|
+
|
|
3
|
+
import "fmt"
|
|
4
|
+
|
|
5
|
+
// A TextEncoder is an Encoder that produces a human-readable representation of
|
|
6
|
+
// the LZ77 compression. Matches are replaced with <Length,Distance> symbols.
|
|
7
|
+
type TextEncoder struct{}
|
|
8
|
+
|
|
9
|
+
func (t TextEncoder) Reset() {}
|
|
10
|
+
|
|
11
|
+
func (t TextEncoder) Encode(dst []byte, src []byte, matches []Match, lastBlock bool) []byte {
|
|
12
|
+
pos := 0
|
|
13
|
+
for _, m := range matches {
|
|
14
|
+
if m.Unmatched > 0 {
|
|
15
|
+
dst = append(dst, src[pos:pos+m.Unmatched]...)
|
|
16
|
+
pos += m.Unmatched
|
|
17
|
+
}
|
|
18
|
+
if m.Length > 0 {
|
|
19
|
+
dst = append(dst, []byte(fmt.Sprintf("<%d,%d>", m.Length, m.Distance))...)
|
|
20
|
+
pos += m.Length
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
if pos < len(src) {
|
|
24
|
+
dst = append(dst, src[pos:]...)
|
|
25
|
+
}
|
|
26
|
+
return dst
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
// A NoMatchFinder implements MatchFinder, but doesn't find any matches.
|
|
30
|
+
// It can be used to implement the equivalent of the standard library flate package's
|
|
31
|
+
// HuffmanOnly setting.
|
|
32
|
+
type NoMatchFinder struct{}
|
|
33
|
+
|
|
34
|
+
func (n NoMatchFinder) Reset() {}
|
|
35
|
+
|
|
36
|
+
func (n NoMatchFinder) FindMatches(dst []Match, src []byte) []Match {
|
|
37
|
+
return append(dst, Match{
|
|
38
|
+
Unmatched: len(src),
|
|
39
|
+
})
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
// AutoReset wraps a MatchFinder that can return references to data in previous
|
|
43
|
+
// blocks, and calls Reset before each block. It is useful for (e.g.) using a
|
|
44
|
+
// snappy Encoder with a MatchFinder designed for flate. (Snappy doesn't
|
|
45
|
+
// support references between blocks.)
|
|
46
|
+
type AutoReset struct {
|
|
47
|
+
MatchFinder
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
func (a AutoReset) FindMatches(dst []Match, src []byte) []Match {
|
|
51
|
+
a.Reset()
|
|
52
|
+
return a.MatchFinder.FindMatches(dst, src)
|
|
53
|
+
}
|
|
@@ -0,0 +1,327 @@
|
|
|
1
|
+
package matchfinder
|
|
2
|
+
|
|
3
|
+
import "encoding/binary"
|
|
4
|
+
|
|
5
|
+
// Trio is a MatchFinder that uses 3 different hash lengths, and
|
|
6
|
+
// overlap parsing.
|
|
7
|
+
type Trio struct {
|
|
8
|
+
MaxDistance int
|
|
9
|
+
history []byte
|
|
10
|
+
table5 [1 << 16]tableEntry
|
|
11
|
+
table8 [1 << 17]tableEntry
|
|
12
|
+
table12 [1 << 18]tableEntry
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
func (z *Trio) Reset() {
|
|
16
|
+
z.table5 = [len(z.table5)]tableEntry{}
|
|
17
|
+
z.table8 = [len(z.table8)]tableEntry{}
|
|
18
|
+
z.table12 = [len(z.table12)]tableEntry{}
|
|
19
|
+
z.history = z.history[:0]
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
func (z *Trio) FindMatches(dst []Match, src []byte) []Match {
|
|
23
|
+
if z.MaxDistance == 0 {
|
|
24
|
+
z.MaxDistance = 1 << 16
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
if len(z.history) > z.MaxDistance*2 {
|
|
28
|
+
delta := len(z.history) - z.MaxDistance
|
|
29
|
+
copy(z.history, z.history[delta:])
|
|
30
|
+
z.history = z.history[:z.MaxDistance]
|
|
31
|
+
|
|
32
|
+
for i := range z.table5 {
|
|
33
|
+
v := z.table5[i].offset
|
|
34
|
+
v -= int32(delta)
|
|
35
|
+
if v < 0 {
|
|
36
|
+
z.table5[i] = tableEntry{}
|
|
37
|
+
} else {
|
|
38
|
+
z.table5[i].offset = v
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
for i := range z.table8 {
|
|
42
|
+
v := z.table8[i].offset
|
|
43
|
+
v -= int32(delta)
|
|
44
|
+
if v < 0 {
|
|
45
|
+
z.table8[i] = tableEntry{}
|
|
46
|
+
} else {
|
|
47
|
+
z.table8[i].offset = v
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
for i := range z.table12 {
|
|
51
|
+
v := z.table12[i].offset
|
|
52
|
+
v -= int32(delta)
|
|
53
|
+
if v < 0 {
|
|
54
|
+
z.table12[i] = tableEntry{}
|
|
55
|
+
} else {
|
|
56
|
+
z.table12[i].offset = v
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
e := matchEmitter{
|
|
62
|
+
Dst: dst,
|
|
63
|
+
NextEmit: len(z.history),
|
|
64
|
+
}
|
|
65
|
+
z.history = append(z.history, src...)
|
|
66
|
+
|
|
67
|
+
if len(src) < 20 {
|
|
68
|
+
return append(dst, Match{
|
|
69
|
+
Unmatched: len(src),
|
|
70
|
+
})
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
src = z.history
|
|
74
|
+
|
|
75
|
+
// matches stores the matches that have been found but not emitted,
|
|
76
|
+
// in reverse order. (matches[0] is the most recent one.)
|
|
77
|
+
var matches [3]absoluteMatch
|
|
78
|
+
|
|
79
|
+
sLimit := int32(len(src)) - 14
|
|
80
|
+
|
|
81
|
+
mainLoop:
|
|
82
|
+
for {
|
|
83
|
+
// Search for a match, starting after the last match emitted.
|
|
84
|
+
s := int32(e.NextEmit)
|
|
85
|
+
if s > sLimit {
|
|
86
|
+
break mainLoop
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
// t will contain the match offset when we find one.
|
|
90
|
+
var t int32
|
|
91
|
+
var hashLengthFound int
|
|
92
|
+
|
|
93
|
+
cv := binary.LittleEndian.Uint64(src[s:])
|
|
94
|
+
extra := binary.LittleEndian.Uint32(src[s+8:])
|
|
95
|
+
|
|
96
|
+
for {
|
|
97
|
+
nextHash12 := z.hash12(cv, extra)
|
|
98
|
+
nextHash8 := z.hash8(cv)
|
|
99
|
+
nextHash5 := z.hash5(cv)
|
|
100
|
+
candidate12 := z.table12[nextHash12]
|
|
101
|
+
candidate8 := z.table8[nextHash8]
|
|
102
|
+
candidate5 := z.table5[nextHash5]
|
|
103
|
+
|
|
104
|
+
entry := tableEntry{offset: s, val: uint32(cv)}
|
|
105
|
+
z.table12[nextHash12] = entry
|
|
106
|
+
z.table8[nextHash8] = entry
|
|
107
|
+
z.table5[nextHash5] = entry
|
|
108
|
+
|
|
109
|
+
// Look for a repeat match one byte after the current position.
|
|
110
|
+
if len(e.Dst) > 0 {
|
|
111
|
+
prevDistance := int32(e.Dst[len(e.Dst)-1].Distance)
|
|
112
|
+
if prevDistance != 0 {
|
|
113
|
+
repIndex := s - prevDistance + 1
|
|
114
|
+
if repIndex >= 0 && binary.LittleEndian.Uint32(src[repIndex:]) == uint32(cv>>8) {
|
|
115
|
+
// There is a repeated match at s+2.
|
|
116
|
+
s++
|
|
117
|
+
t = repIndex
|
|
118
|
+
break
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
if candidate12.offset < s && s-candidate12.offset < int32(z.MaxDistance) && uint32(cv) == candidate12.val &&
|
|
124
|
+
binary.LittleEndian.Uint32(src[candidate12.offset:]) == uint32(cv) {
|
|
125
|
+
// There is a 12-byte match at s.
|
|
126
|
+
t = candidate12.offset
|
|
127
|
+
hashLengthFound = 12
|
|
128
|
+
break
|
|
129
|
+
}
|
|
130
|
+
if candidate8.offset < s && s-candidate8.offset < int32(z.MaxDistance) && uint32(cv) == candidate8.val &&
|
|
131
|
+
binary.LittleEndian.Uint32(src[candidate8.offset:]) == uint32(cv) {
|
|
132
|
+
t = candidate8.offset
|
|
133
|
+
hashLengthFound = 8
|
|
134
|
+
break
|
|
135
|
+
}
|
|
136
|
+
if candidate5.offset < s && s-candidate5.offset < int32(z.MaxDistance) && uint32(cv) == candidate5.val &&
|
|
137
|
+
binary.LittleEndian.Uint32(src[candidate5.offset:]) == uint32(cv) {
|
|
138
|
+
t = candidate5.offset
|
|
139
|
+
hashLengthFound = 5
|
|
140
|
+
break
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
s += 1 + ((s - int32(e.NextEmit)) >> 7)
|
|
144
|
+
if s > sLimit {
|
|
145
|
+
break mainLoop
|
|
146
|
+
}
|
|
147
|
+
cv = binary.LittleEndian.Uint64(src[s:])
|
|
148
|
+
extra = binary.LittleEndian.Uint32(src[s+8:])
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
if hashLengthFound != 0 && hashLengthFound < 12 {
|
|
152
|
+
// Look for a "lazy" match with a longer hash at s+1.
|
|
153
|
+
cv := binary.LittleEndian.Uint64(src[s+1:])
|
|
154
|
+
extra := binary.LittleEndian.Uint32(src[s+9:])
|
|
155
|
+
nextHash12 := z.hash12(cv, extra)
|
|
156
|
+
nextHash8 := z.hash8(cv)
|
|
157
|
+
candidate12 := z.table12[nextHash12]
|
|
158
|
+
candidate8 := z.table8[nextHash8]
|
|
159
|
+
coffset12 := s - candidate12.offset + 1
|
|
160
|
+
coffset8 := s - candidate8.offset + 1
|
|
161
|
+
entry := tableEntry{offset: s + 1, val: uint32(cv)}
|
|
162
|
+
z.table12[nextHash12] = entry
|
|
163
|
+
z.table8[nextHash8] = entry
|
|
164
|
+
if candidate12.offset < s+1 && coffset12 < int32(z.MaxDistance) && uint32(cv) == candidate12.val &&
|
|
165
|
+
binary.LittleEndian.Uint32(src[candidate12.offset:]) == uint32(cv) {
|
|
166
|
+
t = candidate12.offset
|
|
167
|
+
s++
|
|
168
|
+
} else if hashLengthFound < 8 && candidate8.offset < s+1 && coffset8 < int32(z.MaxDistance) && uint32(cv) == candidate8.val &&
|
|
169
|
+
binary.LittleEndian.Uint32(src[candidate8.offset:]) == uint32(cv) {
|
|
170
|
+
t = candidate8.offset
|
|
171
|
+
s++
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
currentMatch := extendMatch2(src, int(s), int(t), e.NextEmit)
|
|
176
|
+
matches[0] = currentMatch
|
|
177
|
+
|
|
178
|
+
index0 := s + 1
|
|
179
|
+
|
|
180
|
+
// We have a match in matches[0].
|
|
181
|
+
// Now look for overlapping matches.
|
|
182
|
+
|
|
183
|
+
for {
|
|
184
|
+
if matches[0].End > int(sLimit) {
|
|
185
|
+
break
|
|
186
|
+
}
|
|
187
|
+
s = int32(max(matches[0].Start+2, matches[0].End-10))
|
|
188
|
+
|
|
189
|
+
// Store some entries that haven't been indexed yet.
|
|
190
|
+
for index0 < s-1 {
|
|
191
|
+
cv0 := binary.LittleEndian.Uint64(src[index0:])
|
|
192
|
+
extra0 := binary.LittleEndian.Uint32(src[index0+8:])
|
|
193
|
+
te0 := tableEntry{offset: index0, val: uint32(cv0)}
|
|
194
|
+
z.table5[z.hash5(cv0)] = te0
|
|
195
|
+
z.table8[z.hash8(cv0)] = te0
|
|
196
|
+
z.table12[z.hash12(cv0, extra0)] = te0
|
|
197
|
+
index0++
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
cv = binary.LittleEndian.Uint64(src[s:])
|
|
201
|
+
extra = binary.LittleEndian.Uint32(src[s+8:])
|
|
202
|
+
|
|
203
|
+
nextHash12 := z.hash12(cv, extra)
|
|
204
|
+
nextHash8 := z.hash8(cv)
|
|
205
|
+
nextHash5 := z.hash5(cv)
|
|
206
|
+
candidate12 := z.table12[nextHash12]
|
|
207
|
+
candidate8 := z.table8[nextHash8]
|
|
208
|
+
|
|
209
|
+
entry := tableEntry{offset: s, val: uint32(cv)}
|
|
210
|
+
z.table12[nextHash12] = entry
|
|
211
|
+
z.table8[nextHash8] = entry
|
|
212
|
+
z.table5[nextHash5] = entry
|
|
213
|
+
|
|
214
|
+
t = -1
|
|
215
|
+
if candidate12.offset < s && s-candidate12.offset < int32(z.MaxDistance) && uint32(cv) == candidate12.val &&
|
|
216
|
+
binary.LittleEndian.Uint32(src[candidate12.offset:]) == uint32(cv) {
|
|
217
|
+
// There is a 12-byte match at s.
|
|
218
|
+
t = candidate12.offset
|
|
219
|
+
} else if candidate8.offset < s && s-candidate8.offset < int32(z.MaxDistance) && uint32(cv) == candidate8.val &&
|
|
220
|
+
binary.LittleEndian.Uint32(src[candidate8.offset:]) == uint32(cv) {
|
|
221
|
+
// There is a long match at s.
|
|
222
|
+
t = candidate8.offset
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
index0 = s + 1
|
|
226
|
+
|
|
227
|
+
if t == -1 {
|
|
228
|
+
// No overlapping match was found.
|
|
229
|
+
break
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
newMatch := extendMatch2(src, int(s), int(t), e.NextEmit)
|
|
233
|
+
|
|
234
|
+
if newMatch.End-newMatch.Start <= matches[0].End-matches[0].Start {
|
|
235
|
+
// The new match isn't longer than the old one, so we break out of the loop
|
|
236
|
+
// of looking for overlapping matches.
|
|
237
|
+
break
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
matches = [3]absoluteMatch{
|
|
241
|
+
newMatch,
|
|
242
|
+
matches[0],
|
|
243
|
+
matches[1],
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
if matches[2] == (absoluteMatch{}) {
|
|
247
|
+
continue
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
// We have three matches, so it's time to emit one and/or eliminate one.
|
|
251
|
+
switch {
|
|
252
|
+
case matches[0].Start < matches[2].End:
|
|
253
|
+
// The first and third matches overlap; discard the one in between.
|
|
254
|
+
matches = [3]absoluteMatch{
|
|
255
|
+
matches[0],
|
|
256
|
+
matches[2],
|
|
257
|
+
{},
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
case matches[0].Start < matches[2].End+4:
|
|
261
|
+
// The first and third matches don't overlap, but there's no room for
|
|
262
|
+
// another match between them. Emit the first match and discard the second.
|
|
263
|
+
e.emit(matches[2])
|
|
264
|
+
matches = [3]absoluteMatch{
|
|
265
|
+
matches[0],
|
|
266
|
+
{},
|
|
267
|
+
{},
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
default:
|
|
271
|
+
// Emit the first match, shortening it if necessary to avoid overlap with the second.
|
|
272
|
+
if matches[2].End > matches[1].Start {
|
|
273
|
+
matches[2].End = matches[1].Start
|
|
274
|
+
}
|
|
275
|
+
if matches[2].End-matches[2].Start >= 4 {
|
|
276
|
+
e.emit(matches[2])
|
|
277
|
+
}
|
|
278
|
+
matches[2] = absoluteMatch{}
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
// Store some entries up to the end of the last match.
|
|
283
|
+
for index0 < int32(matches[0].End) && index0 < sLimit {
|
|
284
|
+
cv0 := binary.LittleEndian.Uint64(src[index0:])
|
|
285
|
+
extra0 := binary.LittleEndian.Uint32(src[index0+8:])
|
|
286
|
+
te0 := tableEntry{offset: index0, val: uint32(cv0)}
|
|
287
|
+
z.table5[z.hash5(cv0)] = te0
|
|
288
|
+
z.table8[z.hash8(cv0)] = te0
|
|
289
|
+
z.table12[z.hash12(cv0, extra0)] = te0
|
|
290
|
+
index0++
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
// We're done looking for overlapping matches; emit the ones we have.
|
|
294
|
+
|
|
295
|
+
if matches[1] != (absoluteMatch{}) {
|
|
296
|
+
if matches[1].End > matches[0].Start {
|
|
297
|
+
matches[1].End = matches[0].Start
|
|
298
|
+
}
|
|
299
|
+
if matches[1].End-matches[1].Start >= 4 {
|
|
300
|
+
e.emit(matches[1])
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
e.emit(matches[0])
|
|
304
|
+
matches = [3]absoluteMatch{}
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
dst = e.Dst
|
|
308
|
+
if e.NextEmit < len(src) {
|
|
309
|
+
dst = append(dst, Match{
|
|
310
|
+
Unmatched: len(src) - e.NextEmit,
|
|
311
|
+
})
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
return dst
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
func (z *Trio) hash5(u uint64) uint32 {
|
|
318
|
+
return uint32(((u << 24) * 889523592379) >> (64 - 16))
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
func (z *Trio) hash8(u uint64) uint32 {
|
|
322
|
+
return uint32((u * 0xcf1bbcdcb7a56463) >> (64 - 17))
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
func (z *Trio) hash12(u uint64, e uint32) uint32 {
|
|
326
|
+
return uint32((u*0xcf1bbcdcb7a56463 + uint64(e)*(2654435761<<32)) >> (64 - 18))
|
|
327
|
+
}
|