typenative 0.3.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +57 -0
- package/README.md +17 -1
- package/bin/cli.js +25 -11
- package/bin/go/convert.js +11 -1
- package/bin/index.js +221 -47
- package/bin/main.js +35 -14
- package/bin/node/assert.js +2 -2
- package/bin/node/buffer.js +9 -0
- package/bin/node/dns.js +1 -1
- package/bin/node/events.js +1 -1
- package/bin/node/fs.js +1 -1
- package/bin/node/fs_watch.js +407 -0
- package/bin/node/http2.js +33 -6
- package/bin/node/readline.js +1 -1
- package/bin/node/registry.js +2 -1
- package/bin/node/stream.js +6 -3
- package/bin/node/timers.js +1 -1
- package/bin/node/typed_arrays.js +608 -0
- package/bin/node/util.js +3 -0
- package/bin/node/zlib.js +131 -16
- package/bin/transpiler.js +635 -44
- package/bin/version.js +1 -1
- package/conformance.md +16 -4
- package/examples/README.md +54 -0
- package/examples/fibonacci/README.md +89 -0
- package/examples/fibonacci/bench.mjs +87 -0
- package/examples/fibonacci/main.ts +21 -0
- package/examples/hello-world/README.md +62 -0
- package/examples/hello-world/bench.mjs +66 -0
- package/examples/hello-world/main.ts +7 -0
- package/examples/json-server/README.md +125 -0
- package/examples/json-server/bench.mjs +171 -0
- package/examples/json-server/main.ts +67 -0
- package/govendor/github.com/andybalholm/brotli/LICENSE +19 -0
- package/govendor/github.com/andybalholm/brotli/README.md +19 -0
- package/govendor/github.com/andybalholm/brotli/backward_references.go +185 -0
- package/govendor/github.com/andybalholm/brotli/backward_references_hq.go +796 -0
- package/govendor/github.com/andybalholm/brotli/bit_cost.go +436 -0
- package/govendor/github.com/andybalholm/brotli/bit_reader.go +266 -0
- package/govendor/github.com/andybalholm/brotli/bitwriter.go +56 -0
- package/govendor/github.com/andybalholm/brotli/block_splitter.go +144 -0
- package/govendor/github.com/andybalholm/brotli/block_splitter_command.go +434 -0
- package/govendor/github.com/andybalholm/brotli/block_splitter_distance.go +433 -0
- package/govendor/github.com/andybalholm/brotli/block_splitter_literal.go +433 -0
- package/govendor/github.com/andybalholm/brotli/brotli_bit_stream.go +1435 -0
- package/govendor/github.com/andybalholm/brotli/cluster.go +30 -0
- package/govendor/github.com/andybalholm/brotli/cluster_command.go +164 -0
- package/govendor/github.com/andybalholm/brotli/cluster_distance.go +326 -0
- package/govendor/github.com/andybalholm/brotli/cluster_literal.go +326 -0
- package/govendor/github.com/andybalholm/brotli/command.go +254 -0
- package/govendor/github.com/andybalholm/brotli/compress_fragment.go +834 -0
- package/govendor/github.com/andybalholm/brotli/compress_fragment_two_pass.go +775 -0
- package/govendor/github.com/andybalholm/brotli/constants.go +77 -0
- package/govendor/github.com/andybalholm/brotli/context.go +2176 -0
- package/govendor/github.com/andybalholm/brotli/decode.go +2570 -0
- package/govendor/github.com/andybalholm/brotli/dictionary.go +122890 -0
- package/govendor/github.com/andybalholm/brotli/dictionary_hash.go +32779 -0
- package/govendor/github.com/andybalholm/brotli/encode.go +1220 -0
- package/govendor/github.com/andybalholm/brotli/encoder.go +177 -0
- package/govendor/github.com/andybalholm/brotli/encoder_dict.go +22 -0
- package/govendor/github.com/andybalholm/brotli/encoder_fast.go +201 -0
- package/govendor/github.com/andybalholm/brotli/entropy_encode.go +592 -0
- package/govendor/github.com/andybalholm/brotli/entropy_encode_static.go +4399 -0
- package/govendor/github.com/andybalholm/brotli/fast_log.go +290 -0
- package/govendor/github.com/andybalholm/brotli/find_match_length.go +45 -0
- package/govendor/github.com/andybalholm/brotli/go.mod +3 -0
- package/govendor/github.com/andybalholm/brotli/h10.go +287 -0
- package/govendor/github.com/andybalholm/brotli/h5.go +214 -0
- package/govendor/github.com/andybalholm/brotli/h6.go +216 -0
- package/govendor/github.com/andybalholm/brotli/hash.go +342 -0
- package/govendor/github.com/andybalholm/brotli/hash_composite.go +93 -0
- package/govendor/github.com/andybalholm/brotli/hash_forgetful_chain.go +252 -0
- package/govendor/github.com/andybalholm/brotli/hash_longest_match_quickly.go +214 -0
- package/govendor/github.com/andybalholm/brotli/hash_rolling.go +168 -0
- package/govendor/github.com/andybalholm/brotli/histogram.go +226 -0
- package/govendor/github.com/andybalholm/brotli/http.go +201 -0
- package/govendor/github.com/andybalholm/brotli/huffman.go +653 -0
- package/govendor/github.com/andybalholm/brotli/literal_cost.go +182 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/bargain1.go +212 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/bargain2.go +248 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/bargain3.go +278 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/emitter.go +34 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/m0.go +175 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/m4.go +339 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/matchfinder.go +103 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/pathfinder.go +328 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/textencoder.go +53 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/trio.go +327 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/zdfast.go +258 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/zfast.go +206 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/zm.go +302 -0
- package/govendor/github.com/andybalholm/brotli/memory.go +66 -0
- package/govendor/github.com/andybalholm/brotli/metablock.go +574 -0
- package/govendor/github.com/andybalholm/brotli/metablock_command.go +165 -0
- package/govendor/github.com/andybalholm/brotli/metablock_distance.go +165 -0
- package/govendor/github.com/andybalholm/brotli/metablock_literal.go +165 -0
- package/govendor/github.com/andybalholm/brotli/params.go +37 -0
- package/govendor/github.com/andybalholm/brotli/platform.go +103 -0
- package/govendor/github.com/andybalholm/brotli/prefix.go +30 -0
- package/govendor/github.com/andybalholm/brotli/prefix_dec.go +723 -0
- package/govendor/github.com/andybalholm/brotli/quality.go +196 -0
- package/govendor/github.com/andybalholm/brotli/reader.go +127 -0
- package/govendor/github.com/andybalholm/brotli/ringbuffer.go +134 -0
- package/govendor/github.com/andybalholm/brotli/state.go +313 -0
- package/govendor/github.com/andybalholm/brotli/static_dict.go +662 -0
- package/govendor/github.com/andybalholm/brotli/static_dict_lut.go +75094 -0
- package/govendor/github.com/andybalholm/brotli/symbol_list.go +22 -0
- package/govendor/github.com/andybalholm/brotli/transform.go +641 -0
- package/govendor/github.com/andybalholm/brotli/utf8_util.go +70 -0
- package/govendor/github.com/andybalholm/brotli/util.go +7 -0
- package/govendor/github.com/andybalholm/brotli/write_bits.go +52 -0
- package/govendor/github.com/andybalholm/brotli/writer.go +162 -0
- package/govendor/github.com/fsnotify/fsnotify/.mailmap +2 -0
- package/govendor/github.com/fsnotify/fsnotify/CHANGELOG.md +651 -0
- package/govendor/github.com/fsnotify/fsnotify/CONTRIBUTING.md +147 -0
- package/govendor/github.com/fsnotify/fsnotify/LICENSE +25 -0
- package/govendor/github.com/fsnotify/fsnotify/README.md +214 -0
- package/govendor/github.com/fsnotify/fsnotify/backend_fen.go +469 -0
- package/govendor/github.com/fsnotify/fsnotify/backend_inotify.go +597 -0
- package/govendor/github.com/fsnotify/fsnotify/backend_kqueue.go +735 -0
- package/govendor/github.com/fsnotify/fsnotify/backend_other.go +22 -0
- package/govendor/github.com/fsnotify/fsnotify/backend_windows.go +702 -0
- package/govendor/github.com/fsnotify/fsnotify/fsnotify.go +526 -0
- package/govendor/github.com/fsnotify/fsnotify/go.mod +3 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/darwin.go +20 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_darwin.go +15 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_dragonfly.go +15 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_freebsd.go +20 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_kqueue.go +32 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_linux.go +56 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_netbsd.go +15 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_openbsd.go +16 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_solaris.go +45 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_windows.go +40 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/freebsd.go +20 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/internal.go +2 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/unix.go +20 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/unix2.go +25 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/windows.go +40 -0
- package/govendor/github.com/fsnotify/fsnotify/shared.go +64 -0
- package/govendor/github.com/fsnotify/fsnotify/staticcheck.conf +3 -0
- package/govendor/github.com/fsnotify/fsnotify/system_bsd.go +7 -0
- package/govendor/github.com/fsnotify/fsnotify/system_darwin.go +8 -0
- package/govendor/manifest.txt +3 -1
- package/package.json +1 -1
- package/types/typenative-npm.d.ts +72 -0
- package/types/typenative.d.ts +117 -1
|
@@ -0,0 +1,339 @@
|
|
|
1
|
+
package matchfinder
|
|
2
|
+
|
|
3
|
+
import (
|
|
4
|
+
"bytes"
|
|
5
|
+
"encoding/binary"
|
|
6
|
+
"math/bits"
|
|
7
|
+
"runtime"
|
|
8
|
+
)
|
|
9
|
+
|
|
10
|
+
// M4 is an implementation of the MatchFinder
|
|
11
|
+
// interface that uses a hash table to find matches,
|
|
12
|
+
// optional match chains,
|
|
13
|
+
// and the advanced parsing technique from
|
|
14
|
+
// https://fastcompression.blogspot.com/2011/12/advanced-parsing-strategies.html.
|
|
15
|
+
type M4 struct {
|
|
16
|
+
// MaxDistance is the maximum distance (in bytes) to look back for
|
|
17
|
+
// a match. The default is 65535.
|
|
18
|
+
MaxDistance int
|
|
19
|
+
|
|
20
|
+
// MinLength is the length of the shortest match to return.
|
|
21
|
+
// The default is 4.
|
|
22
|
+
MinLength int
|
|
23
|
+
|
|
24
|
+
// HashLen is the number of bytes to use to calculate the hashes.
|
|
25
|
+
// The maximum is 8 and the default is 6.
|
|
26
|
+
HashLen int
|
|
27
|
+
|
|
28
|
+
// TableBits is the number of bits in the hash table indexes.
|
|
29
|
+
// The default is 17 (128K entries).
|
|
30
|
+
TableBits int
|
|
31
|
+
|
|
32
|
+
// ChainLength is how many entries to search on the "match chain" of older
|
|
33
|
+
// locations with the same hash as the current location.
|
|
34
|
+
ChainLength int
|
|
35
|
+
|
|
36
|
+
// DistanceBitCost is used when comparing two matches to see
|
|
37
|
+
// which is better. The comparison is primarily based on the length
|
|
38
|
+
// of the matches, but it can also take the distance into account,
|
|
39
|
+
// in terms of the number of bits needed to represent the distance.
|
|
40
|
+
// One byte of length is given a score of 256, so 32 (256/8) would
|
|
41
|
+
// be a reasonable first guess for the value of one bit.
|
|
42
|
+
// (The default is 0, which bases the comparison solely on length.)
|
|
43
|
+
DistanceBitCost int
|
|
44
|
+
|
|
45
|
+
table []uint32
|
|
46
|
+
chain []uint32
|
|
47
|
+
|
|
48
|
+
history []byte
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
func (q *M4) Reset() {
|
|
52
|
+
for i := range q.table {
|
|
53
|
+
q.table[i] = 0
|
|
54
|
+
}
|
|
55
|
+
q.history = q.history[:0]
|
|
56
|
+
q.chain = q.chain[:0]
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
func (q *M4) score(m absoluteMatch) int {
|
|
60
|
+
return (m.End-m.Start)*256 + (bits.LeadingZeros32(uint32(m.Start-m.Match))-32)*q.DistanceBitCost
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
func (q *M4) FindMatches(dst []Match, src []byte) []Match {
|
|
64
|
+
if q.MaxDistance == 0 {
|
|
65
|
+
q.MaxDistance = 65535
|
|
66
|
+
}
|
|
67
|
+
if q.MinLength == 0 {
|
|
68
|
+
q.MinLength = 4
|
|
69
|
+
}
|
|
70
|
+
if q.HashLen == 0 {
|
|
71
|
+
q.HashLen = 6
|
|
72
|
+
}
|
|
73
|
+
if q.TableBits == 0 {
|
|
74
|
+
q.TableBits = 17
|
|
75
|
+
}
|
|
76
|
+
if len(q.table) < 1<<q.TableBits {
|
|
77
|
+
q.table = make([]uint32, 1<<q.TableBits)
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
e := matchEmitter{Dst: dst}
|
|
81
|
+
|
|
82
|
+
if len(q.history) > q.MaxDistance*2 {
|
|
83
|
+
// Trim down the history buffer.
|
|
84
|
+
delta := len(q.history) - q.MaxDistance
|
|
85
|
+
copy(q.history, q.history[delta:])
|
|
86
|
+
q.history = q.history[:q.MaxDistance]
|
|
87
|
+
if q.ChainLength > 0 {
|
|
88
|
+
copy(q.chain, q.chain[delta:])
|
|
89
|
+
q.chain = q.chain[:q.MaxDistance]
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
for i, v := range q.table {
|
|
93
|
+
newV := int(v) - delta
|
|
94
|
+
if newV < 0 {
|
|
95
|
+
newV = 0
|
|
96
|
+
}
|
|
97
|
+
q.table[i] = uint32(newV)
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
// Append src to the history buffer.
|
|
102
|
+
e.NextEmit = len(q.history)
|
|
103
|
+
q.history = append(q.history, src...)
|
|
104
|
+
if q.ChainLength > 0 {
|
|
105
|
+
q.chain = append(q.chain, make([]uint32, len(src))...)
|
|
106
|
+
}
|
|
107
|
+
src = q.history
|
|
108
|
+
|
|
109
|
+
// matches stores the matches that have been found but not emitted,
|
|
110
|
+
// in reverse order. (matches[0] is the most recent one.)
|
|
111
|
+
var matches [3]absoluteMatch
|
|
112
|
+
for i := e.NextEmit; i < len(src)-7; i++ {
|
|
113
|
+
if matches[0] != (absoluteMatch{}) && i >= matches[0].End {
|
|
114
|
+
// We have found some matches, and we're far enough along that we probably
|
|
115
|
+
// won't find overlapping matches, so we might as well emit them.
|
|
116
|
+
if matches[1] != (absoluteMatch{}) {
|
|
117
|
+
if matches[1].End > matches[0].Start {
|
|
118
|
+
matches[1].End = matches[0].Start
|
|
119
|
+
}
|
|
120
|
+
if matches[1].End-matches[1].Start >= q.MinLength && q.score(matches[1]) > 0 {
|
|
121
|
+
e.emit(matches[1])
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
e.emit(matches[0])
|
|
125
|
+
matches = [3]absoluteMatch{}
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
// Look for a repeat match one byte after the current position.
|
|
129
|
+
if matches[0] == (absoluteMatch{}) && len(e.Dst) > 0 {
|
|
130
|
+
prevDistance := e.Dst[len(e.Dst)-1].Distance
|
|
131
|
+
if binary.LittleEndian.Uint32(src[i+1:]) == binary.LittleEndian.Uint32(src[i+1-prevDistance:]) && prevDistance != 0 {
|
|
132
|
+
// We have a 4-byte match.
|
|
133
|
+
m := extendMatch2(src, i+1, i+1-prevDistance, e.NextEmit+1)
|
|
134
|
+
if m.End-m.Start >= q.MinLength {
|
|
135
|
+
matches[0] = m
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
// Calculate and store the hash.
|
|
141
|
+
h := ((binary.LittleEndian.Uint64(src[i:]) & (1<<(8*q.HashLen) - 1)) * hashMul64) >> (64 - q.TableBits)
|
|
142
|
+
candidate := int(q.table[h])
|
|
143
|
+
q.table[h] = uint32(i)
|
|
144
|
+
if q.ChainLength > 0 && candidate != 0 {
|
|
145
|
+
delta := i - candidate
|
|
146
|
+
q.chain[i] = uint32(delta)
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
if i < matches[0].End && i != matches[0].End+2-q.HashLen {
|
|
150
|
+
continue
|
|
151
|
+
}
|
|
152
|
+
if candidate == 0 || i-candidate > q.MaxDistance {
|
|
153
|
+
continue
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
// Look for a match.
|
|
157
|
+
var currentMatch absoluteMatch
|
|
158
|
+
|
|
159
|
+
if binary.LittleEndian.Uint32(src[candidate:]) == binary.LittleEndian.Uint32(src[i:]) {
|
|
160
|
+
m := extendMatch2(src, i, candidate, e.NextEmit)
|
|
161
|
+
if m.End-m.Start > q.MinLength && q.score(m) > 0 {
|
|
162
|
+
currentMatch = m
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
for j := 0; j < q.ChainLength; j++ {
|
|
167
|
+
delta := q.chain[candidate]
|
|
168
|
+
if delta == 0 {
|
|
169
|
+
break
|
|
170
|
+
}
|
|
171
|
+
candidate -= int(delta)
|
|
172
|
+
if candidate <= 0 || i-candidate > q.MaxDistance {
|
|
173
|
+
break
|
|
174
|
+
}
|
|
175
|
+
if binary.LittleEndian.Uint32(src[candidate:]) == binary.LittleEndian.Uint32(src[i:]) {
|
|
176
|
+
m := extendMatch2(src, i, candidate, e.NextEmit)
|
|
177
|
+
if m.End-m.Start > q.MinLength && q.score(m) > q.score(currentMatch) {
|
|
178
|
+
currentMatch = m
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
if currentMatch.End-currentMatch.Start < q.MinLength {
|
|
184
|
+
continue
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
overlapPenalty := 0
|
|
188
|
+
if matches[0] != (absoluteMatch{}) {
|
|
189
|
+
overlapPenalty = 275
|
|
190
|
+
if currentMatch.Start <= matches[1].End {
|
|
191
|
+
// This match would completely replace the previous match,
|
|
192
|
+
// so there is no penalty for overlap.
|
|
193
|
+
overlapPenalty = 0
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
if q.score(currentMatch) <= q.score(matches[0])+overlapPenalty {
|
|
198
|
+
continue
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
matches = [3]absoluteMatch{
|
|
202
|
+
currentMatch,
|
|
203
|
+
matches[0],
|
|
204
|
+
matches[1],
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
if matches[2] == (absoluteMatch{}) {
|
|
208
|
+
continue
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
// We have three matches, so it's time to emit one and/or eliminate one.
|
|
212
|
+
switch {
|
|
213
|
+
case matches[0].Start < matches[2].End:
|
|
214
|
+
// The first and third matches overlap; discard the one in between.
|
|
215
|
+
matches = [3]absoluteMatch{
|
|
216
|
+
matches[0],
|
|
217
|
+
matches[2],
|
|
218
|
+
absoluteMatch{},
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
case matches[0].Start < matches[2].End+q.MinLength:
|
|
222
|
+
// The first and third matches don't overlap, but there's no room for
|
|
223
|
+
// another match between them. Emit the first match and discard the second.
|
|
224
|
+
e.emit(matches[2])
|
|
225
|
+
matches = [3]absoluteMatch{
|
|
226
|
+
matches[0],
|
|
227
|
+
absoluteMatch{},
|
|
228
|
+
absoluteMatch{},
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
default:
|
|
232
|
+
// Emit the first match, shortening it if necessary to avoid overlap with the second.
|
|
233
|
+
if matches[2].End > matches[1].Start {
|
|
234
|
+
matches[2].End = matches[1].Start
|
|
235
|
+
if q.ChainLength > 0 && matches[2].End-matches[2].Start >= q.MinLength {
|
|
236
|
+
// Since the match length was trimmed, we may be able to find a closer match
|
|
237
|
+
// to replace it.
|
|
238
|
+
pos := matches[2].Start
|
|
239
|
+
for {
|
|
240
|
+
delta := int(q.chain[pos])
|
|
241
|
+
if delta == 0 {
|
|
242
|
+
break
|
|
243
|
+
}
|
|
244
|
+
pos -= delta
|
|
245
|
+
if pos <= matches[2].Match {
|
|
246
|
+
break
|
|
247
|
+
}
|
|
248
|
+
if bytes.Equal(src[matches[2].Start:matches[2].End], src[pos:pos+matches[2].End-matches[2].Start]) {
|
|
249
|
+
matches[2].Match = pos
|
|
250
|
+
break
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
if matches[2].End-matches[2].Start >= q.MinLength && q.score(matches[2]) > 0 {
|
|
256
|
+
e.emit(matches[2])
|
|
257
|
+
}
|
|
258
|
+
matches[2] = absoluteMatch{}
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
// We've found all the matches now; emit the remaining ones.
|
|
263
|
+
if matches[1] != (absoluteMatch{}) {
|
|
264
|
+
if matches[1].End > matches[0].Start {
|
|
265
|
+
matches[1].End = matches[0].Start
|
|
266
|
+
}
|
|
267
|
+
if matches[1].End-matches[1].Start >= q.MinLength && q.score(matches[1]) > 0 {
|
|
268
|
+
e.emit(matches[1])
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
if matches[0] != (absoluteMatch{}) {
|
|
272
|
+
e.emit(matches[0])
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
dst = e.Dst
|
|
276
|
+
if e.NextEmit < len(src) {
|
|
277
|
+
dst = append(dst, Match{
|
|
278
|
+
Unmatched: len(src) - e.NextEmit,
|
|
279
|
+
})
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
return dst
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
const hashMul64 = 0x1E35A7BD1E35A7BD
|
|
286
|
+
|
|
287
|
+
// extendMatch returns the largest k such that k <= len(src) and that
|
|
288
|
+
// src[i:i+k-j] and src[j:k] have the same contents.
|
|
289
|
+
//
|
|
290
|
+
// It assumes that:
|
|
291
|
+
//
|
|
292
|
+
// 0 <= i && i < j && j <= len(src)
|
|
293
|
+
func extendMatch(src []byte, i, j int) int {
|
|
294
|
+
switch runtime.GOARCH {
|
|
295
|
+
case "amd64", "arm64":
|
|
296
|
+
// As long as we are 8 or more bytes before the end of src, we can load and
|
|
297
|
+
// compare 8 bytes at a time. If those 8 bytes are equal, repeat.
|
|
298
|
+
for j+8 < len(src) {
|
|
299
|
+
iBytes := binary.LittleEndian.Uint64(src[i:])
|
|
300
|
+
jBytes := binary.LittleEndian.Uint64(src[j:])
|
|
301
|
+
if iBytes != jBytes {
|
|
302
|
+
// If those 8 bytes were not equal, XOR the two 8 byte values, and return
|
|
303
|
+
// the index of the first byte that differs. The BSF instruction finds the
|
|
304
|
+
// least significant 1 bit, the amd64 architecture is little-endian, and
|
|
305
|
+
// the shift by 3 converts a bit index to a byte index.
|
|
306
|
+
return j + bits.TrailingZeros64(iBytes^jBytes)>>3
|
|
307
|
+
}
|
|
308
|
+
i, j = i+8, j+8
|
|
309
|
+
}
|
|
310
|
+
case "386":
|
|
311
|
+
// On a 32-bit CPU, we do it 4 bytes at a time.
|
|
312
|
+
for j+4 < len(src) {
|
|
313
|
+
iBytes := binary.LittleEndian.Uint32(src[i:])
|
|
314
|
+
jBytes := binary.LittleEndian.Uint32(src[j:])
|
|
315
|
+
if iBytes != jBytes {
|
|
316
|
+
return j + bits.TrailingZeros32(iBytes^jBytes)>>3
|
|
317
|
+
}
|
|
318
|
+
i, j = i+4, j+4
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
for ; j < len(src) && src[i] == src[j]; i, j = i+1, j+1 {
|
|
322
|
+
}
|
|
323
|
+
return j
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
// Given a 4-byte match at src[start] and src[candidate], extendMatch2 extends it
|
|
327
|
+
// upward as far as possible, and downward no farther than to min.
|
|
328
|
+
func extendMatch2(src []byte, start, candidate, min int) absoluteMatch {
|
|
329
|
+
end := extendMatch(src, candidate+4, start+4)
|
|
330
|
+
for start > min && candidate > 0 && src[start-1] == src[candidate-1] {
|
|
331
|
+
start--
|
|
332
|
+
candidate--
|
|
333
|
+
}
|
|
334
|
+
return absoluteMatch{
|
|
335
|
+
Start: start,
|
|
336
|
+
End: end,
|
|
337
|
+
Match: candidate,
|
|
338
|
+
}
|
|
339
|
+
}
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
// The matchfinder package defines reusable components for data compression.
|
|
2
|
+
//
|
|
3
|
+
// Many compression libraries have two main parts:
|
|
4
|
+
// - Something that looks for repeated sequences of bytes
|
|
5
|
+
// - An encoder for the compressed data format (often an entropy coder)
|
|
6
|
+
//
|
|
7
|
+
// Although these are logically two separate steps, the implementations are
|
|
8
|
+
// usually closely tied together. You can't use flate's matcher with snappy's
|
|
9
|
+
// encoder, for example. This package defines interfaces and an intermediate
|
|
10
|
+
// representation to allow mixing and matching compression components.
|
|
11
|
+
package matchfinder
|
|
12
|
+
|
|
13
|
+
import "io"
|
|
14
|
+
|
|
15
|
+
// A Match is the basic unit of LZ77 compression.
|
|
16
|
+
type Match struct {
|
|
17
|
+
Unmatched int // the number of unmatched bytes since the previous match
|
|
18
|
+
Length int // the number of bytes in the matched string; it may be 0 at the end of the input
|
|
19
|
+
Distance int // how far back in the stream to copy from
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
// A MatchFinder performs the LZ77 stage of compression, looking for matches.
|
|
23
|
+
type MatchFinder interface {
|
|
24
|
+
// FindMatches looks for matches in src, appends them to dst, and returns dst.
|
|
25
|
+
FindMatches(dst []Match, src []byte) []Match
|
|
26
|
+
|
|
27
|
+
// Reset clears any internal state, preparing the MatchFinder to be used with
|
|
28
|
+
// a new stream.
|
|
29
|
+
Reset()
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
// An Encoder encodes the data in its final format.
|
|
33
|
+
type Encoder interface {
|
|
34
|
+
// Encode appends the encoded format of src to dst, using the match
|
|
35
|
+
// information from matches.
|
|
36
|
+
Encode(dst []byte, src []byte, matches []Match, lastBlock bool) []byte
|
|
37
|
+
|
|
38
|
+
// Reset clears any internal state, preparing the Encoder to be used with
|
|
39
|
+
// a new stream.
|
|
40
|
+
Reset()
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
// A Writer uses MatchFinder and Encoder to write compressed data to Dest.
|
|
44
|
+
type Writer struct {
|
|
45
|
+
Dest io.Writer
|
|
46
|
+
MatchFinder MatchFinder
|
|
47
|
+
Encoder Encoder
|
|
48
|
+
|
|
49
|
+
// BlockSize is the number of bytes to compress at a time. If it is zero,
|
|
50
|
+
// each Write operation will be treated as one block.
|
|
51
|
+
BlockSize int
|
|
52
|
+
|
|
53
|
+
err error
|
|
54
|
+
inBuf []byte
|
|
55
|
+
outBuf []byte
|
|
56
|
+
matches []Match
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
func (w *Writer) Write(p []byte) (n int, err error) {
|
|
60
|
+
if w.err != nil {
|
|
61
|
+
return 0, w.err
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
if w.BlockSize == 0 {
|
|
65
|
+
return w.writeBlock(p, false)
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
w.inBuf = append(w.inBuf, p...)
|
|
69
|
+
var pos int
|
|
70
|
+
for pos = 0; pos+w.BlockSize <= len(w.inBuf) && w.err == nil; pos += w.BlockSize {
|
|
71
|
+
w.writeBlock(w.inBuf[pos:pos+w.BlockSize], false)
|
|
72
|
+
}
|
|
73
|
+
if pos > 0 {
|
|
74
|
+
n := copy(w.inBuf, w.inBuf[pos:])
|
|
75
|
+
w.inBuf = w.inBuf[:n]
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
return len(p), w.err
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
func (w *Writer) writeBlock(p []byte, lastBlock bool) (n int, err error) {
|
|
82
|
+
w.outBuf = w.outBuf[:0]
|
|
83
|
+
w.matches = w.MatchFinder.FindMatches(w.matches[:0], p)
|
|
84
|
+
w.outBuf = w.Encoder.Encode(w.outBuf, p, w.matches, lastBlock)
|
|
85
|
+
_, w.err = w.Dest.Write(w.outBuf)
|
|
86
|
+
return len(p), w.err
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
func (w *Writer) Close() error {
|
|
90
|
+
w.writeBlock(w.inBuf, true)
|
|
91
|
+
w.inBuf = w.inBuf[:0]
|
|
92
|
+
return w.err
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
func (w *Writer) Reset(newDest io.Writer) {
|
|
96
|
+
w.MatchFinder.Reset()
|
|
97
|
+
w.Encoder.Reset()
|
|
98
|
+
w.err = nil
|
|
99
|
+
w.inBuf = w.inBuf[:0]
|
|
100
|
+
w.outBuf = w.outBuf[:0]
|
|
101
|
+
w.matches = w.matches[:0]
|
|
102
|
+
w.Dest = newDest
|
|
103
|
+
}
|