typenative 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +71 -0
- package/README.md +17 -1
- package/bin/cli.js +25 -11
- package/bin/go/convert.js +11 -1
- package/bin/index.js +221 -44
- package/bin/main.js +35 -14
- package/bin/node/assert.js +2 -2
- package/bin/node/buffer.js +9 -0
- package/bin/node/dns.js +1 -1
- package/bin/node/events.js +1 -1
- package/bin/node/fs.js +1 -1
- package/bin/node/fs_watch.js +407 -0
- package/bin/node/http2.js +33 -6
- package/bin/node/readline.js +1 -1
- package/bin/node/registry.js +2 -1
- package/bin/node/stream.js +6 -3
- package/bin/node/timers.js +1 -1
- package/bin/node/tls.js +3 -1
- package/bin/node/typed_arrays.js +608 -0
- package/bin/node/util.js +3 -0
- package/bin/node/zlib.js +131 -16
- package/bin/transpiler.js +684 -62
- package/bin/version.js +1 -1
- package/conformance.md +16 -4
- package/examples/README.md +54 -0
- package/examples/fibonacci/README.md +89 -0
- package/examples/fibonacci/bench.mjs +87 -0
- package/examples/fibonacci/main.ts +21 -0
- package/examples/hello-world/README.md +62 -0
- package/examples/hello-world/bench.mjs +66 -0
- package/examples/hello-world/main.ts +7 -0
- package/examples/json-server/README.md +125 -0
- package/examples/json-server/bench.mjs +171 -0
- package/examples/json-server/main.ts +67 -0
- package/govendor/github.com/andybalholm/brotli/LICENSE +19 -0
- package/govendor/github.com/andybalholm/brotli/README.md +19 -0
- package/govendor/github.com/andybalholm/brotli/backward_references.go +185 -0
- package/govendor/github.com/andybalholm/brotli/backward_references_hq.go +796 -0
- package/govendor/github.com/andybalholm/brotli/bit_cost.go +436 -0
- package/govendor/github.com/andybalholm/brotli/bit_reader.go +266 -0
- package/govendor/github.com/andybalholm/brotli/bitwriter.go +56 -0
- package/govendor/github.com/andybalholm/brotli/block_splitter.go +144 -0
- package/govendor/github.com/andybalholm/brotli/block_splitter_command.go +434 -0
- package/govendor/github.com/andybalholm/brotli/block_splitter_distance.go +433 -0
- package/govendor/github.com/andybalholm/brotli/block_splitter_literal.go +433 -0
- package/govendor/github.com/andybalholm/brotli/brotli_bit_stream.go +1435 -0
- package/govendor/github.com/andybalholm/brotli/cluster.go +30 -0
- package/govendor/github.com/andybalholm/brotli/cluster_command.go +164 -0
- package/govendor/github.com/andybalholm/brotli/cluster_distance.go +326 -0
- package/govendor/github.com/andybalholm/brotli/cluster_literal.go +326 -0
- package/govendor/github.com/andybalholm/brotli/command.go +254 -0
- package/govendor/github.com/andybalholm/brotli/compress_fragment.go +834 -0
- package/govendor/github.com/andybalholm/brotli/compress_fragment_two_pass.go +775 -0
- package/govendor/github.com/andybalholm/brotli/constants.go +77 -0
- package/govendor/github.com/andybalholm/brotli/context.go +2176 -0
- package/govendor/github.com/andybalholm/brotli/decode.go +2570 -0
- package/govendor/github.com/andybalholm/brotli/dictionary.go +122890 -0
- package/govendor/github.com/andybalholm/brotli/dictionary_hash.go +32779 -0
- package/govendor/github.com/andybalholm/brotli/encode.go +1220 -0
- package/govendor/github.com/andybalholm/brotli/encoder.go +177 -0
- package/govendor/github.com/andybalholm/brotli/encoder_dict.go +22 -0
- package/govendor/github.com/andybalholm/brotli/encoder_fast.go +201 -0
- package/govendor/github.com/andybalholm/brotli/entropy_encode.go +592 -0
- package/govendor/github.com/andybalholm/brotli/entropy_encode_static.go +4399 -0
- package/govendor/github.com/andybalholm/brotli/fast_log.go +290 -0
- package/govendor/github.com/andybalholm/brotli/find_match_length.go +45 -0
- package/govendor/github.com/andybalholm/brotli/go.mod +3 -0
- package/govendor/github.com/andybalholm/brotli/h10.go +287 -0
- package/govendor/github.com/andybalholm/brotli/h5.go +214 -0
- package/govendor/github.com/andybalholm/brotli/h6.go +216 -0
- package/govendor/github.com/andybalholm/brotli/hash.go +342 -0
- package/govendor/github.com/andybalholm/brotli/hash_composite.go +93 -0
- package/govendor/github.com/andybalholm/brotli/hash_forgetful_chain.go +252 -0
- package/govendor/github.com/andybalholm/brotli/hash_longest_match_quickly.go +214 -0
- package/govendor/github.com/andybalholm/brotli/hash_rolling.go +168 -0
- package/govendor/github.com/andybalholm/brotli/histogram.go +226 -0
- package/govendor/github.com/andybalholm/brotli/http.go +201 -0
- package/govendor/github.com/andybalholm/brotli/huffman.go +653 -0
- package/govendor/github.com/andybalholm/brotli/literal_cost.go +182 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/bargain1.go +212 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/bargain2.go +248 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/bargain3.go +278 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/emitter.go +34 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/m0.go +175 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/m4.go +339 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/matchfinder.go +103 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/pathfinder.go +328 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/textencoder.go +53 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/trio.go +327 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/zdfast.go +258 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/zfast.go +206 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/zm.go +302 -0
- package/govendor/github.com/andybalholm/brotli/memory.go +66 -0
- package/govendor/github.com/andybalholm/brotli/metablock.go +574 -0
- package/govendor/github.com/andybalholm/brotli/metablock_command.go +165 -0
- package/govendor/github.com/andybalholm/brotli/metablock_distance.go +165 -0
- package/govendor/github.com/andybalholm/brotli/metablock_literal.go +165 -0
- package/govendor/github.com/andybalholm/brotli/params.go +37 -0
- package/govendor/github.com/andybalholm/brotli/platform.go +103 -0
- package/govendor/github.com/andybalholm/brotli/prefix.go +30 -0
- package/govendor/github.com/andybalholm/brotli/prefix_dec.go +723 -0
- package/govendor/github.com/andybalholm/brotli/quality.go +196 -0
- package/govendor/github.com/andybalholm/brotli/reader.go +127 -0
- package/govendor/github.com/andybalholm/brotli/ringbuffer.go +134 -0
- package/govendor/github.com/andybalholm/brotli/state.go +313 -0
- package/govendor/github.com/andybalholm/brotli/static_dict.go +662 -0
- package/govendor/github.com/andybalholm/brotli/static_dict_lut.go +75094 -0
- package/govendor/github.com/andybalholm/brotli/symbol_list.go +22 -0
- package/govendor/github.com/andybalholm/brotli/transform.go +641 -0
- package/govendor/github.com/andybalholm/brotli/utf8_util.go +70 -0
- package/govendor/github.com/andybalholm/brotli/util.go +7 -0
- package/govendor/github.com/andybalholm/brotli/write_bits.go +52 -0
- package/govendor/github.com/andybalholm/brotli/writer.go +162 -0
- package/govendor/github.com/fsnotify/fsnotify/.mailmap +2 -0
- package/govendor/github.com/fsnotify/fsnotify/CHANGELOG.md +651 -0
- package/govendor/github.com/fsnotify/fsnotify/CONTRIBUTING.md +147 -0
- package/govendor/github.com/fsnotify/fsnotify/LICENSE +25 -0
- package/govendor/github.com/fsnotify/fsnotify/README.md +214 -0
- package/govendor/github.com/fsnotify/fsnotify/backend_fen.go +469 -0
- package/govendor/github.com/fsnotify/fsnotify/backend_inotify.go +597 -0
- package/govendor/github.com/fsnotify/fsnotify/backend_kqueue.go +735 -0
- package/govendor/github.com/fsnotify/fsnotify/backend_other.go +22 -0
- package/govendor/github.com/fsnotify/fsnotify/backend_windows.go +702 -0
- package/govendor/github.com/fsnotify/fsnotify/fsnotify.go +526 -0
- package/govendor/github.com/fsnotify/fsnotify/go.mod +3 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/darwin.go +20 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_darwin.go +15 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_dragonfly.go +15 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_freebsd.go +20 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_kqueue.go +32 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_linux.go +56 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_netbsd.go +15 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_openbsd.go +16 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_solaris.go +45 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_windows.go +40 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/freebsd.go +20 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/internal.go +2 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/unix.go +20 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/unix2.go +25 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/windows.go +40 -0
- package/govendor/github.com/fsnotify/fsnotify/shared.go +64 -0
- package/govendor/github.com/fsnotify/fsnotify/staticcheck.conf +3 -0
- package/govendor/github.com/fsnotify/fsnotify/system_bsd.go +7 -0
- package/govendor/github.com/fsnotify/fsnotify/system_darwin.go +8 -0
- package/govendor/manifest.txt +3 -1
- package/package.json +1 -1
- package/types/typenative-npm.d.ts +72 -0
- package/types/typenative.d.ts +117 -1
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
package brotli
|
|
2
|
+
|
|
3
|
+
func utf8Position(last uint, c uint, clamp uint) uint {
|
|
4
|
+
if c < 128 {
|
|
5
|
+
return 0 /* Next one is the 'Byte 1' again. */
|
|
6
|
+
} else if c >= 192 { /* Next one is the 'Byte 2' of utf-8 encoding. */
|
|
7
|
+
return brotli_min_size_t(1, clamp)
|
|
8
|
+
} else {
|
|
9
|
+
/* Let's decide over the last byte if this ends the sequence. */
|
|
10
|
+
if last < 0xE0 {
|
|
11
|
+
return 0 /* Completed two or three byte coding. */ /* Next one is the 'Byte 3' of utf-8 encoding. */
|
|
12
|
+
} else {
|
|
13
|
+
return brotli_min_size_t(2, clamp)
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
func decideMultiByteStatsLevel(pos uint, len uint, mask uint, data []byte) uint {
|
|
19
|
+
var counts = [3]uint{0} /* should be 2, but 1 compresses better. */
|
|
20
|
+
var max_utf8 uint = 1
|
|
21
|
+
var last_c uint = 0
|
|
22
|
+
var i uint
|
|
23
|
+
for i = 0; i < len; i++ {
|
|
24
|
+
var c uint = uint(data[(pos+i)&mask])
|
|
25
|
+
counts[utf8Position(last_c, c, 2)]++
|
|
26
|
+
last_c = c
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
if counts[2] < 500 {
|
|
30
|
+
max_utf8 = 1
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
if counts[1]+counts[2] < 25 {
|
|
34
|
+
max_utf8 = 0
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
return max_utf8
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
func estimateBitCostsForLiteralsUTF8(pos uint, len uint, mask uint, data []byte, cost []float32) {
|
|
41
|
+
var max_utf8 uint = decideMultiByteStatsLevel(pos, uint(len), mask, data)
|
|
42
|
+
/* Bootstrap histograms. */
|
|
43
|
+
var histogram = [3][256]uint{[256]uint{0}}
|
|
44
|
+
var window_half uint = 495
|
|
45
|
+
var in_window uint = brotli_min_size_t(window_half, uint(len))
|
|
46
|
+
var in_window_utf8 = [3]uint{0}
|
|
47
|
+
/* max_utf8 is 0 (normal ASCII single byte modeling),
|
|
48
|
+
1 (for 2-byte UTF-8 modeling), or 2 (for 3-byte UTF-8 modeling). */
|
|
49
|
+
|
|
50
|
+
var i uint
|
|
51
|
+
{
|
|
52
|
+
var last_c uint = 0
|
|
53
|
+
var utf8_pos uint = 0
|
|
54
|
+
for i = 0; i < in_window; i++ {
|
|
55
|
+
var c uint = uint(data[(pos+i)&mask])
|
|
56
|
+
histogram[utf8_pos][c]++
|
|
57
|
+
in_window_utf8[utf8_pos]++
|
|
58
|
+
utf8_pos = utf8Position(last_c, c, max_utf8)
|
|
59
|
+
last_c = c
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/* Compute bit costs with sliding window. */
|
|
64
|
+
for i = 0; i < len; i++ {
|
|
65
|
+
if i >= window_half {
|
|
66
|
+
var c uint
|
|
67
|
+
var last_c uint
|
|
68
|
+
if i < window_half+1 {
|
|
69
|
+
c = 0
|
|
70
|
+
} else {
|
|
71
|
+
c = uint(data[(pos+i-window_half-1)&mask])
|
|
72
|
+
}
|
|
73
|
+
if i < window_half+2 {
|
|
74
|
+
last_c = 0
|
|
75
|
+
} else {
|
|
76
|
+
last_c = uint(data[(pos+i-window_half-2)&mask])
|
|
77
|
+
}
|
|
78
|
+
/* Remove a byte in the past. */
|
|
79
|
+
|
|
80
|
+
var utf8_pos2 uint = utf8Position(last_c, c, max_utf8)
|
|
81
|
+
histogram[utf8_pos2][data[(pos+i-window_half)&mask]]--
|
|
82
|
+
in_window_utf8[utf8_pos2]--
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
if i+window_half < len {
|
|
86
|
+
var c uint = uint(data[(pos+i+window_half-1)&mask])
|
|
87
|
+
var last_c uint = uint(data[(pos+i+window_half-2)&mask])
|
|
88
|
+
/* Add a byte in the future. */
|
|
89
|
+
|
|
90
|
+
var utf8_pos2 uint = utf8Position(last_c, c, max_utf8)
|
|
91
|
+
histogram[utf8_pos2][data[(pos+i+window_half)&mask]]++
|
|
92
|
+
in_window_utf8[utf8_pos2]++
|
|
93
|
+
}
|
|
94
|
+
{
|
|
95
|
+
var c uint
|
|
96
|
+
var last_c uint
|
|
97
|
+
if i < 1 {
|
|
98
|
+
c = 0
|
|
99
|
+
} else {
|
|
100
|
+
c = uint(data[(pos+i-1)&mask])
|
|
101
|
+
}
|
|
102
|
+
if i < 2 {
|
|
103
|
+
last_c = 0
|
|
104
|
+
} else {
|
|
105
|
+
last_c = uint(data[(pos+i-2)&mask])
|
|
106
|
+
}
|
|
107
|
+
var utf8_pos uint = utf8Position(last_c, c, max_utf8)
|
|
108
|
+
var masked_pos uint = (pos + i) & mask
|
|
109
|
+
var histo uint = histogram[utf8_pos][data[masked_pos]]
|
|
110
|
+
var lit_cost float64
|
|
111
|
+
if histo == 0 {
|
|
112
|
+
histo = 1
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
lit_cost = fastLog2(in_window_utf8[utf8_pos]) - fastLog2(histo)
|
|
116
|
+
lit_cost += 0.02905
|
|
117
|
+
if lit_cost < 1.0 {
|
|
118
|
+
lit_cost *= 0.5
|
|
119
|
+
lit_cost += 0.5
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/* Make the first bytes more expensive -- seems to help, not sure why.
|
|
123
|
+
Perhaps because the entropy source is changing its properties
|
|
124
|
+
rapidly in the beginning of the file, perhaps because the beginning
|
|
125
|
+
of the data is a statistical "anomaly". */
|
|
126
|
+
if i < 2000 {
|
|
127
|
+
lit_cost += 0.7 - (float64(2000-i) / 2000.0 * 0.35)
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
cost[i] = float32(lit_cost)
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
func estimateBitCostsForLiterals(pos uint, len uint, mask uint, data []byte, cost []float32) {
|
|
136
|
+
if isMostlyUTF8(data, pos, mask, uint(len), kMinUTF8Ratio) {
|
|
137
|
+
estimateBitCostsForLiteralsUTF8(pos, uint(len), mask, data, cost)
|
|
138
|
+
return
|
|
139
|
+
} else {
|
|
140
|
+
var histogram = [256]uint{0}
|
|
141
|
+
var window_half uint = 2000
|
|
142
|
+
var in_window uint = brotli_min_size_t(window_half, uint(len))
|
|
143
|
+
var i uint
|
|
144
|
+
/* Bootstrap histogram. */
|
|
145
|
+
for i = 0; i < in_window; i++ {
|
|
146
|
+
histogram[data[(pos+i)&mask]]++
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/* Compute bit costs with sliding window. */
|
|
150
|
+
for i = 0; i < len; i++ {
|
|
151
|
+
var histo uint
|
|
152
|
+
if i >= window_half {
|
|
153
|
+
/* Remove a byte in the past. */
|
|
154
|
+
histogram[data[(pos+i-window_half)&mask]]--
|
|
155
|
+
|
|
156
|
+
in_window--
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
if i+window_half < len {
|
|
160
|
+
/* Add a byte in the future. */
|
|
161
|
+
histogram[data[(pos+i+window_half)&mask]]++
|
|
162
|
+
|
|
163
|
+
in_window++
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
histo = histogram[data[(pos+i)&mask]]
|
|
167
|
+
if histo == 0 {
|
|
168
|
+
histo = 1
|
|
169
|
+
}
|
|
170
|
+
{
|
|
171
|
+
var lit_cost float64 = fastLog2(in_window) - fastLog2(histo)
|
|
172
|
+
lit_cost += 0.029
|
|
173
|
+
if lit_cost < 1.0 {
|
|
174
|
+
lit_cost *= 0.5
|
|
175
|
+
lit_cost += 0.5
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
cost[i] = float32(lit_cost)
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
}
|
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
package matchfinder
|
|
2
|
+
|
|
3
|
+
import (
|
|
4
|
+
"encoding/binary"
|
|
5
|
+
"math"
|
|
6
|
+
"math/bits"
|
|
7
|
+
"slices"
|
|
8
|
+
)
|
|
9
|
+
|
|
10
|
+
const (
|
|
11
|
+
bargain1TableBits = 18
|
|
12
|
+
bargain1TableSize = 1 << bargain1TableBits
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
// Bargain1 is a MatchFinder that attempts to find the encoding with the lowest
|
|
16
|
+
// "bit cost", using 1 hash length (6).
|
|
17
|
+
type Bargain1 struct {
|
|
18
|
+
MaxDistance int
|
|
19
|
+
|
|
20
|
+
// Skip is whether to look for matches at every other byte instead of every
|
|
21
|
+
// byte (to increase speed but decrease compression).
|
|
22
|
+
Skip bool
|
|
23
|
+
|
|
24
|
+
history []byte
|
|
25
|
+
table6 [bargain1TableSize]tableEntry
|
|
26
|
+
|
|
27
|
+
// holding onto buffers to reduce allocations:
|
|
28
|
+
|
|
29
|
+
arrivals []arrival
|
|
30
|
+
matches []Match
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
func (z *Bargain1) Reset() {
|
|
34
|
+
z.table6 = [bargain1TableSize]tableEntry{}
|
|
35
|
+
z.history = z.history[:0]
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
func (z *Bargain1) FindMatches(dst []Match, src []byte) []Match {
|
|
39
|
+
if z.MaxDistance == 0 {
|
|
40
|
+
z.MaxDistance = 1 << 16
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
var histogram [256]uint32
|
|
44
|
+
for _, b := range src {
|
|
45
|
+
histogram[b]++
|
|
46
|
+
}
|
|
47
|
+
var byteCost [256]float32
|
|
48
|
+
for b, n := range histogram {
|
|
49
|
+
cost := max(math.Log2(float64(len(src))/float64(n)), 1)
|
|
50
|
+
byteCost[b] = float32(cost)
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
// Each element in arrivals corresponds to the position just after
|
|
54
|
+
// the corresponding byte in src.
|
|
55
|
+
arrivals := z.arrivals
|
|
56
|
+
if len(arrivals) < len(src) {
|
|
57
|
+
arrivals = make([]arrival, len(src))
|
|
58
|
+
z.arrivals = arrivals
|
|
59
|
+
} else {
|
|
60
|
+
arrivals = arrivals[:len(src)]
|
|
61
|
+
for i := range arrivals {
|
|
62
|
+
arrivals[i] = arrival{}
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
if len(z.history) > z.MaxDistance*2 {
|
|
67
|
+
delta := len(z.history) - z.MaxDistance
|
|
68
|
+
copy(z.history, z.history[delta:])
|
|
69
|
+
z.history = z.history[:z.MaxDistance]
|
|
70
|
+
|
|
71
|
+
for i := range z.table6 {
|
|
72
|
+
v := z.table6[i].offset
|
|
73
|
+
v -= int32(delta)
|
|
74
|
+
if v < 0 {
|
|
75
|
+
z.table6[i] = tableEntry{}
|
|
76
|
+
} else {
|
|
77
|
+
z.table6[i].offset = v
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
historyLen := len(z.history)
|
|
83
|
+
z.history = append(z.history, src...)
|
|
84
|
+
src = z.history
|
|
85
|
+
|
|
86
|
+
addMatch := func(m absoluteMatch, unmatched int, repeat bool) {
|
|
87
|
+
var startCost float32
|
|
88
|
+
if m.Start > historyLen {
|
|
89
|
+
startCost = arrivals[m.Start-historyLen-1].cost
|
|
90
|
+
}
|
|
91
|
+
insertCost := float32(bits.Len(uint(unmatched)))
|
|
92
|
+
var distanceCost float32
|
|
93
|
+
if !repeat {
|
|
94
|
+
distanceCost = float32(bits.Len(uint(m.Start - m.Match)))
|
|
95
|
+
}
|
|
96
|
+
cost := startCost + baseMatchCost + insertCost + distanceCost
|
|
97
|
+
for j := m.End; j >= m.Start+3; j-- {
|
|
98
|
+
a := &arrivals[j-historyLen-1]
|
|
99
|
+
if a.cost > 0 && a.cost <= cost {
|
|
100
|
+
break
|
|
101
|
+
}
|
|
102
|
+
*a = arrival{
|
|
103
|
+
length: uint32(j - m.Start),
|
|
104
|
+
distance: uint32(m.Start - m.Match),
|
|
105
|
+
cost: cost,
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
var nextOverlapSearch int
|
|
111
|
+
|
|
112
|
+
for i := historyLen; i < len(src); i++ {
|
|
113
|
+
var arrivedHere arrival
|
|
114
|
+
if i > historyLen {
|
|
115
|
+
arrivedHere = arrivals[i-historyLen-1]
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
unmatched := 0
|
|
119
|
+
if arrivedHere.distance == 0 {
|
|
120
|
+
unmatched = int(arrivedHere.length)
|
|
121
|
+
}
|
|
122
|
+
prevDistance := 0
|
|
123
|
+
if unmatched != 0 && i-unmatched > historyLen {
|
|
124
|
+
prevDistance = int(arrivals[i-historyLen-1-unmatched].distance)
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
literalCost := byteCost[src[i]]
|
|
128
|
+
nextArrival := &arrivals[i-historyLen]
|
|
129
|
+
if nextArrival.cost == 0 || arrivedHere.cost+literalCost < nextArrival.cost {
|
|
130
|
+
*nextArrival = arrival{
|
|
131
|
+
cost: arrivedHere.cost + literalCost,
|
|
132
|
+
length: uint32(unmatched + 1),
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
if i > len(src)-8 {
|
|
137
|
+
// There's no room to check hashes.
|
|
138
|
+
continue
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
cv := binary.LittleEndian.Uint64(src[i:])
|
|
142
|
+
nextHash6 := z.hash6(cv)
|
|
143
|
+
candidate6 := z.table6[nextHash6]
|
|
144
|
+
|
|
145
|
+
entry := tableEntry{offset: int32(i), val: uint32(cv)}
|
|
146
|
+
z.table6[nextHash6] = entry
|
|
147
|
+
|
|
148
|
+
// Look for a repeat match, unless there is no previous distance, or a match at
|
|
149
|
+
// that distance has already been found.
|
|
150
|
+
if prevDistance != 0 && prevDistance != int(arrivals[i-historyLen-1+4].distance) {
|
|
151
|
+
repIndex := i - prevDistance
|
|
152
|
+
if repIndex >= 0 && binary.LittleEndian.Uint32(src[repIndex:]) == uint32(cv) {
|
|
153
|
+
// We have a repeat of the previous match distance.
|
|
154
|
+
m := extendMatch2(src, i, repIndex, i)
|
|
155
|
+
addMatch(m, unmatched, true)
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
if z.Skip && i%2 != 0 {
|
|
160
|
+
continue
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
nextByteIsUnmatched := arrivals[i-historyLen-1+1].distance == 0
|
|
164
|
+
|
|
165
|
+
if unmatched > 0 || i >= nextOverlapSearch || nextByteIsUnmatched {
|
|
166
|
+
if int(candidate6.offset) < i && i-int(candidate6.offset) < z.MaxDistance && uint32(cv) == candidate6.val &&
|
|
167
|
+
binary.LittleEndian.Uint32(src[candidate6.offset:]) == uint32(cv) {
|
|
168
|
+
m := extendMatch2(src, i, int(candidate6.offset), historyLen)
|
|
169
|
+
delta := i - m.Start
|
|
170
|
+
if delta == 0 {
|
|
171
|
+
addMatch(m, unmatched, false)
|
|
172
|
+
} else {
|
|
173
|
+
// The match was extended backwards. Add it with and without the extra.
|
|
174
|
+
addMatch(m, max(unmatched-delta, 0), false)
|
|
175
|
+
m.Start += delta
|
|
176
|
+
m.Match += delta
|
|
177
|
+
addMatch(m, unmatched, false)
|
|
178
|
+
}
|
|
179
|
+
nextOverlapSearch = max(nextOverlapSearch, m.Start+1, m.End-4)
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
// We've found the shortest path; now walk it backward and store the matches.
|
|
185
|
+
matches := z.matches[:0]
|
|
186
|
+
i := len(arrivals) - 1
|
|
187
|
+
for i >= 0 {
|
|
188
|
+
a := arrivals[i]
|
|
189
|
+
if a.distance > 0 {
|
|
190
|
+
matches = append(matches, Match{
|
|
191
|
+
Length: int(a.length),
|
|
192
|
+
Distance: int(a.distance),
|
|
193
|
+
})
|
|
194
|
+
i -= int(a.length)
|
|
195
|
+
} else {
|
|
196
|
+
if len(matches) == 0 {
|
|
197
|
+
matches = append(matches, Match{})
|
|
198
|
+
}
|
|
199
|
+
matches[len(matches)-1].Unmatched = int(a.length)
|
|
200
|
+
i -= int(a.length)
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
z.matches = matches
|
|
204
|
+
|
|
205
|
+
slices.Reverse(matches)
|
|
206
|
+
|
|
207
|
+
return append(dst, matches...)
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
func (z *Bargain1) hash6(u uint64) uint32 {
|
|
211
|
+
return uint32(((u << 16) * 227718039650203) >> (64 - bargain1TableBits))
|
|
212
|
+
}
|
|
@@ -0,0 +1,248 @@
|
|
|
1
|
+
package matchfinder
|
|
2
|
+
|
|
3
|
+
import (
|
|
4
|
+
"encoding/binary"
|
|
5
|
+
"math"
|
|
6
|
+
"math/bits"
|
|
7
|
+
"slices"
|
|
8
|
+
)
|
|
9
|
+
|
|
10
|
+
const (
|
|
11
|
+
bargain2TableBits = 18
|
|
12
|
+
bargain2TableSize = 1 << bargain2TableBits
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
// Bargain2 is a MatchFinder that attempts to find the encoding with the lowest
|
|
16
|
+
// "bit cost", using 2 hash lengths (5 and 8).
|
|
17
|
+
type Bargain2 struct {
|
|
18
|
+
MaxDistance int
|
|
19
|
+
|
|
20
|
+
// Skip is whether to look for matches at every other byte instead of every
|
|
21
|
+
// byte (to increase speed but decrease compression).
|
|
22
|
+
Skip bool
|
|
23
|
+
|
|
24
|
+
history []byte
|
|
25
|
+
table5 [bargain2TableSize]tableEntry
|
|
26
|
+
table8 [bargain2TableSize]tableEntry
|
|
27
|
+
|
|
28
|
+
// holding onto buffers to reduce allocations:
|
|
29
|
+
|
|
30
|
+
arrivals []arrival
|
|
31
|
+
matches []Match
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
func (z *Bargain2) Reset() {
|
|
35
|
+
z.table5 = [bargain2TableSize]tableEntry{}
|
|
36
|
+
z.table8 = [bargain2TableSize]tableEntry{}
|
|
37
|
+
z.history = z.history[:0]
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
func (z *Bargain2) FindMatches(dst []Match, src []byte) []Match {
|
|
41
|
+
if z.MaxDistance == 0 {
|
|
42
|
+
z.MaxDistance = 1 << 16
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
var histogram [256]uint32
|
|
46
|
+
for _, b := range src {
|
|
47
|
+
histogram[b]++
|
|
48
|
+
}
|
|
49
|
+
var byteCost [256]float32
|
|
50
|
+
for b, n := range histogram {
|
|
51
|
+
cost := max(math.Log2(float64(len(src))/float64(n)), 1)
|
|
52
|
+
byteCost[b] = float32(cost)
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
// Each element in arrivals corresponds to the position just after
|
|
56
|
+
// the corresponding byte in src.
|
|
57
|
+
arrivals := z.arrivals
|
|
58
|
+
if len(arrivals) < len(src) {
|
|
59
|
+
arrivals = make([]arrival, len(src))
|
|
60
|
+
z.arrivals = arrivals
|
|
61
|
+
} else {
|
|
62
|
+
arrivals = arrivals[:len(src)]
|
|
63
|
+
for i := range arrivals {
|
|
64
|
+
arrivals[i] = arrival{}
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
if len(z.history) > z.MaxDistance*2 {
|
|
69
|
+
delta := len(z.history) - z.MaxDistance
|
|
70
|
+
copy(z.history, z.history[delta:])
|
|
71
|
+
z.history = z.history[:z.MaxDistance]
|
|
72
|
+
|
|
73
|
+
for i := range z.table5 {
|
|
74
|
+
v := z.table5[i].offset
|
|
75
|
+
v -= int32(delta)
|
|
76
|
+
if v < 0 {
|
|
77
|
+
z.table5[i] = tableEntry{}
|
|
78
|
+
} else {
|
|
79
|
+
z.table5[i].offset = v
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
for i := range z.table8 {
|
|
83
|
+
v := z.table8[i].offset
|
|
84
|
+
v -= int32(delta)
|
|
85
|
+
if v < 0 {
|
|
86
|
+
z.table8[i] = tableEntry{}
|
|
87
|
+
} else {
|
|
88
|
+
z.table8[i].offset = v
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
historyLen := len(z.history)
|
|
94
|
+
z.history = append(z.history, src...)
|
|
95
|
+
src = z.history
|
|
96
|
+
|
|
97
|
+
addMatch := func(m absoluteMatch, unmatched int, repeat bool) {
|
|
98
|
+
var startCost float32
|
|
99
|
+
if m.Start > historyLen {
|
|
100
|
+
startCost = arrivals[m.Start-historyLen-1].cost
|
|
101
|
+
}
|
|
102
|
+
insertCost := float32(bits.Len(uint(unmatched)))
|
|
103
|
+
var distanceCost float32
|
|
104
|
+
if !repeat {
|
|
105
|
+
distanceCost = float32(bits.Len(uint(m.Start - m.Match)))
|
|
106
|
+
}
|
|
107
|
+
cost := startCost + baseMatchCost + insertCost + distanceCost
|
|
108
|
+
for j := m.End; j >= m.Start+3; j-- {
|
|
109
|
+
a := &arrivals[j-historyLen-1]
|
|
110
|
+
if a.cost > 0 && a.cost <= cost {
|
|
111
|
+
break
|
|
112
|
+
}
|
|
113
|
+
*a = arrival{
|
|
114
|
+
length: uint32(j - m.Start),
|
|
115
|
+
distance: uint32(m.Start - m.Match),
|
|
116
|
+
cost: cost,
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
var nextOverlapSearch int
|
|
122
|
+
|
|
123
|
+
for i := historyLen; i < len(src); i++ {
|
|
124
|
+
var arrivedHere arrival
|
|
125
|
+
if i > historyLen {
|
|
126
|
+
arrivedHere = arrivals[i-historyLen-1]
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
unmatched := 0
|
|
130
|
+
if arrivedHere.distance == 0 {
|
|
131
|
+
unmatched = int(arrivedHere.length)
|
|
132
|
+
}
|
|
133
|
+
prevDistance := 0
|
|
134
|
+
if unmatched != 0 && i-unmatched > historyLen {
|
|
135
|
+
prevDistance = int(arrivals[i-historyLen-1-unmatched].distance)
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
literalCost := byteCost[src[i]]
|
|
139
|
+
nextArrival := &arrivals[i-historyLen]
|
|
140
|
+
if nextArrival.cost == 0 || arrivedHere.cost+literalCost < nextArrival.cost {
|
|
141
|
+
*nextArrival = arrival{
|
|
142
|
+
cost: arrivedHere.cost + literalCost,
|
|
143
|
+
length: uint32(unmatched + 1),
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
if i > len(src)-8 {
|
|
148
|
+
// There's no room to check hashes.
|
|
149
|
+
continue
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
cv := binary.LittleEndian.Uint64(src[i:])
|
|
153
|
+
nextHash5 := z.hash5(cv)
|
|
154
|
+
nextHash8 := z.hash8(cv)
|
|
155
|
+
candidate5 := z.table5[nextHash5]
|
|
156
|
+
candidate8 := z.table8[nextHash8]
|
|
157
|
+
|
|
158
|
+
entry := tableEntry{offset: int32(i), val: uint32(cv)}
|
|
159
|
+
z.table5[nextHash5] = entry
|
|
160
|
+
z.table8[nextHash8] = entry
|
|
161
|
+
|
|
162
|
+
// Look for a repeat match, unless there is no previous distance, or a match at
|
|
163
|
+
// that distance has already been found.
|
|
164
|
+
if prevDistance != 0 && prevDistance != int(arrivals[i-historyLen-1+4].distance) {
|
|
165
|
+
repIndex := i - prevDistance
|
|
166
|
+
if repIndex >= 0 && binary.LittleEndian.Uint32(src[repIndex:]) == uint32(cv) {
|
|
167
|
+
// We have a repeat of the previous match distance.
|
|
168
|
+
m := extendMatch2(src, i, repIndex, i)
|
|
169
|
+
addMatch(m, unmatched, true)
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
if z.Skip {
|
|
174
|
+
if i%2 != 0 {
|
|
175
|
+
continue
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
nextByteIsUnmatched := arrivals[i-historyLen-1+1].distance == 0
|
|
180
|
+
|
|
181
|
+
if unmatched > 0 || i >= nextOverlapSearch || nextByteIsUnmatched {
|
|
182
|
+
if int(candidate5.offset) < i && i-int(candidate5.offset) < z.MaxDistance && uint32(cv) == candidate5.val &&
|
|
183
|
+
binary.LittleEndian.Uint32(src[candidate5.offset:]) == uint32(cv) {
|
|
184
|
+
m := extendMatch2(src, i, int(candidate5.offset), historyLen)
|
|
185
|
+
delta := i - m.Start
|
|
186
|
+
if delta == 0 {
|
|
187
|
+
addMatch(m, unmatched, false)
|
|
188
|
+
} else {
|
|
189
|
+
// The match was extended backwards. Add it with and without the extra.
|
|
190
|
+
addMatch(m, max(unmatched-delta, 0), false)
|
|
191
|
+
m.Start += delta
|
|
192
|
+
m.Match += delta
|
|
193
|
+
addMatch(m, unmatched, false)
|
|
194
|
+
}
|
|
195
|
+
nextOverlapSearch = max(nextOverlapSearch, m.Start+1, m.End-6)
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
if int(candidate8.offset) < i && i-int(candidate8.offset) < z.MaxDistance && uint32(cv) == candidate8.val &&
|
|
199
|
+
binary.LittleEndian.Uint32(src[candidate8.offset:]) == uint32(cv) {
|
|
200
|
+
m := extendMatch2(src, i, int(candidate8.offset), historyLen)
|
|
201
|
+
delta := i - m.Start
|
|
202
|
+
if delta == 0 {
|
|
203
|
+
addMatch(m, unmatched, false)
|
|
204
|
+
} else {
|
|
205
|
+
// The match was extended backwards. Add it with and without the extra.
|
|
206
|
+
addMatch(m, max(unmatched-delta, 0), false)
|
|
207
|
+
m.Start += delta
|
|
208
|
+
m.Match += delta
|
|
209
|
+
addMatch(m, unmatched, false)
|
|
210
|
+
}
|
|
211
|
+
nextOverlapSearch = max(nextOverlapSearch, m.Start+1, m.End-6)
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
// We've found the shortest path; now walk it backward and store the matches.
|
|
217
|
+
matches := z.matches[:0]
|
|
218
|
+
i := len(arrivals) - 1
|
|
219
|
+
for i >= 0 {
|
|
220
|
+
a := arrivals[i]
|
|
221
|
+
if a.distance > 0 {
|
|
222
|
+
matches = append(matches, Match{
|
|
223
|
+
Length: int(a.length),
|
|
224
|
+
Distance: int(a.distance),
|
|
225
|
+
})
|
|
226
|
+
i -= int(a.length)
|
|
227
|
+
} else {
|
|
228
|
+
if len(matches) == 0 {
|
|
229
|
+
matches = append(matches, Match{})
|
|
230
|
+
}
|
|
231
|
+
matches[len(matches)-1].Unmatched = int(a.length)
|
|
232
|
+
i -= int(a.length)
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
z.matches = matches
|
|
236
|
+
|
|
237
|
+
slices.Reverse(matches)
|
|
238
|
+
|
|
239
|
+
return append(dst, matches...)
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
func (z *Bargain2) hash5(u uint64) uint32 {
|
|
243
|
+
return uint32(((u << 24) * 889523592379) >> (64 - bargain2TableBits))
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
func (z *Bargain2) hash8(u uint64) uint32 {
|
|
247
|
+
return uint32((u * 0xcf1bbcdcb7a56463) >> (64 - bargain2TableBits))
|
|
248
|
+
}
|