typenative 0.3.1 → 0.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (170) hide show
  1. package/.github/workflows/ci.yml +2 -0
  2. package/CHANGELOG.md +258 -0
  3. package/CONTRIBUTION.md +1 -1
  4. package/README.md +17 -1
  5. package/bin/cli.js +26 -11
  6. package/bin/go/bridge.js +74 -14
  7. package/bin/go/convert.js +59 -1
  8. package/bin/go/metadata.js +13 -4
  9. package/bin/go/overrides.js +4 -6
  10. package/bin/go/stdlib.generated.js +2 -0
  11. package/bin/index.js +338 -47
  12. package/bin/main.js +36 -14
  13. package/bin/node/assert.js +2 -2
  14. package/bin/node/buffer.js +9 -0
  15. package/bin/node/child_process.js +51 -3
  16. package/bin/node/cluster.js +41 -3
  17. package/bin/node/core.js +1 -1
  18. package/bin/node/crypto.js +205 -4
  19. package/bin/node/dgram.js +230 -6
  20. package/bin/node/dns.js +471 -8
  21. package/bin/node/errno_tables.js +445 -0
  22. package/bin/node/events.js +1 -1
  23. package/bin/node/fs.js +22 -15
  24. package/bin/node/fs_watch.js +407 -0
  25. package/bin/node/http.js +128 -30
  26. package/bin/node/http2.js +37 -7
  27. package/bin/node/module_v8.js +142 -0
  28. package/bin/node/net.js +9 -6
  29. package/bin/node/process.js +242 -0
  30. package/bin/node/readline.js +272 -1
  31. package/bin/node/registry.js +3 -1
  32. package/bin/node/sqlite.js +60 -12
  33. package/bin/node/stream.js +145 -10
  34. package/bin/node/test.js +311 -19
  35. package/bin/node/timers.js +1 -1
  36. package/bin/node/typed_arrays.js +608 -0
  37. package/bin/node/url.js +23 -5
  38. package/bin/node/util.js +21 -1
  39. package/bin/node/util_extra.js +6 -30
  40. package/bin/node/worker_threads.js +79 -19
  41. package/bin/node/zlib.js +131 -16
  42. package/bin/transpiler.js +4729 -616
  43. package/bin/typecheck-node.js +1 -0
  44. package/bin/version.js +1 -1
  45. package/conformance.md +1030 -877
  46. package/examples/README.md +54 -0
  47. package/examples/fibonacci/README.md +89 -0
  48. package/examples/fibonacci/bench.mjs +87 -0
  49. package/examples/fibonacci/main.ts +21 -0
  50. package/examples/hello-world/README.md +62 -0
  51. package/examples/hello-world/bench.mjs +66 -0
  52. package/examples/hello-world/main.ts +7 -0
  53. package/examples/json-server/README.md +127 -0
  54. package/examples/json-server/bench.mjs +263 -0
  55. package/examples/json-server/main.ts +67 -0
  56. package/govendor/github.com/andybalholm/brotli/LICENSE +19 -0
  57. package/govendor/github.com/andybalholm/brotli/README.md +19 -0
  58. package/govendor/github.com/andybalholm/brotli/backward_references.go +185 -0
  59. package/govendor/github.com/andybalholm/brotli/backward_references_hq.go +796 -0
  60. package/govendor/github.com/andybalholm/brotli/bit_cost.go +436 -0
  61. package/govendor/github.com/andybalholm/brotli/bit_reader.go +266 -0
  62. package/govendor/github.com/andybalholm/brotli/bitwriter.go +56 -0
  63. package/govendor/github.com/andybalholm/brotli/block_splitter.go +144 -0
  64. package/govendor/github.com/andybalholm/brotli/block_splitter_command.go +434 -0
  65. package/govendor/github.com/andybalholm/brotli/block_splitter_distance.go +433 -0
  66. package/govendor/github.com/andybalholm/brotli/block_splitter_literal.go +433 -0
  67. package/govendor/github.com/andybalholm/brotli/brotli_bit_stream.go +1435 -0
  68. package/govendor/github.com/andybalholm/brotli/cluster.go +30 -0
  69. package/govendor/github.com/andybalholm/brotli/cluster_command.go +164 -0
  70. package/govendor/github.com/andybalholm/brotli/cluster_distance.go +326 -0
  71. package/govendor/github.com/andybalholm/brotli/cluster_literal.go +326 -0
  72. package/govendor/github.com/andybalholm/brotli/command.go +254 -0
  73. package/govendor/github.com/andybalholm/brotli/compress_fragment.go +834 -0
  74. package/govendor/github.com/andybalholm/brotli/compress_fragment_two_pass.go +775 -0
  75. package/govendor/github.com/andybalholm/brotli/constants.go +77 -0
  76. package/govendor/github.com/andybalholm/brotli/context.go +2176 -0
  77. package/govendor/github.com/andybalholm/brotli/decode.go +2570 -0
  78. package/govendor/github.com/andybalholm/brotli/dictionary.go +122890 -0
  79. package/govendor/github.com/andybalholm/brotli/dictionary_hash.go +32779 -0
  80. package/govendor/github.com/andybalholm/brotli/encode.go +1220 -0
  81. package/govendor/github.com/andybalholm/brotli/encoder.go +177 -0
  82. package/govendor/github.com/andybalholm/brotli/encoder_dict.go +22 -0
  83. package/govendor/github.com/andybalholm/brotli/encoder_fast.go +201 -0
  84. package/govendor/github.com/andybalholm/brotli/entropy_encode.go +592 -0
  85. package/govendor/github.com/andybalholm/brotli/entropy_encode_static.go +4399 -0
  86. package/govendor/github.com/andybalholm/brotli/fast_log.go +290 -0
  87. package/govendor/github.com/andybalholm/brotli/find_match_length.go +45 -0
  88. package/govendor/github.com/andybalholm/brotli/go.mod +3 -0
  89. package/govendor/github.com/andybalholm/brotli/h10.go +287 -0
  90. package/govendor/github.com/andybalholm/brotli/h5.go +214 -0
  91. package/govendor/github.com/andybalholm/brotli/h6.go +216 -0
  92. package/govendor/github.com/andybalholm/brotli/hash.go +342 -0
  93. package/govendor/github.com/andybalholm/brotli/hash_composite.go +93 -0
  94. package/govendor/github.com/andybalholm/brotli/hash_forgetful_chain.go +252 -0
  95. package/govendor/github.com/andybalholm/brotli/hash_longest_match_quickly.go +214 -0
  96. package/govendor/github.com/andybalholm/brotli/hash_rolling.go +168 -0
  97. package/govendor/github.com/andybalholm/brotli/histogram.go +226 -0
  98. package/govendor/github.com/andybalholm/brotli/http.go +201 -0
  99. package/govendor/github.com/andybalholm/brotli/huffman.go +653 -0
  100. package/govendor/github.com/andybalholm/brotli/literal_cost.go +182 -0
  101. package/govendor/github.com/andybalholm/brotli/matchfinder/bargain1.go +212 -0
  102. package/govendor/github.com/andybalholm/brotli/matchfinder/bargain2.go +248 -0
  103. package/govendor/github.com/andybalholm/brotli/matchfinder/bargain3.go +278 -0
  104. package/govendor/github.com/andybalholm/brotli/matchfinder/emitter.go +34 -0
  105. package/govendor/github.com/andybalholm/brotli/matchfinder/m0.go +175 -0
  106. package/govendor/github.com/andybalholm/brotli/matchfinder/m4.go +339 -0
  107. package/govendor/github.com/andybalholm/brotli/matchfinder/matchfinder.go +103 -0
  108. package/govendor/github.com/andybalholm/brotli/matchfinder/pathfinder.go +328 -0
  109. package/govendor/github.com/andybalholm/brotli/matchfinder/textencoder.go +53 -0
  110. package/govendor/github.com/andybalholm/brotli/matchfinder/trio.go +327 -0
  111. package/govendor/github.com/andybalholm/brotli/matchfinder/zdfast.go +258 -0
  112. package/govendor/github.com/andybalholm/brotli/matchfinder/zfast.go +206 -0
  113. package/govendor/github.com/andybalholm/brotli/matchfinder/zm.go +302 -0
  114. package/govendor/github.com/andybalholm/brotli/memory.go +66 -0
  115. package/govendor/github.com/andybalholm/brotli/metablock.go +574 -0
  116. package/govendor/github.com/andybalholm/brotli/metablock_command.go +165 -0
  117. package/govendor/github.com/andybalholm/brotli/metablock_distance.go +165 -0
  118. package/govendor/github.com/andybalholm/brotli/metablock_literal.go +165 -0
  119. package/govendor/github.com/andybalholm/brotli/params.go +37 -0
  120. package/govendor/github.com/andybalholm/brotli/platform.go +103 -0
  121. package/govendor/github.com/andybalholm/brotli/prefix.go +30 -0
  122. package/govendor/github.com/andybalholm/brotli/prefix_dec.go +723 -0
  123. package/govendor/github.com/andybalholm/brotli/quality.go +196 -0
  124. package/govendor/github.com/andybalholm/brotli/reader.go +127 -0
  125. package/govendor/github.com/andybalholm/brotli/ringbuffer.go +134 -0
  126. package/govendor/github.com/andybalholm/brotli/state.go +313 -0
  127. package/govendor/github.com/andybalholm/brotli/static_dict.go +662 -0
  128. package/govendor/github.com/andybalholm/brotli/static_dict_lut.go +75094 -0
  129. package/govendor/github.com/andybalholm/brotli/symbol_list.go +22 -0
  130. package/govendor/github.com/andybalholm/brotli/transform.go +641 -0
  131. package/govendor/github.com/andybalholm/brotli/utf8_util.go +70 -0
  132. package/govendor/github.com/andybalholm/brotli/util.go +7 -0
  133. package/govendor/github.com/andybalholm/brotli/write_bits.go +52 -0
  134. package/govendor/github.com/andybalholm/brotli/writer.go +162 -0
  135. package/govendor/github.com/fsnotify/fsnotify/.mailmap +2 -0
  136. package/govendor/github.com/fsnotify/fsnotify/CHANGELOG.md +651 -0
  137. package/govendor/github.com/fsnotify/fsnotify/CONTRIBUTING.md +147 -0
  138. package/govendor/github.com/fsnotify/fsnotify/LICENSE +25 -0
  139. package/govendor/github.com/fsnotify/fsnotify/README.md +214 -0
  140. package/govendor/github.com/fsnotify/fsnotify/backend_fen.go +469 -0
  141. package/govendor/github.com/fsnotify/fsnotify/backend_inotify.go +597 -0
  142. package/govendor/github.com/fsnotify/fsnotify/backend_kqueue.go +735 -0
  143. package/govendor/github.com/fsnotify/fsnotify/backend_other.go +22 -0
  144. package/govendor/github.com/fsnotify/fsnotify/backend_windows.go +702 -0
  145. package/govendor/github.com/fsnotify/fsnotify/fsnotify.go +526 -0
  146. package/govendor/github.com/fsnotify/fsnotify/go.mod +3 -0
  147. package/govendor/github.com/fsnotify/fsnotify/internal/darwin.go +20 -0
  148. package/govendor/github.com/fsnotify/fsnotify/internal/debug_darwin.go +15 -0
  149. package/govendor/github.com/fsnotify/fsnotify/internal/debug_dragonfly.go +15 -0
  150. package/govendor/github.com/fsnotify/fsnotify/internal/debug_freebsd.go +20 -0
  151. package/govendor/github.com/fsnotify/fsnotify/internal/debug_kqueue.go +32 -0
  152. package/govendor/github.com/fsnotify/fsnotify/internal/debug_linux.go +56 -0
  153. package/govendor/github.com/fsnotify/fsnotify/internal/debug_netbsd.go +15 -0
  154. package/govendor/github.com/fsnotify/fsnotify/internal/debug_openbsd.go +16 -0
  155. package/govendor/github.com/fsnotify/fsnotify/internal/debug_solaris.go +45 -0
  156. package/govendor/github.com/fsnotify/fsnotify/internal/debug_windows.go +40 -0
  157. package/govendor/github.com/fsnotify/fsnotify/internal/freebsd.go +20 -0
  158. package/govendor/github.com/fsnotify/fsnotify/internal/internal.go +2 -0
  159. package/govendor/github.com/fsnotify/fsnotify/internal/unix.go +20 -0
  160. package/govendor/github.com/fsnotify/fsnotify/internal/unix2.go +25 -0
  161. package/govendor/github.com/fsnotify/fsnotify/internal/windows.go +40 -0
  162. package/govendor/github.com/fsnotify/fsnotify/shared.go +64 -0
  163. package/govendor/github.com/fsnotify/fsnotify/staticcheck.conf +3 -0
  164. package/govendor/github.com/fsnotify/fsnotify/system_bsd.go +7 -0
  165. package/govendor/github.com/fsnotify/fsnotify/system_darwin.go +8 -0
  166. package/govendor/manifest.txt +3 -1
  167. package/package.json +1 -1
  168. package/types/typenative-go.d.ts +23 -0
  169. package/types/typenative-npm.d.ts +241 -5
  170. package/types/typenative.d.ts +302 -12
@@ -0,0 +1,328 @@
1
+ package matchfinder
2
+
3
+ import (
4
+ "encoding/binary"
5
+ "math"
6
+ "math/bits"
7
+ "slices"
8
+ )
9
+
10
+ // Pathfinder is a MatchFinder that uses hash chains to find matches, and a
11
+ // shortest-path optimizer to choose which matches to use.
12
+ type Pathfinder struct {
13
+ // MaxDistance is the maximum distance (in bytes) to look back for
14
+ // a match. The default is 65535.
15
+ MaxDistance int
16
+
17
+ // MinLength is the length of the shortest match to return.
18
+ // The default is 4.
19
+ MinLength int
20
+
21
+ // HashLen is the number of bytes to use to calculate the hashes.
22
+ // The maximum is 8 and the default is 6.
23
+ HashLen int
24
+
25
+ // TableBits is the number of bits in the hash table indexes.
26
+ // The default is 17 (128K entries).
27
+ TableBits int
28
+
29
+ // ChainLength is how many entries to search on the "match chain" of older
30
+ // locations with the same hash as the current location.
31
+ ChainLength int
32
+
33
+ table []uint32
34
+ chain []uint32
35
+
36
+ history []byte
37
+
38
+ // holding onto buffers to reduce allocations:
39
+
40
+ arrivals []arrival
41
+ foundMatches []absoluteMatch
42
+ matches []Match
43
+ }
44
+
45
+ func (q *Pathfinder) Reset() {
46
+ for i := range q.table {
47
+ q.table[i] = 0
48
+ }
49
+ q.history = q.history[:0]
50
+ q.chain = q.chain[:0]
51
+ }
52
+
53
+ // An arrival represents how we got to a certain byte position.
54
+ // The cost is the total cost to get there from the beginning of the block.
55
+ // If distance > 0, the arrival is with a match.
56
+ // If distance == 0, the arrival is with a run of literals.
57
+ type arrival struct {
58
+ length uint32
59
+ distance uint32
60
+ cost float32
61
+ }
62
+
63
+ const (
64
+ baseMatchCost float32 = 4
65
+ )
66
+
67
+ func (q *Pathfinder) FindMatches(dst []Match, src []byte) []Match {
68
+ if q.MaxDistance == 0 {
69
+ q.MaxDistance = 65535
70
+ }
71
+ if q.MinLength == 0 {
72
+ q.MinLength = 4
73
+ }
74
+ if q.HashLen == 0 {
75
+ q.HashLen = 6
76
+ }
77
+ if q.TableBits == 0 {
78
+ q.TableBits = 17
79
+ }
80
+ if len(q.table) < 1<<q.TableBits {
81
+ q.table = make([]uint32, 1<<q.TableBits)
82
+ }
83
+
84
+ var histogram [256]uint32
85
+ for _, b := range src {
86
+ histogram[b]++
87
+ }
88
+ var byteCost [256]float32
89
+ for b, n := range histogram {
90
+ cost := max(math.Log2(float64(len(src))/float64(n)), 1)
91
+ byteCost[b] = float32(cost)
92
+ }
93
+
94
+ // Each element in arrivals corresponds to the position just after
95
+ // the corresponding byte in src.
96
+ arrivals := q.arrivals
97
+ if len(arrivals) < len(src) {
98
+ arrivals = make([]arrival, len(src))
99
+ q.arrivals = arrivals
100
+ } else {
101
+ arrivals = arrivals[:len(src)]
102
+ for i := range arrivals {
103
+ arrivals[i] = arrival{}
104
+ }
105
+ }
106
+
107
+ if len(q.history) > q.MaxDistance*2 {
108
+ // Trim down the history buffer.
109
+ delta := len(q.history) - q.MaxDistance
110
+ copy(q.history, q.history[delta:])
111
+ q.history = q.history[:q.MaxDistance]
112
+ q.chain = q.chain[:q.MaxDistance]
113
+
114
+ for i, v := range q.table {
115
+ newV := max(int(v)-delta, 0)
116
+ q.table[i] = uint32(newV)
117
+ }
118
+ }
119
+
120
+ // Append src to the history buffer.
121
+ historyLen := len(q.history)
122
+ q.history = append(q.history, src...)
123
+ q.chain = append(q.chain, make([]uint32, len(src))...)
124
+ src = q.history
125
+
126
+ // Calculate hashes and build the chain.
127
+ for i := historyLen; i < len(src)-7; i++ {
128
+ h := ((binary.LittleEndian.Uint64(src[i:]) & (1<<(8*q.HashLen) - 1)) * hashMul64) >> (64 - q.TableBits)
129
+ candidate := int(q.table[h])
130
+ q.table[h] = uint32(i)
131
+ if candidate != 0 {
132
+ delta := i - candidate
133
+ q.chain[i] = uint32(delta)
134
+ }
135
+ }
136
+
137
+ // Look for matches, and collect them in foundMatches. Later we'll figure out
138
+ // which ones to actually use.
139
+ foundMatches := q.foundMatches[:0]
140
+ var prevMatch absoluteMatch
141
+ i := historyLen
142
+ for i < len(src)-7 {
143
+ delta := q.chain[i]
144
+ if delta == 0 {
145
+ i++
146
+ continue
147
+ }
148
+ candidate := i - int(delta)
149
+ if candidate <= 0 || i-candidate > q.MaxDistance {
150
+ i++
151
+ continue
152
+ }
153
+
154
+ var currentMatch absoluteMatch
155
+
156
+ if i >= prevMatch.End && prevMatch != (absoluteMatch{}) {
157
+ // Look for a repeat match at i+1.
158
+ prevDistance := prevMatch.Start - prevMatch.Match
159
+ if binary.LittleEndian.Uint32(src[i+1:]) == binary.LittleEndian.Uint32(src[i+1-prevDistance:]) {
160
+ m := extendMatch2(src, i+1, i+1-prevDistance, i+1)
161
+ if m.End-m.Start > q.MinLength {
162
+ currentMatch = m
163
+ foundMatches = append(foundMatches, m)
164
+ }
165
+ }
166
+ }
167
+
168
+ if binary.LittleEndian.Uint32(src[candidate:]) == binary.LittleEndian.Uint32(src[i:]) {
169
+ m := extendMatch2(src, i, candidate, max(historyLen, prevMatch.Start))
170
+ if m.End-m.Start > q.MinLength {
171
+ currentMatch = m
172
+ foundMatches = append(foundMatches, m)
173
+ }
174
+ }
175
+
176
+ for range q.ChainLength {
177
+ delta := q.chain[candidate]
178
+ if delta == 0 {
179
+ break
180
+ }
181
+ candidate -= int(delta)
182
+ if candidate <= 0 || i-candidate > q.MaxDistance {
183
+ break
184
+ }
185
+ if binary.LittleEndian.Uint32(src[candidate:]) == binary.LittleEndian.Uint32(src[i:]) {
186
+ m := extendMatch2(src, i, candidate, max(historyLen, prevMatch.Start))
187
+ if m.End-m.Start > q.MinLength && m.End-m.Start > currentMatch.End-currentMatch.Start {
188
+ currentMatch = m
189
+ foundMatches = append(foundMatches, m)
190
+ }
191
+ }
192
+ }
193
+
194
+ if i < prevMatch.End && currentMatch.End-currentMatch.Start <= prevMatch.End-prevMatch.Start {
195
+ // We were looking for an overlapping match, but we didn't find one longer
196
+ // than the previous match. So we'll go back to sequential search,
197
+ // starting right after the previous match.
198
+ i = prevMatch.End
199
+ continue
200
+ }
201
+
202
+ if currentMatch == (absoluteMatch{}) {
203
+ // No match found. Continue with sequential search.
204
+ i++
205
+ continue
206
+ }
207
+
208
+ // We've found a match; now look for matches overlapping the end of it.
209
+ prevMatch = currentMatch
210
+ i = currentMatch.End + 2 - q.HashLen
211
+ }
212
+
213
+ q.foundMatches = foundMatches
214
+
215
+ slices.SortFunc(foundMatches, func(a, b absoluteMatch) int { return a.Start - b.Start })
216
+ matchIndex := 0
217
+ var pending absoluteMatch
218
+
219
+ for i := historyLen; i < len(src); i++ {
220
+ var arrivedHere arrival
221
+ if i > historyLen {
222
+ arrivedHere = arrivals[i-historyLen-1]
223
+ }
224
+
225
+ unmatched := 0
226
+ if arrivedHere.distance == 0 {
227
+ unmatched = int(arrivedHere.length)
228
+ }
229
+ prevDistance := 0
230
+ if i-unmatched > historyLen {
231
+ prevDistance = int(arrivals[i-historyLen-1-unmatched].distance)
232
+ }
233
+
234
+ literalCost := byteCost[src[i]]
235
+ nextArrival := &arrivals[i-historyLen]
236
+ if nextArrival.cost == 0 || arrivedHere.cost+literalCost < nextArrival.cost {
237
+ *nextArrival = arrival{
238
+ cost: arrivedHere.cost + literalCost,
239
+ length: uint32(unmatched + 1),
240
+ }
241
+ }
242
+
243
+ for matchIndex < len(foundMatches) && foundMatches[matchIndex].Start == i {
244
+ m := foundMatches[matchIndex]
245
+ matchIndex++
246
+ if m.End > pending.End {
247
+ pending = m
248
+ }
249
+ matchCost := baseMatchCost + float32(bits.Len(uint(unmatched)))
250
+ if m.Start-m.Match != prevDistance {
251
+ matchCost += float32(bits.Len(uint(m.Start - m.Match)))
252
+ }
253
+ for j := m.Start + q.MinLength; j <= m.End; j++ {
254
+ adjustedCost := matchCost
255
+ if j-m.Start < 6 {
256
+ // Matches shorter than 6 are comparatively rare, and therefore
257
+ // have longer codes.
258
+ adjustedCost += float32(6-(j-m.Start)) * 2
259
+ }
260
+ a := &arrivals[j-historyLen-1]
261
+ if a.cost == 0 || arrivedHere.cost+adjustedCost < a.cost {
262
+ *a = arrival{
263
+ length: uint32(j - m.Start),
264
+ distance: uint32(m.Start - m.Match),
265
+ cost: arrivedHere.cost + adjustedCost,
266
+ }
267
+ }
268
+ }
269
+ }
270
+
271
+ // If a match from an earlier position extends far enough past the current
272
+ // position, try using the tail of it, starting from here.
273
+ if unmatched == 0 && pending.Start != i && pending.End >= i+q.MinLength &&
274
+ !(arrivedHere.length != 0 && arrivedHere.distance == uint32(pending.Start-pending.Match)) {
275
+ matchCost := baseMatchCost + float32(bits.Len(uint(pending.Start-pending.Match)))
276
+ for j := i + q.MinLength; j <= pending.End; j++ {
277
+ adjustedCost := matchCost
278
+ if j-i < 6 {
279
+ // Matches shorter than 6 are comparatively rare, and therefore
280
+ // have longer codes.
281
+ adjustedCost += float32(6-(j-i)) * 2
282
+ }
283
+ a := &arrivals[j-historyLen-1]
284
+ if a.cost == 0 || arrivedHere.cost+adjustedCost < a.cost {
285
+ *a = arrival{
286
+ length: uint32(j - i),
287
+ distance: uint32(pending.Start - pending.Match),
288
+ cost: arrivedHere.cost + adjustedCost,
289
+ }
290
+ }
291
+ }
292
+ }
293
+
294
+ delta := q.chain[i]
295
+ if delta == 0 {
296
+ continue
297
+ }
298
+ candidate := i - int(delta)
299
+ if candidate <= 0 || i-candidate > q.MaxDistance {
300
+ continue
301
+ }
302
+ }
303
+
304
+ // We've found the shortest path; now walk it backward and store the matches.
305
+ matches := q.matches[:0]
306
+ i = len(arrivals) - 1
307
+ for i >= 0 {
308
+ a := arrivals[i]
309
+ if a.distance > 0 {
310
+ matches = append(matches, Match{
311
+ Length: int(a.length),
312
+ Distance: int(a.distance),
313
+ })
314
+ i -= int(a.length)
315
+ } else {
316
+ if len(matches) == 0 {
317
+ matches = append(matches, Match{})
318
+ }
319
+ matches[len(matches)-1].Unmatched = int(a.length)
320
+ i -= int(a.length)
321
+ }
322
+ }
323
+ q.matches = matches
324
+
325
+ slices.Reverse(matches)
326
+
327
+ return append(dst, matches...)
328
+ }
@@ -0,0 +1,53 @@
1
+ package matchfinder
2
+
3
+ import "fmt"
4
+
5
+ // A TextEncoder is an Encoder that produces a human-readable representation of
6
+ // the LZ77 compression. Matches are replaced with <Length,Distance> symbols.
7
+ type TextEncoder struct{}
8
+
9
+ func (t TextEncoder) Reset() {}
10
+
11
+ func (t TextEncoder) Encode(dst []byte, src []byte, matches []Match, lastBlock bool) []byte {
12
+ pos := 0
13
+ for _, m := range matches {
14
+ if m.Unmatched > 0 {
15
+ dst = append(dst, src[pos:pos+m.Unmatched]...)
16
+ pos += m.Unmatched
17
+ }
18
+ if m.Length > 0 {
19
+ dst = append(dst, []byte(fmt.Sprintf("<%d,%d>", m.Length, m.Distance))...)
20
+ pos += m.Length
21
+ }
22
+ }
23
+ if pos < len(src) {
24
+ dst = append(dst, src[pos:]...)
25
+ }
26
+ return dst
27
+ }
28
+
29
+ // A NoMatchFinder implements MatchFinder, but doesn't find any matches.
30
+ // It can be used to implement the equivalent of the standard library flate package's
31
+ // HuffmanOnly setting.
32
+ type NoMatchFinder struct{}
33
+
34
+ func (n NoMatchFinder) Reset() {}
35
+
36
+ func (n NoMatchFinder) FindMatches(dst []Match, src []byte) []Match {
37
+ return append(dst, Match{
38
+ Unmatched: len(src),
39
+ })
40
+ }
41
+
42
+ // AutoReset wraps a MatchFinder that can return references to data in previous
43
+ // blocks, and calls Reset before each block. It is useful for (e.g.) using a
44
+ // snappy Encoder with a MatchFinder designed for flate. (Snappy doesn't
45
+ // support references between blocks.)
46
+ type AutoReset struct {
47
+ MatchFinder
48
+ }
49
+
50
+ func (a AutoReset) FindMatches(dst []Match, src []byte) []Match {
51
+ a.Reset()
52
+ return a.MatchFinder.FindMatches(dst, src)
53
+ }
@@ -0,0 +1,327 @@
1
+ package matchfinder
2
+
3
+ import "encoding/binary"
4
+
5
+ // Trio is a MatchFinder that uses 3 different hash lengths, and
6
+ // overlap parsing.
7
+ type Trio struct {
8
+ MaxDistance int
9
+ history []byte
10
+ table5 [1 << 16]tableEntry
11
+ table8 [1 << 17]tableEntry
12
+ table12 [1 << 18]tableEntry
13
+ }
14
+
15
+ func (z *Trio) Reset() {
16
+ z.table5 = [len(z.table5)]tableEntry{}
17
+ z.table8 = [len(z.table8)]tableEntry{}
18
+ z.table12 = [len(z.table12)]tableEntry{}
19
+ z.history = z.history[:0]
20
+ }
21
+
22
+ func (z *Trio) FindMatches(dst []Match, src []byte) []Match {
23
+ if z.MaxDistance == 0 {
24
+ z.MaxDistance = 1 << 16
25
+ }
26
+
27
+ if len(z.history) > z.MaxDistance*2 {
28
+ delta := len(z.history) - z.MaxDistance
29
+ copy(z.history, z.history[delta:])
30
+ z.history = z.history[:z.MaxDistance]
31
+
32
+ for i := range z.table5 {
33
+ v := z.table5[i].offset
34
+ v -= int32(delta)
35
+ if v < 0 {
36
+ z.table5[i] = tableEntry{}
37
+ } else {
38
+ z.table5[i].offset = v
39
+ }
40
+ }
41
+ for i := range z.table8 {
42
+ v := z.table8[i].offset
43
+ v -= int32(delta)
44
+ if v < 0 {
45
+ z.table8[i] = tableEntry{}
46
+ } else {
47
+ z.table8[i].offset = v
48
+ }
49
+ }
50
+ for i := range z.table12 {
51
+ v := z.table12[i].offset
52
+ v -= int32(delta)
53
+ if v < 0 {
54
+ z.table12[i] = tableEntry{}
55
+ } else {
56
+ z.table12[i].offset = v
57
+ }
58
+ }
59
+ }
60
+
61
+ e := matchEmitter{
62
+ Dst: dst,
63
+ NextEmit: len(z.history),
64
+ }
65
+ z.history = append(z.history, src...)
66
+
67
+ if len(src) < 20 {
68
+ return append(dst, Match{
69
+ Unmatched: len(src),
70
+ })
71
+ }
72
+
73
+ src = z.history
74
+
75
+ // matches stores the matches that have been found but not emitted,
76
+ // in reverse order. (matches[0] is the most recent one.)
77
+ var matches [3]absoluteMatch
78
+
79
+ sLimit := int32(len(src)) - 14
80
+
81
+ mainLoop:
82
+ for {
83
+ // Search for a match, starting after the last match emitted.
84
+ s := int32(e.NextEmit)
85
+ if s > sLimit {
86
+ break mainLoop
87
+ }
88
+
89
+ // t will contain the match offset when we find one.
90
+ var t int32
91
+ var hashLengthFound int
92
+
93
+ cv := binary.LittleEndian.Uint64(src[s:])
94
+ extra := binary.LittleEndian.Uint32(src[s+8:])
95
+
96
+ for {
97
+ nextHash12 := z.hash12(cv, extra)
98
+ nextHash8 := z.hash8(cv)
99
+ nextHash5 := z.hash5(cv)
100
+ candidate12 := z.table12[nextHash12]
101
+ candidate8 := z.table8[nextHash8]
102
+ candidate5 := z.table5[nextHash5]
103
+
104
+ entry := tableEntry{offset: s, val: uint32(cv)}
105
+ z.table12[nextHash12] = entry
106
+ z.table8[nextHash8] = entry
107
+ z.table5[nextHash5] = entry
108
+
109
+ // Look for a repeat match one byte after the current position.
110
+ if len(e.Dst) > 0 {
111
+ prevDistance := int32(e.Dst[len(e.Dst)-1].Distance)
112
+ if prevDistance != 0 {
113
+ repIndex := s - prevDistance + 1
114
+ if repIndex >= 0 && binary.LittleEndian.Uint32(src[repIndex:]) == uint32(cv>>8) {
115
+ // There is a repeated match at s+2.
116
+ s++
117
+ t = repIndex
118
+ break
119
+ }
120
+ }
121
+ }
122
+
123
+ if candidate12.offset < s && s-candidate12.offset < int32(z.MaxDistance) && uint32(cv) == candidate12.val &&
124
+ binary.LittleEndian.Uint32(src[candidate12.offset:]) == uint32(cv) {
125
+ // There is a 12-byte match at s.
126
+ t = candidate12.offset
127
+ hashLengthFound = 12
128
+ break
129
+ }
130
+ if candidate8.offset < s && s-candidate8.offset < int32(z.MaxDistance) && uint32(cv) == candidate8.val &&
131
+ binary.LittleEndian.Uint32(src[candidate8.offset:]) == uint32(cv) {
132
+ t = candidate8.offset
133
+ hashLengthFound = 8
134
+ break
135
+ }
136
+ if candidate5.offset < s && s-candidate5.offset < int32(z.MaxDistance) && uint32(cv) == candidate5.val &&
137
+ binary.LittleEndian.Uint32(src[candidate5.offset:]) == uint32(cv) {
138
+ t = candidate5.offset
139
+ hashLengthFound = 5
140
+ break
141
+ }
142
+
143
+ s += 1 + ((s - int32(e.NextEmit)) >> 7)
144
+ if s > sLimit {
145
+ break mainLoop
146
+ }
147
+ cv = binary.LittleEndian.Uint64(src[s:])
148
+ extra = binary.LittleEndian.Uint32(src[s+8:])
149
+ }
150
+
151
+ if hashLengthFound != 0 && hashLengthFound < 12 {
152
+ // Look for a "lazy" match with a longer hash at s+1.
153
+ cv := binary.LittleEndian.Uint64(src[s+1:])
154
+ extra := binary.LittleEndian.Uint32(src[s+9:])
155
+ nextHash12 := z.hash12(cv, extra)
156
+ nextHash8 := z.hash8(cv)
157
+ candidate12 := z.table12[nextHash12]
158
+ candidate8 := z.table8[nextHash8]
159
+ coffset12 := s - candidate12.offset + 1
160
+ coffset8 := s - candidate8.offset + 1
161
+ entry := tableEntry{offset: s + 1, val: uint32(cv)}
162
+ z.table12[nextHash12] = entry
163
+ z.table8[nextHash8] = entry
164
+ if candidate12.offset < s+1 && coffset12 < int32(z.MaxDistance) && uint32(cv) == candidate12.val &&
165
+ binary.LittleEndian.Uint32(src[candidate12.offset:]) == uint32(cv) {
166
+ t = candidate12.offset
167
+ s++
168
+ } else if hashLengthFound < 8 && candidate8.offset < s+1 && coffset8 < int32(z.MaxDistance) && uint32(cv) == candidate8.val &&
169
+ binary.LittleEndian.Uint32(src[candidate8.offset:]) == uint32(cv) {
170
+ t = candidate8.offset
171
+ s++
172
+ }
173
+ }
174
+
175
+ currentMatch := extendMatch2(src, int(s), int(t), e.NextEmit)
176
+ matches[0] = currentMatch
177
+
178
+ index0 := s + 1
179
+
180
+ // We have a match in matches[0].
181
+ // Now look for overlapping matches.
182
+
183
+ for {
184
+ if matches[0].End > int(sLimit) {
185
+ break
186
+ }
187
+ s = int32(max(matches[0].Start+2, matches[0].End-10))
188
+
189
+ // Store some entries that haven't been indexed yet.
190
+ for index0 < s-1 {
191
+ cv0 := binary.LittleEndian.Uint64(src[index0:])
192
+ extra0 := binary.LittleEndian.Uint32(src[index0+8:])
193
+ te0 := tableEntry{offset: index0, val: uint32(cv0)}
194
+ z.table5[z.hash5(cv0)] = te0
195
+ z.table8[z.hash8(cv0)] = te0
196
+ z.table12[z.hash12(cv0, extra0)] = te0
197
+ index0++
198
+ }
199
+
200
+ cv = binary.LittleEndian.Uint64(src[s:])
201
+ extra = binary.LittleEndian.Uint32(src[s+8:])
202
+
203
+ nextHash12 := z.hash12(cv, extra)
204
+ nextHash8 := z.hash8(cv)
205
+ nextHash5 := z.hash5(cv)
206
+ candidate12 := z.table12[nextHash12]
207
+ candidate8 := z.table8[nextHash8]
208
+
209
+ entry := tableEntry{offset: s, val: uint32(cv)}
210
+ z.table12[nextHash12] = entry
211
+ z.table8[nextHash8] = entry
212
+ z.table5[nextHash5] = entry
213
+
214
+ t = -1
215
+ if candidate12.offset < s && s-candidate12.offset < int32(z.MaxDistance) && uint32(cv) == candidate12.val &&
216
+ binary.LittleEndian.Uint32(src[candidate12.offset:]) == uint32(cv) {
217
+ // There is a 12-byte match at s.
218
+ t = candidate12.offset
219
+ } else if candidate8.offset < s && s-candidate8.offset < int32(z.MaxDistance) && uint32(cv) == candidate8.val &&
220
+ binary.LittleEndian.Uint32(src[candidate8.offset:]) == uint32(cv) {
221
+ // There is a long match at s.
222
+ t = candidate8.offset
223
+ }
224
+
225
+ index0 = s + 1
226
+
227
+ if t == -1 {
228
+ // No overlapping match was found.
229
+ break
230
+ }
231
+
232
+ newMatch := extendMatch2(src, int(s), int(t), e.NextEmit)
233
+
234
+ if newMatch.End-newMatch.Start <= matches[0].End-matches[0].Start {
235
+ // The new match isn't longer than the old one, so we break out of the loop
236
+ // of looking for overlapping matches.
237
+ break
238
+ }
239
+
240
+ matches = [3]absoluteMatch{
241
+ newMatch,
242
+ matches[0],
243
+ matches[1],
244
+ }
245
+
246
+ if matches[2] == (absoluteMatch{}) {
247
+ continue
248
+ }
249
+
250
+ // We have three matches, so it's time to emit one and/or eliminate one.
251
+ switch {
252
+ case matches[0].Start < matches[2].End:
253
+ // The first and third matches overlap; discard the one in between.
254
+ matches = [3]absoluteMatch{
255
+ matches[0],
256
+ matches[2],
257
+ {},
258
+ }
259
+
260
+ case matches[0].Start < matches[2].End+4:
261
+ // The first and third matches don't overlap, but there's no room for
262
+ // another match between them. Emit the first match and discard the second.
263
+ e.emit(matches[2])
264
+ matches = [3]absoluteMatch{
265
+ matches[0],
266
+ {},
267
+ {},
268
+ }
269
+
270
+ default:
271
+ // Emit the first match, shortening it if necessary to avoid overlap with the second.
272
+ if matches[2].End > matches[1].Start {
273
+ matches[2].End = matches[1].Start
274
+ }
275
+ if matches[2].End-matches[2].Start >= 4 {
276
+ e.emit(matches[2])
277
+ }
278
+ matches[2] = absoluteMatch{}
279
+ }
280
+ }
281
+
282
+ // Store some entries up to the end of the last match.
283
+ for index0 < int32(matches[0].End) && index0 < sLimit {
284
+ cv0 := binary.LittleEndian.Uint64(src[index0:])
285
+ extra0 := binary.LittleEndian.Uint32(src[index0+8:])
286
+ te0 := tableEntry{offset: index0, val: uint32(cv0)}
287
+ z.table5[z.hash5(cv0)] = te0
288
+ z.table8[z.hash8(cv0)] = te0
289
+ z.table12[z.hash12(cv0, extra0)] = te0
290
+ index0++
291
+ }
292
+
293
+ // We're done looking for overlapping matches; emit the ones we have.
294
+
295
+ if matches[1] != (absoluteMatch{}) {
296
+ if matches[1].End > matches[0].Start {
297
+ matches[1].End = matches[0].Start
298
+ }
299
+ if matches[1].End-matches[1].Start >= 4 {
300
+ e.emit(matches[1])
301
+ }
302
+ }
303
+ e.emit(matches[0])
304
+ matches = [3]absoluteMatch{}
305
+ }
306
+
307
+ dst = e.Dst
308
+ if e.NextEmit < len(src) {
309
+ dst = append(dst, Match{
310
+ Unmatched: len(src) - e.NextEmit,
311
+ })
312
+ }
313
+
314
+ return dst
315
+ }
316
+
317
+ func (z *Trio) hash5(u uint64) uint32 {
318
+ return uint32(((u << 24) * 889523592379) >> (64 - 16))
319
+ }
320
+
321
+ func (z *Trio) hash8(u uint64) uint32 {
322
+ return uint32((u * 0xcf1bbcdcb7a56463) >> (64 - 17))
323
+ }
324
+
325
+ func (z *Trio) hash12(u uint64, e uint32) uint32 {
326
+ return uint32((u*0xcf1bbcdcb7a56463 + uint64(e)*(2654435761<<32)) >> (64 - 18))
327
+ }