typenative 0.3.1 → 0.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (170) hide show
  1. package/.github/workflows/ci.yml +2 -0
  2. package/CHANGELOG.md +258 -0
  3. package/CONTRIBUTION.md +1 -1
  4. package/README.md +17 -1
  5. package/bin/cli.js +26 -11
  6. package/bin/go/bridge.js +74 -14
  7. package/bin/go/convert.js +59 -1
  8. package/bin/go/metadata.js +13 -4
  9. package/bin/go/overrides.js +4 -6
  10. package/bin/go/stdlib.generated.js +2 -0
  11. package/bin/index.js +338 -47
  12. package/bin/main.js +36 -14
  13. package/bin/node/assert.js +2 -2
  14. package/bin/node/buffer.js +9 -0
  15. package/bin/node/child_process.js +51 -3
  16. package/bin/node/cluster.js +41 -3
  17. package/bin/node/core.js +1 -1
  18. package/bin/node/crypto.js +205 -4
  19. package/bin/node/dgram.js +230 -6
  20. package/bin/node/dns.js +471 -8
  21. package/bin/node/errno_tables.js +445 -0
  22. package/bin/node/events.js +1 -1
  23. package/bin/node/fs.js +22 -15
  24. package/bin/node/fs_watch.js +407 -0
  25. package/bin/node/http.js +128 -30
  26. package/bin/node/http2.js +37 -7
  27. package/bin/node/module_v8.js +142 -0
  28. package/bin/node/net.js +9 -6
  29. package/bin/node/process.js +242 -0
  30. package/bin/node/readline.js +272 -1
  31. package/bin/node/registry.js +3 -1
  32. package/bin/node/sqlite.js +60 -12
  33. package/bin/node/stream.js +145 -10
  34. package/bin/node/test.js +311 -19
  35. package/bin/node/timers.js +1 -1
  36. package/bin/node/typed_arrays.js +608 -0
  37. package/bin/node/url.js +23 -5
  38. package/bin/node/util.js +21 -1
  39. package/bin/node/util_extra.js +6 -30
  40. package/bin/node/worker_threads.js +79 -19
  41. package/bin/node/zlib.js +131 -16
  42. package/bin/transpiler.js +4729 -616
  43. package/bin/typecheck-node.js +1 -0
  44. package/bin/version.js +1 -1
  45. package/conformance.md +1030 -877
  46. package/examples/README.md +54 -0
  47. package/examples/fibonacci/README.md +89 -0
  48. package/examples/fibonacci/bench.mjs +87 -0
  49. package/examples/fibonacci/main.ts +21 -0
  50. package/examples/hello-world/README.md +62 -0
  51. package/examples/hello-world/bench.mjs +66 -0
  52. package/examples/hello-world/main.ts +7 -0
  53. package/examples/json-server/README.md +127 -0
  54. package/examples/json-server/bench.mjs +263 -0
  55. package/examples/json-server/main.ts +67 -0
  56. package/govendor/github.com/andybalholm/brotli/LICENSE +19 -0
  57. package/govendor/github.com/andybalholm/brotli/README.md +19 -0
  58. package/govendor/github.com/andybalholm/brotli/backward_references.go +185 -0
  59. package/govendor/github.com/andybalholm/brotli/backward_references_hq.go +796 -0
  60. package/govendor/github.com/andybalholm/brotli/bit_cost.go +436 -0
  61. package/govendor/github.com/andybalholm/brotli/bit_reader.go +266 -0
  62. package/govendor/github.com/andybalholm/brotli/bitwriter.go +56 -0
  63. package/govendor/github.com/andybalholm/brotli/block_splitter.go +144 -0
  64. package/govendor/github.com/andybalholm/brotli/block_splitter_command.go +434 -0
  65. package/govendor/github.com/andybalholm/brotli/block_splitter_distance.go +433 -0
  66. package/govendor/github.com/andybalholm/brotli/block_splitter_literal.go +433 -0
  67. package/govendor/github.com/andybalholm/brotli/brotli_bit_stream.go +1435 -0
  68. package/govendor/github.com/andybalholm/brotli/cluster.go +30 -0
  69. package/govendor/github.com/andybalholm/brotli/cluster_command.go +164 -0
  70. package/govendor/github.com/andybalholm/brotli/cluster_distance.go +326 -0
  71. package/govendor/github.com/andybalholm/brotli/cluster_literal.go +326 -0
  72. package/govendor/github.com/andybalholm/brotli/command.go +254 -0
  73. package/govendor/github.com/andybalholm/brotli/compress_fragment.go +834 -0
  74. package/govendor/github.com/andybalholm/brotli/compress_fragment_two_pass.go +775 -0
  75. package/govendor/github.com/andybalholm/brotli/constants.go +77 -0
  76. package/govendor/github.com/andybalholm/brotli/context.go +2176 -0
  77. package/govendor/github.com/andybalholm/brotli/decode.go +2570 -0
  78. package/govendor/github.com/andybalholm/brotli/dictionary.go +122890 -0
  79. package/govendor/github.com/andybalholm/brotli/dictionary_hash.go +32779 -0
  80. package/govendor/github.com/andybalholm/brotli/encode.go +1220 -0
  81. package/govendor/github.com/andybalholm/brotli/encoder.go +177 -0
  82. package/govendor/github.com/andybalholm/brotli/encoder_dict.go +22 -0
  83. package/govendor/github.com/andybalholm/brotli/encoder_fast.go +201 -0
  84. package/govendor/github.com/andybalholm/brotli/entropy_encode.go +592 -0
  85. package/govendor/github.com/andybalholm/brotli/entropy_encode_static.go +4399 -0
  86. package/govendor/github.com/andybalholm/brotli/fast_log.go +290 -0
  87. package/govendor/github.com/andybalholm/brotli/find_match_length.go +45 -0
  88. package/govendor/github.com/andybalholm/brotli/go.mod +3 -0
  89. package/govendor/github.com/andybalholm/brotli/h10.go +287 -0
  90. package/govendor/github.com/andybalholm/brotli/h5.go +214 -0
  91. package/govendor/github.com/andybalholm/brotli/h6.go +216 -0
  92. package/govendor/github.com/andybalholm/brotli/hash.go +342 -0
  93. package/govendor/github.com/andybalholm/brotli/hash_composite.go +93 -0
  94. package/govendor/github.com/andybalholm/brotli/hash_forgetful_chain.go +252 -0
  95. package/govendor/github.com/andybalholm/brotli/hash_longest_match_quickly.go +214 -0
  96. package/govendor/github.com/andybalholm/brotli/hash_rolling.go +168 -0
  97. package/govendor/github.com/andybalholm/brotli/histogram.go +226 -0
  98. package/govendor/github.com/andybalholm/brotli/http.go +201 -0
  99. package/govendor/github.com/andybalholm/brotli/huffman.go +653 -0
  100. package/govendor/github.com/andybalholm/brotli/literal_cost.go +182 -0
  101. package/govendor/github.com/andybalholm/brotli/matchfinder/bargain1.go +212 -0
  102. package/govendor/github.com/andybalholm/brotli/matchfinder/bargain2.go +248 -0
  103. package/govendor/github.com/andybalholm/brotli/matchfinder/bargain3.go +278 -0
  104. package/govendor/github.com/andybalholm/brotli/matchfinder/emitter.go +34 -0
  105. package/govendor/github.com/andybalholm/brotli/matchfinder/m0.go +175 -0
  106. package/govendor/github.com/andybalholm/brotli/matchfinder/m4.go +339 -0
  107. package/govendor/github.com/andybalholm/brotli/matchfinder/matchfinder.go +103 -0
  108. package/govendor/github.com/andybalholm/brotli/matchfinder/pathfinder.go +328 -0
  109. package/govendor/github.com/andybalholm/brotli/matchfinder/textencoder.go +53 -0
  110. package/govendor/github.com/andybalholm/brotli/matchfinder/trio.go +327 -0
  111. package/govendor/github.com/andybalholm/brotli/matchfinder/zdfast.go +258 -0
  112. package/govendor/github.com/andybalholm/brotli/matchfinder/zfast.go +206 -0
  113. package/govendor/github.com/andybalholm/brotli/matchfinder/zm.go +302 -0
  114. package/govendor/github.com/andybalholm/brotli/memory.go +66 -0
  115. package/govendor/github.com/andybalholm/brotli/metablock.go +574 -0
  116. package/govendor/github.com/andybalholm/brotli/metablock_command.go +165 -0
  117. package/govendor/github.com/andybalholm/brotli/metablock_distance.go +165 -0
  118. package/govendor/github.com/andybalholm/brotli/metablock_literal.go +165 -0
  119. package/govendor/github.com/andybalholm/brotli/params.go +37 -0
  120. package/govendor/github.com/andybalholm/brotli/platform.go +103 -0
  121. package/govendor/github.com/andybalholm/brotli/prefix.go +30 -0
  122. package/govendor/github.com/andybalholm/brotli/prefix_dec.go +723 -0
  123. package/govendor/github.com/andybalholm/brotli/quality.go +196 -0
  124. package/govendor/github.com/andybalholm/brotli/reader.go +127 -0
  125. package/govendor/github.com/andybalholm/brotli/ringbuffer.go +134 -0
  126. package/govendor/github.com/andybalholm/brotli/state.go +313 -0
  127. package/govendor/github.com/andybalholm/brotli/static_dict.go +662 -0
  128. package/govendor/github.com/andybalholm/brotli/static_dict_lut.go +75094 -0
  129. package/govendor/github.com/andybalholm/brotli/symbol_list.go +22 -0
  130. package/govendor/github.com/andybalholm/brotli/transform.go +641 -0
  131. package/govendor/github.com/andybalholm/brotli/utf8_util.go +70 -0
  132. package/govendor/github.com/andybalholm/brotli/util.go +7 -0
  133. package/govendor/github.com/andybalholm/brotli/write_bits.go +52 -0
  134. package/govendor/github.com/andybalholm/brotli/writer.go +162 -0
  135. package/govendor/github.com/fsnotify/fsnotify/.mailmap +2 -0
  136. package/govendor/github.com/fsnotify/fsnotify/CHANGELOG.md +651 -0
  137. package/govendor/github.com/fsnotify/fsnotify/CONTRIBUTING.md +147 -0
  138. package/govendor/github.com/fsnotify/fsnotify/LICENSE +25 -0
  139. package/govendor/github.com/fsnotify/fsnotify/README.md +214 -0
  140. package/govendor/github.com/fsnotify/fsnotify/backend_fen.go +469 -0
  141. package/govendor/github.com/fsnotify/fsnotify/backend_inotify.go +597 -0
  142. package/govendor/github.com/fsnotify/fsnotify/backend_kqueue.go +735 -0
  143. package/govendor/github.com/fsnotify/fsnotify/backend_other.go +22 -0
  144. package/govendor/github.com/fsnotify/fsnotify/backend_windows.go +702 -0
  145. package/govendor/github.com/fsnotify/fsnotify/fsnotify.go +526 -0
  146. package/govendor/github.com/fsnotify/fsnotify/go.mod +3 -0
  147. package/govendor/github.com/fsnotify/fsnotify/internal/darwin.go +20 -0
  148. package/govendor/github.com/fsnotify/fsnotify/internal/debug_darwin.go +15 -0
  149. package/govendor/github.com/fsnotify/fsnotify/internal/debug_dragonfly.go +15 -0
  150. package/govendor/github.com/fsnotify/fsnotify/internal/debug_freebsd.go +20 -0
  151. package/govendor/github.com/fsnotify/fsnotify/internal/debug_kqueue.go +32 -0
  152. package/govendor/github.com/fsnotify/fsnotify/internal/debug_linux.go +56 -0
  153. package/govendor/github.com/fsnotify/fsnotify/internal/debug_netbsd.go +15 -0
  154. package/govendor/github.com/fsnotify/fsnotify/internal/debug_openbsd.go +16 -0
  155. package/govendor/github.com/fsnotify/fsnotify/internal/debug_solaris.go +45 -0
  156. package/govendor/github.com/fsnotify/fsnotify/internal/debug_windows.go +40 -0
  157. package/govendor/github.com/fsnotify/fsnotify/internal/freebsd.go +20 -0
  158. package/govendor/github.com/fsnotify/fsnotify/internal/internal.go +2 -0
  159. package/govendor/github.com/fsnotify/fsnotify/internal/unix.go +20 -0
  160. package/govendor/github.com/fsnotify/fsnotify/internal/unix2.go +25 -0
  161. package/govendor/github.com/fsnotify/fsnotify/internal/windows.go +40 -0
  162. package/govendor/github.com/fsnotify/fsnotify/shared.go +64 -0
  163. package/govendor/github.com/fsnotify/fsnotify/staticcheck.conf +3 -0
  164. package/govendor/github.com/fsnotify/fsnotify/system_bsd.go +7 -0
  165. package/govendor/github.com/fsnotify/fsnotify/system_darwin.go +8 -0
  166. package/govendor/manifest.txt +3 -1
  167. package/package.json +1 -1
  168. package/types/typenative-go.d.ts +23 -0
  169. package/types/typenative-npm.d.ts +241 -5
  170. package/types/typenative.d.ts +302 -12
@@ -0,0 +1,278 @@
1
+ package matchfinder
2
+
3
+ import (
4
+ "encoding/binary"
5
+ "math"
6
+ "math/bits"
7
+ "slices"
8
+ )
9
+
10
+ // Bargain3 is a MatchFinder that attempts to find the encoding with the lowest
11
+ // "bit cost", using 3 hash lengths (5, 8, and 12).
12
+ type Bargain3 struct {
13
+ MaxDistance int
14
+
15
+ // Skip is whether to look for matches at every other byte instead of every
16
+ // byte (to increase speed but decrease compression).
17
+ Skip bool
18
+
19
+ history []byte
20
+ table5 [1 << 17]tableEntry
21
+ table8 [1 << 18]tableEntry
22
+ table12 [1 << 19]tableEntry
23
+
24
+ // holding onto buffers to reduce allocations:
25
+
26
+ arrivals []arrival
27
+ matches []Match
28
+ }
29
+
30
+ func (z *Bargain3) Reset() {
31
+ z.table5 = [len(z.table5)]tableEntry{}
32
+ z.table8 = [len(z.table8)]tableEntry{}
33
+ z.table12 = [len(z.table12)]tableEntry{}
34
+ z.history = z.history[:0]
35
+ }
36
+
37
+ func (z *Bargain3) FindMatches(dst []Match, src []byte) []Match {
38
+ if z.MaxDistance == 0 {
39
+ z.MaxDistance = 1 << 16
40
+ }
41
+
42
+ var histogram [256]uint32
43
+ for _, b := range src {
44
+ histogram[b]++
45
+ }
46
+ var byteCost [256]float32
47
+ for b, n := range histogram {
48
+ cost := max(math.Log2(float64(len(src))/float64(n)), 1)
49
+ byteCost[b] = float32(cost)
50
+ }
51
+
52
+ // Each element in arrivals corresponds to the position just after
53
+ // the corresponding byte in src.
54
+ arrivals := z.arrivals
55
+ if len(arrivals) < len(src) {
56
+ arrivals = make([]arrival, len(src))
57
+ z.arrivals = arrivals
58
+ } else {
59
+ arrivals = arrivals[:len(src)]
60
+ for i := range arrivals {
61
+ arrivals[i] = arrival{}
62
+ }
63
+ }
64
+
65
+ if len(z.history) > z.MaxDistance*2 {
66
+ delta := len(z.history) - z.MaxDistance
67
+ copy(z.history, z.history[delta:])
68
+ z.history = z.history[:z.MaxDistance]
69
+
70
+ for i := range z.table5 {
71
+ v := z.table5[i].offset
72
+ v -= int32(delta)
73
+ if v < 0 {
74
+ z.table5[i] = tableEntry{}
75
+ } else {
76
+ z.table5[i].offset = v
77
+ }
78
+ }
79
+ for i := range z.table8 {
80
+ v := z.table8[i].offset
81
+ v -= int32(delta)
82
+ if v < 0 {
83
+ z.table8[i] = tableEntry{}
84
+ } else {
85
+ z.table8[i].offset = v
86
+ }
87
+ }
88
+ for i := range z.table12 {
89
+ v := z.table12[i].offset
90
+ v -= int32(delta)
91
+ if v < 0 {
92
+ z.table12[i] = tableEntry{}
93
+ } else {
94
+ z.table12[i].offset = v
95
+ }
96
+ }
97
+ }
98
+
99
+ historyLen := len(z.history)
100
+ z.history = append(z.history, src...)
101
+ src = z.history
102
+
103
+ addMatch := func(m absoluteMatch, unmatched int, repeat bool) {
104
+ var startCost float32
105
+ if m.Start > historyLen {
106
+ startCost = arrivals[m.Start-historyLen-1].cost
107
+ }
108
+ insertCost := float32(bits.Len(uint(unmatched)))
109
+ var distanceCost float32
110
+ if !repeat {
111
+ distanceCost = float32(bits.Len(uint(m.Start - m.Match)))
112
+ }
113
+ cost := startCost + baseMatchCost + insertCost + distanceCost
114
+ for j := m.End; j >= m.Start+3; j-- {
115
+ a := &arrivals[j-historyLen-1]
116
+ if a.cost > 0 && a.cost <= cost {
117
+ break
118
+ }
119
+ *a = arrival{
120
+ length: uint32(j - m.Start),
121
+ distance: uint32(m.Start - m.Match),
122
+ cost: cost,
123
+ }
124
+ }
125
+ }
126
+
127
+ var nextOverlapSearch int
128
+
129
+ for i := historyLen; i < len(src); i++ {
130
+ var arrivedHere arrival
131
+ if i > historyLen {
132
+ arrivedHere = arrivals[i-historyLen-1]
133
+ }
134
+
135
+ unmatched := 0
136
+ if arrivedHere.distance == 0 {
137
+ unmatched = int(arrivedHere.length)
138
+ }
139
+ prevDistance := 0
140
+ if unmatched != 0 && i-unmatched > historyLen {
141
+ prevDistance = int(arrivals[i-historyLen-1-unmatched].distance)
142
+ }
143
+
144
+ literalCost := byteCost[src[i]]
145
+ nextArrival := &arrivals[i-historyLen]
146
+ if nextArrival.cost == 0 || arrivedHere.cost+literalCost < nextArrival.cost {
147
+ *nextArrival = arrival{
148
+ cost: arrivedHere.cost + literalCost,
149
+ length: uint32(unmatched + 1),
150
+ }
151
+ }
152
+
153
+ if i > len(src)-12 {
154
+ // There's no room to check hashes.
155
+ continue
156
+ }
157
+
158
+ cv := binary.LittleEndian.Uint64(src[i:])
159
+ extra := binary.LittleEndian.Uint32(src[i+8:])
160
+ nextHash5 := z.hash5(cv)
161
+ nextHash8 := z.hash8(cv)
162
+ nextHash12 := z.hash12(cv, extra)
163
+ candidate5 := z.table5[nextHash5]
164
+ candidate8 := z.table8[nextHash8]
165
+ candidate12 := z.table12[nextHash12]
166
+
167
+ entry := tableEntry{offset: int32(i), val: uint32(cv)}
168
+ z.table5[nextHash5] = entry
169
+ z.table8[nextHash8] = entry
170
+ z.table12[nextHash12] = entry
171
+
172
+ // Look for a repeat match, unless there is no previous distance, or a match at
173
+ // that distance has already been found.
174
+ if prevDistance != 0 && prevDistance != int(arrivals[i-historyLen-1+4].distance) {
175
+ repIndex := i - prevDistance
176
+ if repIndex >= 0 && binary.LittleEndian.Uint32(src[repIndex:]) == uint32(cv) {
177
+ // We have a repeat of the previous match distance.
178
+ m := extendMatch2(src, i, repIndex, i)
179
+ addMatch(m, unmatched, true)
180
+ }
181
+ }
182
+
183
+ if z.Skip {
184
+ if i%2 != 0 {
185
+ continue
186
+ }
187
+ }
188
+
189
+ nextByteIsUnmatched := arrivals[i-historyLen-1+1].distance == 0
190
+
191
+ if unmatched > 0 || i >= nextOverlapSearch || nextByteIsUnmatched {
192
+ if int(candidate5.offset) < i && i-int(candidate5.offset) < z.MaxDistance && uint32(cv) == candidate5.val &&
193
+ binary.LittleEndian.Uint32(src[candidate5.offset:]) == uint32(cv) {
194
+ m := extendMatch2(src, i, int(candidate5.offset), historyLen)
195
+ delta := i - m.Start
196
+ if delta == 0 {
197
+ addMatch(m, unmatched, false)
198
+ } else {
199
+ // The match was extended backwards. Add it with and without the extra.
200
+ addMatch(m, max(unmatched-delta, 0), false)
201
+ m.Start += delta
202
+ m.Match += delta
203
+ addMatch(m, unmatched, false)
204
+ }
205
+ nextOverlapSearch = max(nextOverlapSearch, m.Start+1, m.End-6)
206
+ }
207
+
208
+ if int(candidate8.offset) < i && i-int(candidate8.offset) < z.MaxDistance && uint32(cv) == candidate8.val &&
209
+ binary.LittleEndian.Uint32(src[candidate8.offset:]) == uint32(cv) {
210
+ m := extendMatch2(src, i, int(candidate8.offset), historyLen)
211
+ delta := i - m.Start
212
+ if delta == 0 {
213
+ addMatch(m, unmatched, false)
214
+ } else {
215
+ // The match was extended backwards. Add it with and without the extra.
216
+ addMatch(m, max(unmatched-delta, 0), false)
217
+ m.Start += delta
218
+ m.Match += delta
219
+ addMatch(m, unmatched, false)
220
+ }
221
+ nextOverlapSearch = max(nextOverlapSearch, m.Start+1, m.End-6)
222
+ }
223
+
224
+ if int(candidate12.offset) < i && i-int(candidate12.offset) < z.MaxDistance && uint32(cv) == candidate12.val &&
225
+ binary.LittleEndian.Uint32(src[candidate12.offset:]) == uint32(cv) {
226
+ m := extendMatch2(src, i, int(candidate12.offset), historyLen)
227
+ delta := i - m.Start
228
+ if delta == 0 {
229
+ addMatch(m, unmatched, false)
230
+ } else {
231
+ // The match was extended backwards. Add it with and without the extra.
232
+ addMatch(m, max(unmatched-delta, 0), false)
233
+ m.Start += delta
234
+ m.Match += delta
235
+ addMatch(m, unmatched, false)
236
+ }
237
+ nextOverlapSearch = max(nextOverlapSearch, m.Start+1, m.End-6)
238
+ }
239
+ }
240
+ }
241
+
242
+ // We've found the shortest path; now walk it backward and store the matches.
243
+ matches := z.matches[:0]
244
+ i := len(arrivals) - 1
245
+ for i >= 0 {
246
+ a := arrivals[i]
247
+ if a.distance > 0 {
248
+ matches = append(matches, Match{
249
+ Length: int(a.length),
250
+ Distance: int(a.distance),
251
+ })
252
+ i -= int(a.length)
253
+ } else {
254
+ if len(matches) == 0 {
255
+ matches = append(matches, Match{})
256
+ }
257
+ matches[len(matches)-1].Unmatched = int(a.length)
258
+ i -= int(a.length)
259
+ }
260
+ }
261
+ z.matches = matches
262
+
263
+ slices.Reverse(matches)
264
+
265
+ return append(dst, matches...)
266
+ }
267
+
268
+ func (z *Bargain3) hash5(u uint64) uint32 {
269
+ return uint32(((u << 24) * 889523592379) >> (64 - 17))
270
+ }
271
+
272
+ func (z *Bargain3) hash8(u uint64) uint32 {
273
+ return uint32((u * 0xcf1bbcdcb7a56463) >> (64 - 18))
274
+ }
275
+
276
+ func (z *Bargain3) hash12(u uint64, e uint32) uint32 {
277
+ return uint32((u*0xcf1bbcdcb7a56463 + uint64(e)*(2654435761<<32)) >> (64 - 19))
278
+ }
@@ -0,0 +1,34 @@
1
+ package matchfinder
2
+
3
+ // An absoluteMatch is like a Match, but it stores indexes into the byte
4
+ // stream instead of lengths.
5
+ type absoluteMatch struct {
6
+ // Start is the index of the first byte.
7
+ Start int
8
+
9
+ // End is the index of the byte after the last byte
10
+ // (so that End - Start = Length).
11
+ End int
12
+
13
+ // Match is the index of the previous data that matches
14
+ // (Start - Match = Distance).
15
+ Match int
16
+ }
17
+
18
+ // A matchEmitter manages the output of matches for a MatchFinder.
19
+ type matchEmitter struct {
20
+ // Dst is the destination slice that Matches are added to.
21
+ Dst []Match
22
+
23
+ // NextEmit is the index of the next byte to emit.
24
+ NextEmit int
25
+ }
26
+
27
+ func (e *matchEmitter) emit(m absoluteMatch) {
28
+ e.Dst = append(e.Dst, Match{
29
+ Unmatched: m.Start - e.NextEmit,
30
+ Length: m.End - m.Start,
31
+ Distance: m.Start - m.Match,
32
+ })
33
+ e.NextEmit = m.End
34
+ }
@@ -0,0 +1,175 @@
1
+ package matchfinder
2
+
3
+ import (
4
+ "encoding/binary"
5
+ )
6
+
7
+ // M0 is an implementation of the MatchFinder interface based
8
+ // on the algorithm used by snappy, but modified to be more like the algorithm
9
+ // used by compression level 0 of the brotli reference implementation.
10
+ //
11
+ // It has a maximum block size of 65536 bytes.
12
+ type M0 struct {
13
+ // Lazy turns on "lazy matching," for higher compression but less speed.
14
+ Lazy bool
15
+
16
+ MaxDistance int
17
+ MaxLength int
18
+ }
19
+
20
+ func (M0) Reset() {}
21
+
22
+ const (
23
+ m0HashLen = 5
24
+
25
+ m0TableBits = 16
26
+ m0TableSize = 1 << m0TableBits
27
+ m0Shift = 32 - m0TableBits
28
+ // m0TableMask is redundant, but helps the compiler eliminate bounds
29
+ // checks.
30
+ m0TableMask = m0TableSize - 1
31
+ )
32
+
33
+ func (m M0) hash(data uint64) uint64 {
34
+ hash := (data << (64 - 8*m0HashLen)) * hashMul64
35
+ return hash >> (64 - m0TableBits)
36
+ }
37
+
38
+ // FindMatches looks for matches in src, appends them to dst, and returns dst.
39
+ // src must not be longer than 65536 bytes.
40
+ func (m M0) FindMatches(dst []Match, src []byte) []Match {
41
+ const inputMargin = 16 - 1
42
+ const minNonLiteralBlockSize = 1 + 1 + inputMargin
43
+
44
+ if len(src) < minNonLiteralBlockSize {
45
+ dst = append(dst, Match{
46
+ Unmatched: len(src),
47
+ })
48
+ return dst
49
+ }
50
+ if len(src) > 65536 {
51
+ panic("block too long")
52
+ }
53
+
54
+ var table [m0TableSize]uint16
55
+
56
+ // sLimit is when to stop looking for offset/length copies. The inputMargin
57
+ // lets us use a fast path for emitLiteral in the main loop, while we are
58
+ // looking for copies.
59
+ sLimit := len(src) - inputMargin
60
+
61
+ // nextEmit is where in src the next emitLiteral should start from.
62
+ nextEmit := 0
63
+
64
+ // The encoded form must start with a literal, as there are no previous
65
+ // bytes to copy, so we start looking for hash matches at s == 1.
66
+ s := 1
67
+ nextHash := m.hash(binary.LittleEndian.Uint64(src[s:]))
68
+
69
+ for {
70
+ // Copied from the C++ snappy implementation:
71
+ //
72
+ // Heuristic match skipping: If 32 bytes are scanned with no matches
73
+ // found, start looking only at every other byte. If 32 more bytes are
74
+ // scanned (or skipped), look at every third byte, etc.. When a match
75
+ // is found, immediately go back to looking at every byte. This is a
76
+ // small loss (~5% performance, ~0.1% density) for compressible data
77
+ // due to more bookkeeping, but for non-compressible data (such as
78
+ // JPEG) it's a huge win since the compressor quickly "realizes" the
79
+ // data is incompressible and doesn't bother looking for matches
80
+ // everywhere.
81
+ //
82
+ // The "skip" variable keeps track of how many bytes there are since
83
+ // the last match; dividing it by 32 (ie. right-shifting by five) gives
84
+ // the number of bytes to move ahead for each iteration.
85
+ skip := 32
86
+
87
+ nextS := s
88
+ candidate := 0
89
+ for {
90
+ s = nextS
91
+ bytesBetweenHashLookups := skip >> 5
92
+ nextS = s + bytesBetweenHashLookups
93
+ skip += bytesBetweenHashLookups
94
+ if nextS > sLimit {
95
+ goto emitRemainder
96
+ }
97
+ candidate = int(table[nextHash&m0TableMask])
98
+ table[nextHash&m0TableMask] = uint16(s)
99
+ nextHash = m.hash(binary.LittleEndian.Uint64(src[nextS:]))
100
+ if m.MaxDistance != 0 && s-candidate > m.MaxDistance {
101
+ continue
102
+ }
103
+ if binary.LittleEndian.Uint32(src[s:]) == binary.LittleEndian.Uint32(src[candidate:]) {
104
+ break
105
+ }
106
+ }
107
+
108
+ // Invariant: we have a 4-byte match at s.
109
+ base := s
110
+ s = extendMatch(src, candidate+4, s+4)
111
+
112
+ origBase := base
113
+ if m.Lazy && base+1 < sLimit {
114
+ newBase := base + 1
115
+ h := m.hash(binary.LittleEndian.Uint64(src[newBase:]))
116
+ newCandidate := int(table[h&m0TableMask])
117
+ table[h&m0TableMask] = uint16(newBase)
118
+ okDistance := true
119
+ if m.MaxDistance != 0 && newBase-newCandidate > m.MaxDistance {
120
+ okDistance = false
121
+ }
122
+ if okDistance && binary.LittleEndian.Uint32(src[newBase:]) == binary.LittleEndian.Uint32(src[newCandidate:]) {
123
+ newS := extendMatch(src, newCandidate+4, newBase+4)
124
+ if newS-newBase > s-base+1 {
125
+ s = newS
126
+ base = newBase
127
+ candidate = newCandidate
128
+ }
129
+ }
130
+ }
131
+
132
+ if m.MaxLength != 0 && s-base > m.MaxLength {
133
+ s = base + m.MaxLength
134
+ }
135
+ dst = append(dst, Match{
136
+ Unmatched: base - nextEmit,
137
+ Length: s - base,
138
+ Distance: base - candidate,
139
+ })
140
+ nextEmit = s
141
+ if s >= sLimit {
142
+ goto emitRemainder
143
+ }
144
+
145
+ if m.Lazy {
146
+ // If lazy matching is enabled, we update the hash table for
147
+ // every byte in the match.
148
+ for i := origBase + 2; i < s-3; i++ {
149
+ x := binary.LittleEndian.Uint64(src[i:])
150
+ table[m.hash(x)&m0TableMask] = uint16(i)
151
+ }
152
+ } else {
153
+ // We update the hash table only at base+1
154
+ x := binary.LittleEndian.Uint64(src[base+1:])
155
+ table[m.hash(x)&m0TableMask] = uint16(base + 1)
156
+ }
157
+
158
+ // We could immediately start working at s now, but to improve
159
+ // compression we first update the hash table.
160
+ x := binary.LittleEndian.Uint64(src[s-3:])
161
+ table[m.hash(x)&m0TableMask] = uint16(s - 3)
162
+ table[m.hash(x>>8)&m0TableMask] = uint16(s - 2)
163
+ prevHash := m.hash(x >> 16)
164
+ table[prevHash&m0TableMask] = uint16(s - 1)
165
+ nextHash = m.hash(x >> 24)
166
+ }
167
+
168
+ emitRemainder:
169
+ if nextEmit < len(src) {
170
+ dst = append(dst, Match{
171
+ Unmatched: len(src) - nextEmit,
172
+ })
173
+ }
174
+ return dst
175
+ }