typenative 0.3.1 → 0.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (170) hide show
  1. package/.github/workflows/ci.yml +2 -0
  2. package/CHANGELOG.md +258 -0
  3. package/CONTRIBUTION.md +1 -1
  4. package/README.md +17 -1
  5. package/bin/cli.js +26 -11
  6. package/bin/go/bridge.js +74 -14
  7. package/bin/go/convert.js +59 -1
  8. package/bin/go/metadata.js +13 -4
  9. package/bin/go/overrides.js +4 -6
  10. package/bin/go/stdlib.generated.js +2 -0
  11. package/bin/index.js +338 -47
  12. package/bin/main.js +36 -14
  13. package/bin/node/assert.js +2 -2
  14. package/bin/node/buffer.js +9 -0
  15. package/bin/node/child_process.js +51 -3
  16. package/bin/node/cluster.js +41 -3
  17. package/bin/node/core.js +1 -1
  18. package/bin/node/crypto.js +205 -4
  19. package/bin/node/dgram.js +230 -6
  20. package/bin/node/dns.js +471 -8
  21. package/bin/node/errno_tables.js +445 -0
  22. package/bin/node/events.js +1 -1
  23. package/bin/node/fs.js +22 -15
  24. package/bin/node/fs_watch.js +407 -0
  25. package/bin/node/http.js +128 -30
  26. package/bin/node/http2.js +37 -7
  27. package/bin/node/module_v8.js +142 -0
  28. package/bin/node/net.js +9 -6
  29. package/bin/node/process.js +242 -0
  30. package/bin/node/readline.js +272 -1
  31. package/bin/node/registry.js +3 -1
  32. package/bin/node/sqlite.js +60 -12
  33. package/bin/node/stream.js +145 -10
  34. package/bin/node/test.js +311 -19
  35. package/bin/node/timers.js +1 -1
  36. package/bin/node/typed_arrays.js +608 -0
  37. package/bin/node/url.js +23 -5
  38. package/bin/node/util.js +21 -1
  39. package/bin/node/util_extra.js +6 -30
  40. package/bin/node/worker_threads.js +79 -19
  41. package/bin/node/zlib.js +131 -16
  42. package/bin/transpiler.js +4729 -616
  43. package/bin/typecheck-node.js +1 -0
  44. package/bin/version.js +1 -1
  45. package/conformance.md +1030 -877
  46. package/examples/README.md +54 -0
  47. package/examples/fibonacci/README.md +89 -0
  48. package/examples/fibonacci/bench.mjs +87 -0
  49. package/examples/fibonacci/main.ts +21 -0
  50. package/examples/hello-world/README.md +62 -0
  51. package/examples/hello-world/bench.mjs +66 -0
  52. package/examples/hello-world/main.ts +7 -0
  53. package/examples/json-server/README.md +127 -0
  54. package/examples/json-server/bench.mjs +263 -0
  55. package/examples/json-server/main.ts +67 -0
  56. package/govendor/github.com/andybalholm/brotli/LICENSE +19 -0
  57. package/govendor/github.com/andybalholm/brotli/README.md +19 -0
  58. package/govendor/github.com/andybalholm/brotli/backward_references.go +185 -0
  59. package/govendor/github.com/andybalholm/brotli/backward_references_hq.go +796 -0
  60. package/govendor/github.com/andybalholm/brotli/bit_cost.go +436 -0
  61. package/govendor/github.com/andybalholm/brotli/bit_reader.go +266 -0
  62. package/govendor/github.com/andybalholm/brotli/bitwriter.go +56 -0
  63. package/govendor/github.com/andybalholm/brotli/block_splitter.go +144 -0
  64. package/govendor/github.com/andybalholm/brotli/block_splitter_command.go +434 -0
  65. package/govendor/github.com/andybalholm/brotli/block_splitter_distance.go +433 -0
  66. package/govendor/github.com/andybalholm/brotli/block_splitter_literal.go +433 -0
  67. package/govendor/github.com/andybalholm/brotli/brotli_bit_stream.go +1435 -0
  68. package/govendor/github.com/andybalholm/brotli/cluster.go +30 -0
  69. package/govendor/github.com/andybalholm/brotli/cluster_command.go +164 -0
  70. package/govendor/github.com/andybalholm/brotli/cluster_distance.go +326 -0
  71. package/govendor/github.com/andybalholm/brotli/cluster_literal.go +326 -0
  72. package/govendor/github.com/andybalholm/brotli/command.go +254 -0
  73. package/govendor/github.com/andybalholm/brotli/compress_fragment.go +834 -0
  74. package/govendor/github.com/andybalholm/brotli/compress_fragment_two_pass.go +775 -0
  75. package/govendor/github.com/andybalholm/brotli/constants.go +77 -0
  76. package/govendor/github.com/andybalholm/brotli/context.go +2176 -0
  77. package/govendor/github.com/andybalholm/brotli/decode.go +2570 -0
  78. package/govendor/github.com/andybalholm/brotli/dictionary.go +122890 -0
  79. package/govendor/github.com/andybalholm/brotli/dictionary_hash.go +32779 -0
  80. package/govendor/github.com/andybalholm/brotli/encode.go +1220 -0
  81. package/govendor/github.com/andybalholm/brotli/encoder.go +177 -0
  82. package/govendor/github.com/andybalholm/brotli/encoder_dict.go +22 -0
  83. package/govendor/github.com/andybalholm/brotli/encoder_fast.go +201 -0
  84. package/govendor/github.com/andybalholm/brotli/entropy_encode.go +592 -0
  85. package/govendor/github.com/andybalholm/brotli/entropy_encode_static.go +4399 -0
  86. package/govendor/github.com/andybalholm/brotli/fast_log.go +290 -0
  87. package/govendor/github.com/andybalholm/brotli/find_match_length.go +45 -0
  88. package/govendor/github.com/andybalholm/brotli/go.mod +3 -0
  89. package/govendor/github.com/andybalholm/brotli/h10.go +287 -0
  90. package/govendor/github.com/andybalholm/brotli/h5.go +214 -0
  91. package/govendor/github.com/andybalholm/brotli/h6.go +216 -0
  92. package/govendor/github.com/andybalholm/brotli/hash.go +342 -0
  93. package/govendor/github.com/andybalholm/brotli/hash_composite.go +93 -0
  94. package/govendor/github.com/andybalholm/brotli/hash_forgetful_chain.go +252 -0
  95. package/govendor/github.com/andybalholm/brotli/hash_longest_match_quickly.go +214 -0
  96. package/govendor/github.com/andybalholm/brotli/hash_rolling.go +168 -0
  97. package/govendor/github.com/andybalholm/brotli/histogram.go +226 -0
  98. package/govendor/github.com/andybalholm/brotli/http.go +201 -0
  99. package/govendor/github.com/andybalholm/brotli/huffman.go +653 -0
  100. package/govendor/github.com/andybalholm/brotli/literal_cost.go +182 -0
  101. package/govendor/github.com/andybalholm/brotli/matchfinder/bargain1.go +212 -0
  102. package/govendor/github.com/andybalholm/brotli/matchfinder/bargain2.go +248 -0
  103. package/govendor/github.com/andybalholm/brotli/matchfinder/bargain3.go +278 -0
  104. package/govendor/github.com/andybalholm/brotli/matchfinder/emitter.go +34 -0
  105. package/govendor/github.com/andybalholm/brotli/matchfinder/m0.go +175 -0
  106. package/govendor/github.com/andybalholm/brotli/matchfinder/m4.go +339 -0
  107. package/govendor/github.com/andybalholm/brotli/matchfinder/matchfinder.go +103 -0
  108. package/govendor/github.com/andybalholm/brotli/matchfinder/pathfinder.go +328 -0
  109. package/govendor/github.com/andybalholm/brotli/matchfinder/textencoder.go +53 -0
  110. package/govendor/github.com/andybalholm/brotli/matchfinder/trio.go +327 -0
  111. package/govendor/github.com/andybalholm/brotli/matchfinder/zdfast.go +258 -0
  112. package/govendor/github.com/andybalholm/brotli/matchfinder/zfast.go +206 -0
  113. package/govendor/github.com/andybalholm/brotli/matchfinder/zm.go +302 -0
  114. package/govendor/github.com/andybalholm/brotli/memory.go +66 -0
  115. package/govendor/github.com/andybalholm/brotli/metablock.go +574 -0
  116. package/govendor/github.com/andybalholm/brotli/metablock_command.go +165 -0
  117. package/govendor/github.com/andybalholm/brotli/metablock_distance.go +165 -0
  118. package/govendor/github.com/andybalholm/brotli/metablock_literal.go +165 -0
  119. package/govendor/github.com/andybalholm/brotli/params.go +37 -0
  120. package/govendor/github.com/andybalholm/brotli/platform.go +103 -0
  121. package/govendor/github.com/andybalholm/brotli/prefix.go +30 -0
  122. package/govendor/github.com/andybalholm/brotli/prefix_dec.go +723 -0
  123. package/govendor/github.com/andybalholm/brotli/quality.go +196 -0
  124. package/govendor/github.com/andybalholm/brotli/reader.go +127 -0
  125. package/govendor/github.com/andybalholm/brotli/ringbuffer.go +134 -0
  126. package/govendor/github.com/andybalholm/brotli/state.go +313 -0
  127. package/govendor/github.com/andybalholm/brotli/static_dict.go +662 -0
  128. package/govendor/github.com/andybalholm/brotli/static_dict_lut.go +75094 -0
  129. package/govendor/github.com/andybalholm/brotli/symbol_list.go +22 -0
  130. package/govendor/github.com/andybalholm/brotli/transform.go +641 -0
  131. package/govendor/github.com/andybalholm/brotli/utf8_util.go +70 -0
  132. package/govendor/github.com/andybalholm/brotli/util.go +7 -0
  133. package/govendor/github.com/andybalholm/brotli/write_bits.go +52 -0
  134. package/govendor/github.com/andybalholm/brotli/writer.go +162 -0
  135. package/govendor/github.com/fsnotify/fsnotify/.mailmap +2 -0
  136. package/govendor/github.com/fsnotify/fsnotify/CHANGELOG.md +651 -0
  137. package/govendor/github.com/fsnotify/fsnotify/CONTRIBUTING.md +147 -0
  138. package/govendor/github.com/fsnotify/fsnotify/LICENSE +25 -0
  139. package/govendor/github.com/fsnotify/fsnotify/README.md +214 -0
  140. package/govendor/github.com/fsnotify/fsnotify/backend_fen.go +469 -0
  141. package/govendor/github.com/fsnotify/fsnotify/backend_inotify.go +597 -0
  142. package/govendor/github.com/fsnotify/fsnotify/backend_kqueue.go +735 -0
  143. package/govendor/github.com/fsnotify/fsnotify/backend_other.go +22 -0
  144. package/govendor/github.com/fsnotify/fsnotify/backend_windows.go +702 -0
  145. package/govendor/github.com/fsnotify/fsnotify/fsnotify.go +526 -0
  146. package/govendor/github.com/fsnotify/fsnotify/go.mod +3 -0
  147. package/govendor/github.com/fsnotify/fsnotify/internal/darwin.go +20 -0
  148. package/govendor/github.com/fsnotify/fsnotify/internal/debug_darwin.go +15 -0
  149. package/govendor/github.com/fsnotify/fsnotify/internal/debug_dragonfly.go +15 -0
  150. package/govendor/github.com/fsnotify/fsnotify/internal/debug_freebsd.go +20 -0
  151. package/govendor/github.com/fsnotify/fsnotify/internal/debug_kqueue.go +32 -0
  152. package/govendor/github.com/fsnotify/fsnotify/internal/debug_linux.go +56 -0
  153. package/govendor/github.com/fsnotify/fsnotify/internal/debug_netbsd.go +15 -0
  154. package/govendor/github.com/fsnotify/fsnotify/internal/debug_openbsd.go +16 -0
  155. package/govendor/github.com/fsnotify/fsnotify/internal/debug_solaris.go +45 -0
  156. package/govendor/github.com/fsnotify/fsnotify/internal/debug_windows.go +40 -0
  157. package/govendor/github.com/fsnotify/fsnotify/internal/freebsd.go +20 -0
  158. package/govendor/github.com/fsnotify/fsnotify/internal/internal.go +2 -0
  159. package/govendor/github.com/fsnotify/fsnotify/internal/unix.go +20 -0
  160. package/govendor/github.com/fsnotify/fsnotify/internal/unix2.go +25 -0
  161. package/govendor/github.com/fsnotify/fsnotify/internal/windows.go +40 -0
  162. package/govendor/github.com/fsnotify/fsnotify/shared.go +64 -0
  163. package/govendor/github.com/fsnotify/fsnotify/staticcheck.conf +3 -0
  164. package/govendor/github.com/fsnotify/fsnotify/system_bsd.go +7 -0
  165. package/govendor/github.com/fsnotify/fsnotify/system_darwin.go +8 -0
  166. package/govendor/manifest.txt +3 -1
  167. package/package.json +1 -1
  168. package/types/typenative-go.d.ts +23 -0
  169. package/types/typenative-npm.d.ts +241 -5
  170. package/types/typenative.d.ts +302 -12
@@ -0,0 +1,342 @@
1
+ package brotli
2
+
3
+ import (
4
+ "encoding/binary"
5
+ "fmt"
6
+ )
7
+
8
+ type hasherCommon struct {
9
+ params hasherParams
10
+ is_prepared_ bool
11
+ dict_num_lookups uint
12
+ dict_num_matches uint
13
+ }
14
+
15
+ func (h *hasherCommon) Common() *hasherCommon {
16
+ return h
17
+ }
18
+
19
+ type hasherHandle interface {
20
+ Common() *hasherCommon
21
+ Initialize(params *encoderParams)
22
+ Prepare(one_shot bool, input_size uint, data []byte)
23
+ StitchToPreviousBlock(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint)
24
+ HashTypeLength() uint
25
+ StoreLookahead() uint
26
+ PrepareDistanceCache(distance_cache []int)
27
+ FindLongestMatch(dictionary *encoderDictionary, data []byte, ring_buffer_mask uint, distance_cache []int, cur_ix uint, max_length uint, max_backward uint, gap uint, max_distance uint, out *hasherSearchResult)
28
+ StoreRange(data []byte, mask uint, ix_start uint, ix_end uint)
29
+ Store(data []byte, mask uint, ix uint)
30
+ }
31
+
32
+ const kCutoffTransformsCount uint32 = 10
33
+
34
+ /* 0, 12, 27, 23, 42, 63, 56, 48, 59, 64 */
35
+ /* 0+0, 4+8, 8+19, 12+11, 16+26, 20+43, 24+32, 28+20, 32+27, 36+28 */
36
+ const kCutoffTransforms uint64 = 0x071B520ADA2D3200
37
+
38
+ type hasherSearchResult struct {
39
+ len uint
40
+ distance uint
41
+ score uint
42
+ len_code_delta int
43
+ }
44
+
45
+ /* kHashMul32 multiplier has these properties:
46
+ * The multiplier must be odd. Otherwise we may lose the highest bit.
47
+ * No long streaks of ones or zeros.
48
+ * There is no effort to ensure that it is a prime, the oddity is enough
49
+ for this use.
50
+ * The number has been tuned heuristically against compression benchmarks. */
51
+ const kHashMul32 uint32 = 0x1E35A7BD
52
+
53
+ const kHashMul64 uint64 = 0x1E35A7BD1E35A7BD
54
+
55
+ const kHashMul64Long uint64 = 0x1FE35A7BD3579BD3
56
+
57
+ func hash14(data []byte) uint32 {
58
+ var h uint32 = binary.LittleEndian.Uint32(data) * kHashMul32
59
+
60
+ /* The higher bits contain more mixture from the multiplication,
61
+ so we take our results from there. */
62
+ return h >> (32 - 14)
63
+ }
64
+
65
+ func prepareDistanceCache(distance_cache []int, num_distances int) {
66
+ if num_distances > 4 {
67
+ var last_distance int = distance_cache[0]
68
+ distance_cache[4] = last_distance - 1
69
+ distance_cache[5] = last_distance + 1
70
+ distance_cache[6] = last_distance - 2
71
+ distance_cache[7] = last_distance + 2
72
+ distance_cache[8] = last_distance - 3
73
+ distance_cache[9] = last_distance + 3
74
+ if num_distances > 10 {
75
+ var next_last_distance int = distance_cache[1]
76
+ distance_cache[10] = next_last_distance - 1
77
+ distance_cache[11] = next_last_distance + 1
78
+ distance_cache[12] = next_last_distance - 2
79
+ distance_cache[13] = next_last_distance + 2
80
+ distance_cache[14] = next_last_distance - 3
81
+ distance_cache[15] = next_last_distance + 3
82
+ }
83
+ }
84
+ }
85
+
86
+ const literalByteScore = 135
87
+
88
+ const distanceBitPenalty = 30
89
+
90
+ /* Score must be positive after applying maximal penalty. */
91
+ const scoreBase = (distanceBitPenalty * 8 * 8)
92
+
93
+ /* Usually, we always choose the longest backward reference. This function
94
+ allows for the exception of that rule.
95
+
96
+ If we choose a backward reference that is further away, it will
97
+ usually be coded with more bits. We approximate this by assuming
98
+ log2(distance). If the distance can be expressed in terms of the
99
+ last four distances, we use some heuristic constants to estimate
100
+ the bits cost. For the first up to four literals we use the bit
101
+ cost of the literals from the literal cost model, after that we
102
+ use the average bit cost of the cost model.
103
+
104
+ This function is used to sometimes discard a longer backward reference
105
+ when it is not much longer and the bit cost for encoding it is more
106
+ than the saved literals.
107
+
108
+ backward_reference_offset MUST be positive. */
109
+ func backwardReferenceScore(copy_length uint, backward_reference_offset uint) uint {
110
+ return scoreBase + literalByteScore*uint(copy_length) - distanceBitPenalty*uint(log2FloorNonZero(backward_reference_offset))
111
+ }
112
+
113
+ func backwardReferenceScoreUsingLastDistance(copy_length uint) uint {
114
+ return literalByteScore*uint(copy_length) + scoreBase + 15
115
+ }
116
+
117
+ func backwardReferencePenaltyUsingLastDistance(distance_short_code uint) uint {
118
+ return uint(39) + ((0x1CA10 >> (distance_short_code & 0xE)) & 0xE)
119
+ }
120
+
121
+ func testStaticDictionaryItem(dictionary *encoderDictionary, item uint, data []byte, max_length uint, max_backward uint, max_distance uint, out *hasherSearchResult) bool {
122
+ var len uint
123
+ var word_idx uint
124
+ var offset uint
125
+ var matchlen uint
126
+ var backward uint
127
+ var score uint
128
+ len = item & 0x1F
129
+ word_idx = item >> 5
130
+ offset = uint(dictionary.words.offsets_by_length[len]) + len*word_idx
131
+ if len > max_length {
132
+ return false
133
+ }
134
+
135
+ matchlen = findMatchLengthWithLimit(data, dictionary.words.data[offset:], uint(len))
136
+ if matchlen+uint(dictionary.cutoffTransformsCount) <= len || matchlen == 0 {
137
+ return false
138
+ }
139
+ {
140
+ var cut uint = len - matchlen
141
+ var transform_id uint = (cut << 2) + uint((dictionary.cutoffTransforms>>(cut*6))&0x3F)
142
+ backward = max_backward + 1 + word_idx + (transform_id << dictionary.words.size_bits_by_length[len])
143
+ }
144
+
145
+ if backward > max_distance {
146
+ return false
147
+ }
148
+
149
+ score = backwardReferenceScore(matchlen, backward)
150
+ if score < out.score {
151
+ return false
152
+ }
153
+
154
+ out.len = matchlen
155
+ out.len_code_delta = int(len) - int(matchlen)
156
+ out.distance = backward
157
+ out.score = score
158
+ return true
159
+ }
160
+
161
+ func searchInStaticDictionary(dictionary *encoderDictionary, handle hasherHandle, data []byte, max_length uint, max_backward uint, max_distance uint, out *hasherSearchResult, shallow bool) {
162
+ var key uint
163
+ var i uint
164
+ var self *hasherCommon = handle.Common()
165
+ if self.dict_num_matches < self.dict_num_lookups>>7 {
166
+ return
167
+ }
168
+
169
+ key = uint(hash14(data) << 1)
170
+ for i = 0; ; (func() { i++; key++ })() {
171
+ var tmp uint
172
+ if shallow {
173
+ tmp = 1
174
+ } else {
175
+ tmp = 2
176
+ }
177
+ if i >= tmp {
178
+ break
179
+ }
180
+ var item uint = uint(dictionary.hash_table[key])
181
+ self.dict_num_lookups++
182
+ if item != 0 {
183
+ var item_matches bool = testStaticDictionaryItem(dictionary, item, data, max_length, max_backward, max_distance, out)
184
+ if item_matches {
185
+ self.dict_num_matches++
186
+ }
187
+ }
188
+ }
189
+ }
190
+
191
+ type backwardMatch struct {
192
+ distance uint32
193
+ length_and_code uint32
194
+ }
195
+
196
+ func initBackwardMatch(self *backwardMatch, dist uint, len uint) {
197
+ self.distance = uint32(dist)
198
+ self.length_and_code = uint32(len << 5)
199
+ }
200
+
201
+ func initDictionaryBackwardMatch(self *backwardMatch, dist uint, len uint, len_code uint) {
202
+ self.distance = uint32(dist)
203
+ var tmp uint
204
+ if len == len_code {
205
+ tmp = 0
206
+ } else {
207
+ tmp = len_code
208
+ }
209
+ self.length_and_code = uint32(len<<5 | tmp)
210
+ }
211
+
212
+ func backwardMatchLength(self *backwardMatch) uint {
213
+ return uint(self.length_and_code >> 5)
214
+ }
215
+
216
+ func backwardMatchLengthCode(self *backwardMatch) uint {
217
+ var code uint = uint(self.length_and_code) & 31
218
+ if code != 0 {
219
+ return code
220
+ } else {
221
+ return backwardMatchLength(self)
222
+ }
223
+ }
224
+
225
+ func hasherReset(handle hasherHandle) {
226
+ if handle == nil {
227
+ return
228
+ }
229
+ handle.Common().is_prepared_ = false
230
+ }
231
+
232
+ func newHasher(typ int) hasherHandle {
233
+ switch typ {
234
+ case 2:
235
+ return &hashLongestMatchQuickly{
236
+ bucketBits: 16,
237
+ bucketSweep: 1,
238
+ hashLen: 5,
239
+ useDictionary: true,
240
+ }
241
+ case 3:
242
+ return &hashLongestMatchQuickly{
243
+ bucketBits: 16,
244
+ bucketSweep: 2,
245
+ hashLen: 5,
246
+ useDictionary: false,
247
+ }
248
+ case 4:
249
+ return &hashLongestMatchQuickly{
250
+ bucketBits: 17,
251
+ bucketSweep: 4,
252
+ hashLen: 5,
253
+ useDictionary: true,
254
+ }
255
+ case 5:
256
+ return new(h5)
257
+ case 6:
258
+ return new(h6)
259
+ case 10:
260
+ return new(h10)
261
+ case 35:
262
+ return &hashComposite{
263
+ ha: newHasher(3),
264
+ hb: &hashRolling{jump: 4},
265
+ }
266
+ case 40:
267
+ return &hashForgetfulChain{
268
+ bucketBits: 15,
269
+ numBanks: 1,
270
+ bankBits: 16,
271
+ numLastDistancesToCheck: 4,
272
+ }
273
+ case 41:
274
+ return &hashForgetfulChain{
275
+ bucketBits: 15,
276
+ numBanks: 1,
277
+ bankBits: 16,
278
+ numLastDistancesToCheck: 10,
279
+ }
280
+ case 42:
281
+ return &hashForgetfulChain{
282
+ bucketBits: 15,
283
+ numBanks: 512,
284
+ bankBits: 9,
285
+ numLastDistancesToCheck: 16,
286
+ }
287
+ case 54:
288
+ return &hashLongestMatchQuickly{
289
+ bucketBits: 20,
290
+ bucketSweep: 4,
291
+ hashLen: 7,
292
+ useDictionary: false,
293
+ }
294
+ case 55:
295
+ return &hashComposite{
296
+ ha: newHasher(54),
297
+ hb: &hashRolling{jump: 4},
298
+ }
299
+ case 65:
300
+ return &hashComposite{
301
+ ha: newHasher(6),
302
+ hb: &hashRolling{jump: 1},
303
+ }
304
+ }
305
+
306
+ panic(fmt.Sprintf("unknown hasher type: %d", typ))
307
+ }
308
+
309
+ func hasherSetup(handle *hasherHandle, params *encoderParams, data []byte, position uint, input_size uint, is_last bool) {
310
+ var self hasherHandle = nil
311
+ var common *hasherCommon = nil
312
+ var one_shot bool = (position == 0 && is_last)
313
+ if *handle == nil {
314
+ chooseHasher(params, &params.hasher)
315
+ self = newHasher(params.hasher.type_)
316
+
317
+ *handle = self
318
+ common = self.Common()
319
+ common.params = params.hasher
320
+ self.Initialize(params)
321
+ }
322
+
323
+ self = *handle
324
+ common = self.Common()
325
+ if !common.is_prepared_ {
326
+ self.Prepare(one_shot, input_size, data)
327
+
328
+ if position == 0 {
329
+ common.dict_num_lookups = 0
330
+ common.dict_num_matches = 0
331
+ }
332
+
333
+ common.is_prepared_ = true
334
+ }
335
+ }
336
+
337
+ func initOrStitchToPreviousBlock(handle *hasherHandle, data []byte, mask uint, params *encoderParams, position uint, input_size uint, is_last bool) {
338
+ var self hasherHandle
339
+ hasherSetup(handle, params, data, position, input_size, is_last)
340
+ self = *handle
341
+ self.StitchToPreviousBlock(input_size, position, data, mask)
342
+ }
@@ -0,0 +1,93 @@
1
+ package brotli
2
+
3
+ /* Copyright 2018 Google Inc. All Rights Reserved.
4
+
5
+ Distributed under MIT license.
6
+ See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
7
+ */
8
+
9
+ func (h *hashComposite) HashTypeLength() uint {
10
+ var a uint = h.ha.HashTypeLength()
11
+ var b uint = h.hb.HashTypeLength()
12
+ if a > b {
13
+ return a
14
+ } else {
15
+ return b
16
+ }
17
+ }
18
+
19
+ func (h *hashComposite) StoreLookahead() uint {
20
+ var a uint = h.ha.StoreLookahead()
21
+ var b uint = h.hb.StoreLookahead()
22
+ if a > b {
23
+ return a
24
+ } else {
25
+ return b
26
+ }
27
+ }
28
+
29
+ /* Composite hasher: This hasher allows to combine two other hashers, HASHER_A
30
+ and HASHER_B. */
31
+ type hashComposite struct {
32
+ hasherCommon
33
+ ha hasherHandle
34
+ hb hasherHandle
35
+ params *encoderParams
36
+ }
37
+
38
+ func (h *hashComposite) Initialize(params *encoderParams) {
39
+ h.params = params
40
+ }
41
+
42
+ /* TODO: Initialize of the hashers is defered to Prepare (and params
43
+ remembered here) because we don't get the one_shot and input_size params
44
+ here that are needed to know the memory size of them. Instead provide
45
+ those params to all hashers InitializehashComposite */
46
+ func (h *hashComposite) Prepare(one_shot bool, input_size uint, data []byte) {
47
+ if h.ha == nil {
48
+ var common_a *hasherCommon
49
+ var common_b *hasherCommon
50
+
51
+ common_a = h.ha.Common()
52
+ common_a.params = h.params.hasher
53
+ common_a.is_prepared_ = false
54
+ common_a.dict_num_lookups = 0
55
+ common_a.dict_num_matches = 0
56
+ h.ha.Initialize(h.params)
57
+
58
+ common_b = h.hb.Common()
59
+ common_b.params = h.params.hasher
60
+ common_b.is_prepared_ = false
61
+ common_b.dict_num_lookups = 0
62
+ common_b.dict_num_matches = 0
63
+ h.hb.Initialize(h.params)
64
+ }
65
+
66
+ h.ha.Prepare(one_shot, input_size, data)
67
+ h.hb.Prepare(one_shot, input_size, data)
68
+ }
69
+
70
+ func (h *hashComposite) Store(data []byte, mask uint, ix uint) {
71
+ h.ha.Store(data, mask, ix)
72
+ h.hb.Store(data, mask, ix)
73
+ }
74
+
75
+ func (h *hashComposite) StoreRange(data []byte, mask uint, ix_start uint, ix_end uint) {
76
+ h.ha.StoreRange(data, mask, ix_start, ix_end)
77
+ h.hb.StoreRange(data, mask, ix_start, ix_end)
78
+ }
79
+
80
+ func (h *hashComposite) StitchToPreviousBlock(num_bytes uint, position uint, ringbuffer []byte, ring_buffer_mask uint) {
81
+ h.ha.StitchToPreviousBlock(num_bytes, position, ringbuffer, ring_buffer_mask)
82
+ h.hb.StitchToPreviousBlock(num_bytes, position, ringbuffer, ring_buffer_mask)
83
+ }
84
+
85
+ func (h *hashComposite) PrepareDistanceCache(distance_cache []int) {
86
+ h.ha.PrepareDistanceCache(distance_cache)
87
+ h.hb.PrepareDistanceCache(distance_cache)
88
+ }
89
+
90
+ func (h *hashComposite) FindLongestMatch(dictionary *encoderDictionary, data []byte, ring_buffer_mask uint, distance_cache []int, cur_ix uint, max_length uint, max_backward uint, gap uint, max_distance uint, out *hasherSearchResult) {
91
+ h.ha.FindLongestMatch(dictionary, data, ring_buffer_mask, distance_cache, cur_ix, max_length, max_backward, gap, max_distance, out)
92
+ h.hb.FindLongestMatch(dictionary, data, ring_buffer_mask, distance_cache, cur_ix, max_length, max_backward, gap, max_distance, out)
93
+ }
@@ -0,0 +1,252 @@
1
+ package brotli
2
+
3
+ import "encoding/binary"
4
+
5
+ /* Copyright 2016 Google Inc. All Rights Reserved.
6
+
7
+ Distributed under MIT license.
8
+ See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
9
+ */
10
+
11
+ func (*hashForgetfulChain) HashTypeLength() uint {
12
+ return 4
13
+ }
14
+
15
+ func (*hashForgetfulChain) StoreLookahead() uint {
16
+ return 4
17
+ }
18
+
19
+ /* HashBytes is the function that chooses the bucket to place the address in.*/
20
+ func (h *hashForgetfulChain) HashBytes(data []byte) uint {
21
+ var hash uint32 = binary.LittleEndian.Uint32(data) * kHashMul32
22
+
23
+ /* The higher bits contain more mixture from the multiplication,
24
+ so we take our results from there. */
25
+ return uint(hash >> (32 - h.bucketBits))
26
+ }
27
+
28
+ type slot struct {
29
+ delta uint16
30
+ next uint16
31
+ }
32
+
33
+ /* A (forgetful) hash table to the data seen by the compressor, to
34
+ help create backward references to previous data.
35
+
36
+ Hashes are stored in chains which are bucketed to groups. Group of chains
37
+ share a storage "bank". When more than "bank size" chain nodes are added,
38
+ oldest nodes are replaced; this way several chains may share a tail. */
39
+ type hashForgetfulChain struct {
40
+ hasherCommon
41
+
42
+ bucketBits uint
43
+ numBanks uint
44
+ bankBits uint
45
+ numLastDistancesToCheck int
46
+
47
+ addr []uint32
48
+ head []uint16
49
+ tiny_hash [65536]byte
50
+ banks [][]slot
51
+ free_slot_idx []uint16
52
+ max_hops uint
53
+ }
54
+
55
+ func (h *hashForgetfulChain) Initialize(params *encoderParams) {
56
+ var q uint
57
+ if params.quality > 6 {
58
+ q = 7
59
+ } else {
60
+ q = 8
61
+ }
62
+ h.max_hops = q << uint(params.quality-4)
63
+
64
+ bankSize := 1 << h.bankBits
65
+ bucketSize := 1 << h.bucketBits
66
+
67
+ h.addr = make([]uint32, bucketSize)
68
+ h.head = make([]uint16, bucketSize)
69
+ h.banks = make([][]slot, h.numBanks)
70
+ for i := range h.banks {
71
+ h.banks[i] = make([]slot, bankSize)
72
+ }
73
+ h.free_slot_idx = make([]uint16, h.numBanks)
74
+ }
75
+
76
+ func (h *hashForgetfulChain) Prepare(one_shot bool, input_size uint, data []byte) {
77
+ var partial_prepare_threshold uint = (1 << h.bucketBits) >> 6
78
+ /* Partial preparation is 100 times slower (per socket). */
79
+ if one_shot && input_size <= partial_prepare_threshold {
80
+ var i uint
81
+ for i = 0; i < input_size; i++ {
82
+ var bucket uint = h.HashBytes(data[i:])
83
+
84
+ /* See InitEmpty comment. */
85
+ h.addr[bucket] = 0xCCCCCCCC
86
+
87
+ h.head[bucket] = 0xCCCC
88
+ }
89
+ } else {
90
+ /* Fill |addr| array with 0xCCCCCCCC value. Because of wrapping, position
91
+ processed by hasher never reaches 3GB + 64M; this makes all new chains
92
+ to be terminated after the first node. */
93
+ for i := range h.addr {
94
+ h.addr[i] = 0xCCCCCCCC
95
+ }
96
+
97
+ for i := range h.head {
98
+ h.head[i] = 0
99
+ }
100
+ }
101
+
102
+ h.tiny_hash = [65536]byte{}
103
+ for i := range h.free_slot_idx {
104
+ h.free_slot_idx[i] = 0
105
+ }
106
+ }
107
+
108
+ /* Look at 4 bytes at &data[ix & mask]. Compute a hash from these, and prepend
109
+ node to corresponding chain; also update tiny_hash for current position. */
110
+ func (h *hashForgetfulChain) Store(data []byte, mask uint, ix uint) {
111
+ var key uint = h.HashBytes(data[ix&mask:])
112
+ var bank uint = key & (h.numBanks - 1)
113
+ idx := uint(h.free_slot_idx[bank]) & ((1 << h.bankBits) - 1)
114
+ h.free_slot_idx[bank]++
115
+ var delta uint = ix - uint(h.addr[key])
116
+ h.tiny_hash[uint16(ix)] = byte(key)
117
+ if delta > 0xFFFF {
118
+ delta = 0xFFFF
119
+ }
120
+ h.banks[bank][idx].delta = uint16(delta)
121
+ h.banks[bank][idx].next = h.head[key]
122
+ h.addr[key] = uint32(ix)
123
+ h.head[key] = uint16(idx)
124
+ }
125
+
126
+ func (h *hashForgetfulChain) StoreRange(data []byte, mask uint, ix_start uint, ix_end uint) {
127
+ var i uint
128
+ for i = ix_start; i < ix_end; i++ {
129
+ h.Store(data, mask, i)
130
+ }
131
+ }
132
+
133
+ func (h *hashForgetfulChain) StitchToPreviousBlock(num_bytes uint, position uint, ringbuffer []byte, ring_buffer_mask uint) {
134
+ if num_bytes >= h.HashTypeLength()-1 && position >= 3 {
135
+ /* Prepare the hashes for three last bytes of the last write.
136
+ These could not be calculated before, since they require knowledge
137
+ of both the previous and the current block. */
138
+ h.Store(ringbuffer, ring_buffer_mask, position-3)
139
+ h.Store(ringbuffer, ring_buffer_mask, position-2)
140
+ h.Store(ringbuffer, ring_buffer_mask, position-1)
141
+ }
142
+ }
143
+
144
+ func (h *hashForgetfulChain) PrepareDistanceCache(distance_cache []int) {
145
+ prepareDistanceCache(distance_cache, h.numLastDistancesToCheck)
146
+ }
147
+
148
+ /* Find a longest backward match of &data[cur_ix] up to the length of
149
+ max_length and stores the position cur_ix in the hash table.
150
+
151
+ REQUIRES: PrepareDistanceCachehashForgetfulChain must be invoked for current distance cache
152
+ values; if this method is invoked repeatedly with the same distance
153
+ cache values, it is enough to invoke PrepareDistanceCachehashForgetfulChain once.
154
+
155
+ Does not look for matches longer than max_length.
156
+ Does not look for matches further away than max_backward.
157
+ Writes the best match into |out|.
158
+ |out|->score is updated only if a better match is found. */
159
+ func (h *hashForgetfulChain) FindLongestMatch(dictionary *encoderDictionary, data []byte, ring_buffer_mask uint, distance_cache []int, cur_ix uint, max_length uint, max_backward uint, gap uint, max_distance uint, out *hasherSearchResult) {
160
+ var cur_ix_masked uint = cur_ix & ring_buffer_mask
161
+ var min_score uint = out.score
162
+ var best_score uint = out.score
163
+ var best_len uint = out.len
164
+ var key uint = h.HashBytes(data[cur_ix_masked:])
165
+ var tiny_hash byte = byte(key)
166
+ /* Don't accept a short copy from far away. */
167
+ out.len = 0
168
+
169
+ out.len_code_delta = 0
170
+
171
+ /* Try last distance first. */
172
+ for i := 0; i < h.numLastDistancesToCheck; i++ {
173
+ var backward uint = uint(distance_cache[i])
174
+ var prev_ix uint = (cur_ix - backward)
175
+
176
+ /* For distance code 0 we want to consider 2-byte matches. */
177
+ if i > 0 && h.tiny_hash[uint16(prev_ix)] != tiny_hash {
178
+ continue
179
+ }
180
+ if prev_ix >= cur_ix || backward > max_backward {
181
+ continue
182
+ }
183
+
184
+ prev_ix &= ring_buffer_mask
185
+ {
186
+ var len uint = findMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length)
187
+ if len >= 2 {
188
+ var score uint = backwardReferenceScoreUsingLastDistance(uint(len))
189
+ if best_score < score {
190
+ if i != 0 {
191
+ score -= backwardReferencePenaltyUsingLastDistance(uint(i))
192
+ }
193
+ if best_score < score {
194
+ best_score = score
195
+ best_len = uint(len)
196
+ out.len = best_len
197
+ out.distance = backward
198
+ out.score = best_score
199
+ }
200
+ }
201
+ }
202
+ }
203
+ }
204
+ {
205
+ var bank uint = key & (h.numBanks - 1)
206
+ var backward uint = 0
207
+ var hops uint = h.max_hops
208
+ var delta uint = cur_ix - uint(h.addr[key])
209
+ var slot uint = uint(h.head[key])
210
+ for {
211
+ tmp6 := hops
212
+ hops--
213
+ if tmp6 == 0 {
214
+ break
215
+ }
216
+ var prev_ix uint
217
+ var last uint = slot
218
+ backward += delta
219
+ if backward > max_backward {
220
+ break
221
+ }
222
+ prev_ix = (cur_ix - backward) & ring_buffer_mask
223
+ slot = uint(h.banks[bank][last].next)
224
+ delta = uint(h.banks[bank][last].delta)
225
+ if cur_ix_masked+best_len > ring_buffer_mask || prev_ix+best_len > ring_buffer_mask || data[cur_ix_masked+best_len] != data[prev_ix+best_len] {
226
+ continue
227
+ }
228
+ {
229
+ var len uint = findMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length)
230
+ if len >= 4 {
231
+ /* Comparing for >= 3 does not change the semantics, but just saves
232
+ for a few unnecessary binary logarithms in backward reference
233
+ score, since we are not interested in such short matches. */
234
+ var score uint = backwardReferenceScore(uint(len), backward)
235
+ if best_score < score {
236
+ best_score = score
237
+ best_len = uint(len)
238
+ out.len = best_len
239
+ out.distance = backward
240
+ out.score = best_score
241
+ }
242
+ }
243
+ }
244
+ }
245
+
246
+ h.Store(data, ring_buffer_mask, cur_ix)
247
+ }
248
+
249
+ if out.score == min_score {
250
+ searchInStaticDictionary(dictionary, h, data[cur_ix_masked:], max_length, max_backward+gap, max_distance, out, false)
251
+ }
252
+ }