typenative 0.3.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (148) hide show
  1. package/CHANGELOG.md +71 -0
  2. package/README.md +17 -1
  3. package/bin/cli.js +25 -11
  4. package/bin/go/convert.js +11 -1
  5. package/bin/index.js +221 -44
  6. package/bin/main.js +35 -14
  7. package/bin/node/assert.js +2 -2
  8. package/bin/node/buffer.js +9 -0
  9. package/bin/node/dns.js +1 -1
  10. package/bin/node/events.js +1 -1
  11. package/bin/node/fs.js +1 -1
  12. package/bin/node/fs_watch.js +407 -0
  13. package/bin/node/http2.js +33 -6
  14. package/bin/node/readline.js +1 -1
  15. package/bin/node/registry.js +2 -1
  16. package/bin/node/stream.js +6 -3
  17. package/bin/node/timers.js +1 -1
  18. package/bin/node/tls.js +3 -1
  19. package/bin/node/typed_arrays.js +608 -0
  20. package/bin/node/util.js +3 -0
  21. package/bin/node/zlib.js +131 -16
  22. package/bin/transpiler.js +684 -62
  23. package/bin/version.js +1 -1
  24. package/conformance.md +16 -4
  25. package/examples/README.md +54 -0
  26. package/examples/fibonacci/README.md +89 -0
  27. package/examples/fibonacci/bench.mjs +87 -0
  28. package/examples/fibonacci/main.ts +21 -0
  29. package/examples/hello-world/README.md +62 -0
  30. package/examples/hello-world/bench.mjs +66 -0
  31. package/examples/hello-world/main.ts +7 -0
  32. package/examples/json-server/README.md +125 -0
  33. package/examples/json-server/bench.mjs +171 -0
  34. package/examples/json-server/main.ts +67 -0
  35. package/govendor/github.com/andybalholm/brotli/LICENSE +19 -0
  36. package/govendor/github.com/andybalholm/brotli/README.md +19 -0
  37. package/govendor/github.com/andybalholm/brotli/backward_references.go +185 -0
  38. package/govendor/github.com/andybalholm/brotli/backward_references_hq.go +796 -0
  39. package/govendor/github.com/andybalholm/brotli/bit_cost.go +436 -0
  40. package/govendor/github.com/andybalholm/brotli/bit_reader.go +266 -0
  41. package/govendor/github.com/andybalholm/brotli/bitwriter.go +56 -0
  42. package/govendor/github.com/andybalholm/brotli/block_splitter.go +144 -0
  43. package/govendor/github.com/andybalholm/brotli/block_splitter_command.go +434 -0
  44. package/govendor/github.com/andybalholm/brotli/block_splitter_distance.go +433 -0
  45. package/govendor/github.com/andybalholm/brotli/block_splitter_literal.go +433 -0
  46. package/govendor/github.com/andybalholm/brotli/brotli_bit_stream.go +1435 -0
  47. package/govendor/github.com/andybalholm/brotli/cluster.go +30 -0
  48. package/govendor/github.com/andybalholm/brotli/cluster_command.go +164 -0
  49. package/govendor/github.com/andybalholm/brotli/cluster_distance.go +326 -0
  50. package/govendor/github.com/andybalholm/brotli/cluster_literal.go +326 -0
  51. package/govendor/github.com/andybalholm/brotli/command.go +254 -0
  52. package/govendor/github.com/andybalholm/brotli/compress_fragment.go +834 -0
  53. package/govendor/github.com/andybalholm/brotli/compress_fragment_two_pass.go +775 -0
  54. package/govendor/github.com/andybalholm/brotli/constants.go +77 -0
  55. package/govendor/github.com/andybalholm/brotli/context.go +2176 -0
  56. package/govendor/github.com/andybalholm/brotli/decode.go +2570 -0
  57. package/govendor/github.com/andybalholm/brotli/dictionary.go +122890 -0
  58. package/govendor/github.com/andybalholm/brotli/dictionary_hash.go +32779 -0
  59. package/govendor/github.com/andybalholm/brotli/encode.go +1220 -0
  60. package/govendor/github.com/andybalholm/brotli/encoder.go +177 -0
  61. package/govendor/github.com/andybalholm/brotli/encoder_dict.go +22 -0
  62. package/govendor/github.com/andybalholm/brotli/encoder_fast.go +201 -0
  63. package/govendor/github.com/andybalholm/brotli/entropy_encode.go +592 -0
  64. package/govendor/github.com/andybalholm/brotli/entropy_encode_static.go +4399 -0
  65. package/govendor/github.com/andybalholm/brotli/fast_log.go +290 -0
  66. package/govendor/github.com/andybalholm/brotli/find_match_length.go +45 -0
  67. package/govendor/github.com/andybalholm/brotli/go.mod +3 -0
  68. package/govendor/github.com/andybalholm/brotli/h10.go +287 -0
  69. package/govendor/github.com/andybalholm/brotli/h5.go +214 -0
  70. package/govendor/github.com/andybalholm/brotli/h6.go +216 -0
  71. package/govendor/github.com/andybalholm/brotli/hash.go +342 -0
  72. package/govendor/github.com/andybalholm/brotli/hash_composite.go +93 -0
  73. package/govendor/github.com/andybalholm/brotli/hash_forgetful_chain.go +252 -0
  74. package/govendor/github.com/andybalholm/brotli/hash_longest_match_quickly.go +214 -0
  75. package/govendor/github.com/andybalholm/brotli/hash_rolling.go +168 -0
  76. package/govendor/github.com/andybalholm/brotli/histogram.go +226 -0
  77. package/govendor/github.com/andybalholm/brotli/http.go +201 -0
  78. package/govendor/github.com/andybalholm/brotli/huffman.go +653 -0
  79. package/govendor/github.com/andybalholm/brotli/literal_cost.go +182 -0
  80. package/govendor/github.com/andybalholm/brotli/matchfinder/bargain1.go +212 -0
  81. package/govendor/github.com/andybalholm/brotli/matchfinder/bargain2.go +248 -0
  82. package/govendor/github.com/andybalholm/brotli/matchfinder/bargain3.go +278 -0
  83. package/govendor/github.com/andybalholm/brotli/matchfinder/emitter.go +34 -0
  84. package/govendor/github.com/andybalholm/brotli/matchfinder/m0.go +175 -0
  85. package/govendor/github.com/andybalholm/brotli/matchfinder/m4.go +339 -0
  86. package/govendor/github.com/andybalholm/brotli/matchfinder/matchfinder.go +103 -0
  87. package/govendor/github.com/andybalholm/brotli/matchfinder/pathfinder.go +328 -0
  88. package/govendor/github.com/andybalholm/brotli/matchfinder/textencoder.go +53 -0
  89. package/govendor/github.com/andybalholm/brotli/matchfinder/trio.go +327 -0
  90. package/govendor/github.com/andybalholm/brotli/matchfinder/zdfast.go +258 -0
  91. package/govendor/github.com/andybalholm/brotli/matchfinder/zfast.go +206 -0
  92. package/govendor/github.com/andybalholm/brotli/matchfinder/zm.go +302 -0
  93. package/govendor/github.com/andybalholm/brotli/memory.go +66 -0
  94. package/govendor/github.com/andybalholm/brotli/metablock.go +574 -0
  95. package/govendor/github.com/andybalholm/brotli/metablock_command.go +165 -0
  96. package/govendor/github.com/andybalholm/brotli/metablock_distance.go +165 -0
  97. package/govendor/github.com/andybalholm/brotli/metablock_literal.go +165 -0
  98. package/govendor/github.com/andybalholm/brotli/params.go +37 -0
  99. package/govendor/github.com/andybalholm/brotli/platform.go +103 -0
  100. package/govendor/github.com/andybalholm/brotli/prefix.go +30 -0
  101. package/govendor/github.com/andybalholm/brotli/prefix_dec.go +723 -0
  102. package/govendor/github.com/andybalholm/brotli/quality.go +196 -0
  103. package/govendor/github.com/andybalholm/brotli/reader.go +127 -0
  104. package/govendor/github.com/andybalholm/brotli/ringbuffer.go +134 -0
  105. package/govendor/github.com/andybalholm/brotli/state.go +313 -0
  106. package/govendor/github.com/andybalholm/brotli/static_dict.go +662 -0
  107. package/govendor/github.com/andybalholm/brotli/static_dict_lut.go +75094 -0
  108. package/govendor/github.com/andybalholm/brotli/symbol_list.go +22 -0
  109. package/govendor/github.com/andybalholm/brotli/transform.go +641 -0
  110. package/govendor/github.com/andybalholm/brotli/utf8_util.go +70 -0
  111. package/govendor/github.com/andybalholm/brotli/util.go +7 -0
  112. package/govendor/github.com/andybalholm/brotli/write_bits.go +52 -0
  113. package/govendor/github.com/andybalholm/brotli/writer.go +162 -0
  114. package/govendor/github.com/fsnotify/fsnotify/.mailmap +2 -0
  115. package/govendor/github.com/fsnotify/fsnotify/CHANGELOG.md +651 -0
  116. package/govendor/github.com/fsnotify/fsnotify/CONTRIBUTING.md +147 -0
  117. package/govendor/github.com/fsnotify/fsnotify/LICENSE +25 -0
  118. package/govendor/github.com/fsnotify/fsnotify/README.md +214 -0
  119. package/govendor/github.com/fsnotify/fsnotify/backend_fen.go +469 -0
  120. package/govendor/github.com/fsnotify/fsnotify/backend_inotify.go +597 -0
  121. package/govendor/github.com/fsnotify/fsnotify/backend_kqueue.go +735 -0
  122. package/govendor/github.com/fsnotify/fsnotify/backend_other.go +22 -0
  123. package/govendor/github.com/fsnotify/fsnotify/backend_windows.go +702 -0
  124. package/govendor/github.com/fsnotify/fsnotify/fsnotify.go +526 -0
  125. package/govendor/github.com/fsnotify/fsnotify/go.mod +3 -0
  126. package/govendor/github.com/fsnotify/fsnotify/internal/darwin.go +20 -0
  127. package/govendor/github.com/fsnotify/fsnotify/internal/debug_darwin.go +15 -0
  128. package/govendor/github.com/fsnotify/fsnotify/internal/debug_dragonfly.go +15 -0
  129. package/govendor/github.com/fsnotify/fsnotify/internal/debug_freebsd.go +20 -0
  130. package/govendor/github.com/fsnotify/fsnotify/internal/debug_kqueue.go +32 -0
  131. package/govendor/github.com/fsnotify/fsnotify/internal/debug_linux.go +56 -0
  132. package/govendor/github.com/fsnotify/fsnotify/internal/debug_netbsd.go +15 -0
  133. package/govendor/github.com/fsnotify/fsnotify/internal/debug_openbsd.go +16 -0
  134. package/govendor/github.com/fsnotify/fsnotify/internal/debug_solaris.go +45 -0
  135. package/govendor/github.com/fsnotify/fsnotify/internal/debug_windows.go +40 -0
  136. package/govendor/github.com/fsnotify/fsnotify/internal/freebsd.go +20 -0
  137. package/govendor/github.com/fsnotify/fsnotify/internal/internal.go +2 -0
  138. package/govendor/github.com/fsnotify/fsnotify/internal/unix.go +20 -0
  139. package/govendor/github.com/fsnotify/fsnotify/internal/unix2.go +25 -0
  140. package/govendor/github.com/fsnotify/fsnotify/internal/windows.go +40 -0
  141. package/govendor/github.com/fsnotify/fsnotify/shared.go +64 -0
  142. package/govendor/github.com/fsnotify/fsnotify/staticcheck.conf +3 -0
  143. package/govendor/github.com/fsnotify/fsnotify/system_bsd.go +7 -0
  144. package/govendor/github.com/fsnotify/fsnotify/system_darwin.go +8 -0
  145. package/govendor/manifest.txt +3 -1
  146. package/package.json +1 -1
  147. package/types/typenative-npm.d.ts +72 -0
  148. package/types/typenative.d.ts +117 -1
@@ -0,0 +1,574 @@
1
+ package brotli
2
+
3
+ import (
4
+ "sync"
5
+ )
6
+
7
+ /* Copyright 2014 Google Inc. All Rights Reserved.
8
+
9
+ Distributed under MIT license.
10
+ See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
11
+ */
12
+
13
+ /* Algorithms for distributing the literals and commands of a metablock between
14
+ block types and contexts. */
15
+
16
+ type metaBlockSplit struct {
17
+ literal_split blockSplit
18
+ command_split blockSplit
19
+ distance_split blockSplit
20
+ literal_context_map []uint32
21
+ literal_context_map_size uint
22
+ distance_context_map []uint32
23
+ distance_context_map_size uint
24
+ literal_histograms []histogramLiteral
25
+ literal_histograms_size uint
26
+ command_histograms []histogramCommand
27
+ command_histograms_size uint
28
+ distance_histograms []histogramDistance
29
+ distance_histograms_size uint
30
+ }
31
+
32
+ var metaBlockPool sync.Pool
33
+
34
+ func getMetaBlockSplit() *metaBlockSplit {
35
+ mb, _ := metaBlockPool.Get().(*metaBlockSplit)
36
+
37
+ if mb == nil {
38
+ mb = &metaBlockSplit{}
39
+ } else {
40
+ initBlockSplit(&mb.literal_split)
41
+ initBlockSplit(&mb.command_split)
42
+ initBlockSplit(&mb.distance_split)
43
+ mb.literal_context_map = mb.literal_context_map[:0]
44
+ mb.literal_context_map_size = 0
45
+ mb.distance_context_map = mb.distance_context_map[:0]
46
+ mb.distance_context_map_size = 0
47
+ mb.literal_histograms = mb.literal_histograms[:0]
48
+ mb.command_histograms = mb.command_histograms[:0]
49
+ mb.distance_histograms = mb.distance_histograms[:0]
50
+ }
51
+ return mb
52
+ }
53
+
54
+ func freeMetaBlockSplit(mb *metaBlockSplit) {
55
+ metaBlockPool.Put(mb)
56
+ }
57
+
58
+ func initDistanceParams(params *encoderParams, npostfix uint32, ndirect uint32) {
59
+ var dist_params *distanceParams = &params.dist
60
+ var alphabet_size uint32
61
+ var max_distance uint32
62
+
63
+ dist_params.distance_postfix_bits = npostfix
64
+ dist_params.num_direct_distance_codes = ndirect
65
+
66
+ alphabet_size = uint32(distanceAlphabetSize(uint(npostfix), uint(ndirect), maxDistanceBits))
67
+ max_distance = ndirect + (1 << (maxDistanceBits + npostfix + 2)) - (1 << (npostfix + 2))
68
+
69
+ if params.large_window {
70
+ var bound = [maxNpostfix + 1]uint32{0, 4, 12, 28}
71
+ var postfix uint32 = 1 << npostfix
72
+ alphabet_size = uint32(distanceAlphabetSize(uint(npostfix), uint(ndirect), largeMaxDistanceBits))
73
+
74
+ /* The maximum distance is set so that no distance symbol used can encode
75
+ a distance larger than BROTLI_MAX_ALLOWED_DISTANCE with all
76
+ its extra bits set. */
77
+ if ndirect < bound[npostfix] {
78
+ max_distance = maxAllowedDistance - (bound[npostfix] - ndirect)
79
+ } else if ndirect >= bound[npostfix]+postfix {
80
+ max_distance = (3 << 29) - 4 + (ndirect - bound[npostfix])
81
+ } else {
82
+ max_distance = maxAllowedDistance
83
+ }
84
+ }
85
+
86
+ dist_params.alphabet_size = alphabet_size
87
+ dist_params.max_distance = uint(max_distance)
88
+ }
89
+
90
+ func recomputeDistancePrefixes(cmds []command, orig_params *distanceParams, new_params *distanceParams) {
91
+ if orig_params.distance_postfix_bits == new_params.distance_postfix_bits && orig_params.num_direct_distance_codes == new_params.num_direct_distance_codes {
92
+ return
93
+ }
94
+
95
+ for i := range cmds {
96
+ var cmd *command = &cmds[i]
97
+ if commandCopyLen(cmd) != 0 && cmd.cmd_prefix_ >= 128 {
98
+ prefixEncodeCopyDistance(uint(commandRestoreDistanceCode(cmd, orig_params)), uint(new_params.num_direct_distance_codes), uint(new_params.distance_postfix_bits), &cmd.dist_prefix_, &cmd.dist_extra_)
99
+ }
100
+ }
101
+ }
102
+
103
+ func computeDistanceCost(cmds []command, orig_params *distanceParams, new_params *distanceParams, cost *float64) bool {
104
+ var equal_params bool = false
105
+ var dist_prefix uint16
106
+ var dist_extra uint32
107
+ var extra_bits float64 = 0.0
108
+ var histo histogramDistance
109
+ histogramClearDistance(&histo)
110
+
111
+ if orig_params.distance_postfix_bits == new_params.distance_postfix_bits && orig_params.num_direct_distance_codes == new_params.num_direct_distance_codes {
112
+ equal_params = true
113
+ }
114
+
115
+ for i := range cmds {
116
+ cmd := &cmds[i]
117
+ if commandCopyLen(cmd) != 0 && cmd.cmd_prefix_ >= 128 {
118
+ if equal_params {
119
+ dist_prefix = cmd.dist_prefix_
120
+ } else {
121
+ var distance uint32 = commandRestoreDistanceCode(cmd, orig_params)
122
+ if distance > uint32(new_params.max_distance) {
123
+ return false
124
+ }
125
+
126
+ prefixEncodeCopyDistance(uint(distance), uint(new_params.num_direct_distance_codes), uint(new_params.distance_postfix_bits), &dist_prefix, &dist_extra)
127
+ }
128
+
129
+ histogramAddDistance(&histo, uint(dist_prefix)&0x3FF)
130
+ extra_bits += float64(dist_prefix >> 10)
131
+ }
132
+ }
133
+
134
+ *cost = populationCostDistance(&histo) + extra_bits
135
+ return true
136
+ }
137
+
138
+ var buildMetaBlock_kMaxNumberOfHistograms uint = 256
139
+
140
+ func buildMetaBlock(ringbuffer []byte, pos uint, mask uint, params *encoderParams, prev_byte byte, prev_byte2 byte, cmds []command, literal_context_mode int, mb *metaBlockSplit) {
141
+ var distance_histograms []histogramDistance
142
+ var literal_histograms []histogramLiteral
143
+ var literal_context_modes []int = nil
144
+ var literal_histograms_size uint
145
+ var distance_histograms_size uint
146
+ var i uint
147
+ var literal_context_multiplier uint = 1
148
+ var npostfix uint32
149
+ var ndirect_msb uint32 = 0
150
+ var check_orig bool = true
151
+ var best_dist_cost float64 = 1e99
152
+ var orig_params encoderParams = *params
153
+ /* Histogram ids need to fit in one byte. */
154
+
155
+ var new_params encoderParams = *params
156
+
157
+ for npostfix = 0; npostfix <= maxNpostfix; npostfix++ {
158
+ for ; ndirect_msb < 16; ndirect_msb++ {
159
+ var ndirect uint32 = ndirect_msb << npostfix
160
+ var skip bool
161
+ var dist_cost float64
162
+ initDistanceParams(&new_params, npostfix, ndirect)
163
+ if npostfix == orig_params.dist.distance_postfix_bits && ndirect == orig_params.dist.num_direct_distance_codes {
164
+ check_orig = false
165
+ }
166
+
167
+ skip = !computeDistanceCost(cmds, &orig_params.dist, &new_params.dist, &dist_cost)
168
+ if skip || (dist_cost > best_dist_cost) {
169
+ break
170
+ }
171
+
172
+ best_dist_cost = dist_cost
173
+ params.dist = new_params.dist
174
+ }
175
+
176
+ if ndirect_msb > 0 {
177
+ ndirect_msb--
178
+ }
179
+ ndirect_msb /= 2
180
+ }
181
+
182
+ if check_orig {
183
+ var dist_cost float64
184
+ computeDistanceCost(cmds, &orig_params.dist, &orig_params.dist, &dist_cost)
185
+ if dist_cost < best_dist_cost {
186
+ /* NB: currently unused; uncomment when more param tuning is added. */
187
+ /* best_dist_cost = dist_cost; */
188
+ params.dist = orig_params.dist
189
+ }
190
+ }
191
+
192
+ recomputeDistancePrefixes(cmds, &orig_params.dist, &params.dist)
193
+
194
+ splitBlock(cmds, ringbuffer, pos, mask, params, &mb.literal_split, &mb.command_split, &mb.distance_split)
195
+
196
+ if !params.disable_literal_context_modeling {
197
+ literal_context_multiplier = 1 << literalContextBits
198
+ literal_context_modes = make([]int, (mb.literal_split.num_types))
199
+ for i = 0; i < mb.literal_split.num_types; i++ {
200
+ literal_context_modes[i] = literal_context_mode
201
+ }
202
+ }
203
+
204
+ literal_histograms_size = mb.literal_split.num_types * literal_context_multiplier
205
+ literal_histograms = make([]histogramLiteral, literal_histograms_size)
206
+ clearHistogramsLiteral(literal_histograms, literal_histograms_size)
207
+
208
+ distance_histograms_size = mb.distance_split.num_types << distanceContextBits
209
+ distance_histograms = make([]histogramDistance, distance_histograms_size)
210
+ clearHistogramsDistance(distance_histograms, distance_histograms_size)
211
+
212
+ mb.command_histograms_size = mb.command_split.num_types
213
+ if cap(mb.command_histograms) < int(mb.command_histograms_size) {
214
+ mb.command_histograms = make([]histogramCommand, (mb.command_histograms_size))
215
+ } else {
216
+ mb.command_histograms = mb.command_histograms[:mb.command_histograms_size]
217
+ }
218
+ clearHistogramsCommand(mb.command_histograms, mb.command_histograms_size)
219
+
220
+ buildHistogramsWithContext(cmds, &mb.literal_split, &mb.command_split, &mb.distance_split, ringbuffer, pos, mask, prev_byte, prev_byte2, literal_context_modes, literal_histograms, mb.command_histograms, distance_histograms)
221
+ literal_context_modes = nil
222
+
223
+ mb.literal_context_map_size = mb.literal_split.num_types << literalContextBits
224
+ if cap(mb.literal_context_map) < int(mb.literal_context_map_size) {
225
+ mb.literal_context_map = make([]uint32, (mb.literal_context_map_size))
226
+ } else {
227
+ mb.literal_context_map = mb.literal_context_map[:mb.literal_context_map_size]
228
+ }
229
+
230
+ mb.literal_histograms_size = mb.literal_context_map_size
231
+ if cap(mb.literal_histograms) < int(mb.literal_histograms_size) {
232
+ mb.literal_histograms = make([]histogramLiteral, (mb.literal_histograms_size))
233
+ } else {
234
+ mb.literal_histograms = mb.literal_histograms[:mb.literal_histograms_size]
235
+ }
236
+
237
+ clusterHistogramsLiteral(literal_histograms, literal_histograms_size, buildMetaBlock_kMaxNumberOfHistograms, mb.literal_histograms, &mb.literal_histograms_size, mb.literal_context_map)
238
+ literal_histograms = nil
239
+
240
+ if params.disable_literal_context_modeling {
241
+ /* Distribute assignment to all contexts. */
242
+ for i = mb.literal_split.num_types; i != 0; {
243
+ var j uint = 0
244
+ i--
245
+ for ; j < 1<<literalContextBits; j++ {
246
+ mb.literal_context_map[(i<<literalContextBits)+j] = mb.literal_context_map[i]
247
+ }
248
+ }
249
+ }
250
+
251
+ mb.distance_context_map_size = mb.distance_split.num_types << distanceContextBits
252
+ if cap(mb.distance_context_map) < int(mb.distance_context_map_size) {
253
+ mb.distance_context_map = make([]uint32, (mb.distance_context_map_size))
254
+ } else {
255
+ mb.distance_context_map = mb.distance_context_map[:mb.distance_context_map_size]
256
+ }
257
+
258
+ mb.distance_histograms_size = mb.distance_context_map_size
259
+ if cap(mb.distance_histograms) < int(mb.distance_histograms_size) {
260
+ mb.distance_histograms = make([]histogramDistance, (mb.distance_histograms_size))
261
+ } else {
262
+ mb.distance_histograms = mb.distance_histograms[:mb.distance_histograms_size]
263
+ }
264
+
265
+ clusterHistogramsDistance(distance_histograms, mb.distance_context_map_size, buildMetaBlock_kMaxNumberOfHistograms, mb.distance_histograms, &mb.distance_histograms_size, mb.distance_context_map)
266
+ distance_histograms = nil
267
+ }
268
+
269
+ const maxStaticContexts = 13
270
+
271
+ /* Greedy block splitter for one block category (literal, command or distance).
272
+ Gathers histograms for all context buckets. */
273
+ type contextBlockSplitter struct {
274
+ alphabet_size_ uint
275
+ num_contexts_ uint
276
+ max_block_types_ uint
277
+ min_block_size_ uint
278
+ split_threshold_ float64
279
+ num_blocks_ uint
280
+ split_ *blockSplit
281
+ histograms_ []histogramLiteral
282
+ histograms_size_ *uint
283
+ target_block_size_ uint
284
+ block_size_ uint
285
+ curr_histogram_ix_ uint
286
+ last_histogram_ix_ [2]uint
287
+ last_entropy_ [2 * maxStaticContexts]float64
288
+ merge_last_count_ uint
289
+ }
290
+
291
+ func initContextBlockSplitter(self *contextBlockSplitter, alphabet_size uint, num_contexts uint, min_block_size uint, split_threshold float64, num_symbols uint, split *blockSplit, histograms *[]histogramLiteral, histograms_size *uint) {
292
+ var max_num_blocks uint = num_symbols/min_block_size + 1
293
+ var max_num_types uint
294
+ assert(num_contexts <= maxStaticContexts)
295
+
296
+ self.alphabet_size_ = alphabet_size
297
+ self.num_contexts_ = num_contexts
298
+ self.max_block_types_ = maxNumberOfBlockTypes / num_contexts
299
+ self.min_block_size_ = min_block_size
300
+ self.split_threshold_ = split_threshold
301
+ self.num_blocks_ = 0
302
+ self.split_ = split
303
+ self.histograms_size_ = histograms_size
304
+ self.target_block_size_ = min_block_size
305
+ self.block_size_ = 0
306
+ self.curr_histogram_ix_ = 0
307
+ self.merge_last_count_ = 0
308
+
309
+ /* We have to allocate one more histogram than the maximum number of block
310
+ types for the current histogram when the meta-block is too big. */
311
+ max_num_types = brotli_min_size_t(max_num_blocks, self.max_block_types_+1)
312
+
313
+ brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, max_num_blocks)
314
+ brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, max_num_blocks)
315
+ split.num_blocks = max_num_blocks
316
+ *histograms_size = max_num_types * num_contexts
317
+ if histograms == nil || cap(*histograms) < int(*histograms_size) {
318
+ *histograms = make([]histogramLiteral, (*histograms_size))
319
+ } else {
320
+ *histograms = (*histograms)[:*histograms_size]
321
+ }
322
+ self.histograms_ = *histograms
323
+
324
+ /* Clear only current histogram. */
325
+ clearHistogramsLiteral(self.histograms_[0:], num_contexts)
326
+
327
+ self.last_histogram_ix_[1] = 0
328
+ self.last_histogram_ix_[0] = self.last_histogram_ix_[1]
329
+ }
330
+
331
+ /* Does either of three things:
332
+ (1) emits the current block with a new block type;
333
+ (2) emits the current block with the type of the second last block;
334
+ (3) merges the current block with the last block. */
335
+ func contextBlockSplitterFinishBlock(self *contextBlockSplitter, is_final bool) {
336
+ var split *blockSplit = self.split_
337
+ var num_contexts uint = self.num_contexts_
338
+ var last_entropy []float64 = self.last_entropy_[:]
339
+ var histograms []histogramLiteral = self.histograms_
340
+
341
+ if self.block_size_ < self.min_block_size_ {
342
+ self.block_size_ = self.min_block_size_
343
+ }
344
+
345
+ if self.num_blocks_ == 0 {
346
+ var i uint
347
+
348
+ /* Create first block. */
349
+ split.lengths[0] = uint32(self.block_size_)
350
+
351
+ split.types[0] = 0
352
+
353
+ for i = 0; i < num_contexts; i++ {
354
+ last_entropy[i] = bitsEntropy(histograms[i].data_[:], self.alphabet_size_)
355
+ last_entropy[num_contexts+i] = last_entropy[i]
356
+ }
357
+
358
+ self.num_blocks_++
359
+ split.num_types++
360
+ self.curr_histogram_ix_ += num_contexts
361
+ if self.curr_histogram_ix_ < *self.histograms_size_ {
362
+ clearHistogramsLiteral(self.histograms_[self.curr_histogram_ix_:], self.num_contexts_)
363
+ }
364
+
365
+ self.block_size_ = 0
366
+ } else if self.block_size_ > 0 {
367
+ var entropy [maxStaticContexts]float64
368
+ var combined_histo []histogramLiteral = make([]histogramLiteral, (2 * num_contexts))
369
+ var combined_entropy [2 * maxStaticContexts]float64
370
+ var diff = [2]float64{0.0}
371
+ /* Try merging the set of histograms for the current block type with the
372
+ respective set of histograms for the last and second last block types.
373
+ Decide over the split based on the total reduction of entropy across
374
+ all contexts. */
375
+
376
+ var i uint
377
+ for i = 0; i < num_contexts; i++ {
378
+ var curr_histo_ix uint = self.curr_histogram_ix_ + i
379
+ var j uint
380
+ entropy[i] = bitsEntropy(histograms[curr_histo_ix].data_[:], self.alphabet_size_)
381
+ for j = 0; j < 2; j++ {
382
+ var jx uint = j*num_contexts + i
383
+ var last_histogram_ix uint = self.last_histogram_ix_[j] + i
384
+ combined_histo[jx] = histograms[curr_histo_ix]
385
+ histogramAddHistogramLiteral(&combined_histo[jx], &histograms[last_histogram_ix])
386
+ combined_entropy[jx] = bitsEntropy(combined_histo[jx].data_[0:], self.alphabet_size_)
387
+ diff[j] += combined_entropy[jx] - entropy[i] - last_entropy[jx]
388
+ }
389
+ }
390
+
391
+ if split.num_types < self.max_block_types_ && diff[0] > self.split_threshold_ && diff[1] > self.split_threshold_ {
392
+ /* Create new block. */
393
+ split.lengths[self.num_blocks_] = uint32(self.block_size_)
394
+
395
+ split.types[self.num_blocks_] = byte(split.num_types)
396
+ self.last_histogram_ix_[1] = self.last_histogram_ix_[0]
397
+ self.last_histogram_ix_[0] = split.num_types * num_contexts
398
+ for i = 0; i < num_contexts; i++ {
399
+ last_entropy[num_contexts+i] = last_entropy[i]
400
+ last_entropy[i] = entropy[i]
401
+ }
402
+
403
+ self.num_blocks_++
404
+ split.num_types++
405
+ self.curr_histogram_ix_ += num_contexts
406
+ if self.curr_histogram_ix_ < *self.histograms_size_ {
407
+ clearHistogramsLiteral(self.histograms_[self.curr_histogram_ix_:], self.num_contexts_)
408
+ }
409
+
410
+ self.block_size_ = 0
411
+ self.merge_last_count_ = 0
412
+ self.target_block_size_ = self.min_block_size_
413
+ } else if diff[1] < diff[0]-20.0 {
414
+ split.lengths[self.num_blocks_] = uint32(self.block_size_)
415
+ split.types[self.num_blocks_] = split.types[self.num_blocks_-2]
416
+ /* Combine this block with second last block. */
417
+
418
+ var tmp uint = self.last_histogram_ix_[0]
419
+ self.last_histogram_ix_[0] = self.last_histogram_ix_[1]
420
+ self.last_histogram_ix_[1] = tmp
421
+ for i = 0; i < num_contexts; i++ {
422
+ histograms[self.last_histogram_ix_[0]+i] = combined_histo[num_contexts+i]
423
+ last_entropy[num_contexts+i] = last_entropy[i]
424
+ last_entropy[i] = combined_entropy[num_contexts+i]
425
+ histogramClearLiteral(&histograms[self.curr_histogram_ix_+i])
426
+ }
427
+
428
+ self.num_blocks_++
429
+ self.block_size_ = 0
430
+ self.merge_last_count_ = 0
431
+ self.target_block_size_ = self.min_block_size_
432
+ } else {
433
+ /* Combine this block with last block. */
434
+ split.lengths[self.num_blocks_-1] += uint32(self.block_size_)
435
+
436
+ for i = 0; i < num_contexts; i++ {
437
+ histograms[self.last_histogram_ix_[0]+i] = combined_histo[i]
438
+ last_entropy[i] = combined_entropy[i]
439
+ if split.num_types == 1 {
440
+ last_entropy[num_contexts+i] = last_entropy[i]
441
+ }
442
+
443
+ histogramClearLiteral(&histograms[self.curr_histogram_ix_+i])
444
+ }
445
+
446
+ self.block_size_ = 0
447
+ self.merge_last_count_++
448
+ if self.merge_last_count_ > 1 {
449
+ self.target_block_size_ += self.min_block_size_
450
+ }
451
+ }
452
+
453
+ combined_histo = nil
454
+ }
455
+
456
+ if is_final {
457
+ *self.histograms_size_ = split.num_types * num_contexts
458
+ split.num_blocks = self.num_blocks_
459
+ }
460
+ }
461
+
462
+ /* Adds the next symbol to the current block type and context. When the
463
+ current block reaches the target size, decides on merging the block. */
464
+ func contextBlockSplitterAddSymbol(self *contextBlockSplitter, symbol uint, context uint) {
465
+ histogramAddLiteral(&self.histograms_[self.curr_histogram_ix_+context], symbol)
466
+ self.block_size_++
467
+ if self.block_size_ == self.target_block_size_ {
468
+ contextBlockSplitterFinishBlock(self, false) /* is_final = */
469
+ }
470
+ }
471
+
472
+ func mapStaticContexts(num_contexts uint, static_context_map []uint32, mb *metaBlockSplit) {
473
+ var i uint
474
+ mb.literal_context_map_size = mb.literal_split.num_types << literalContextBits
475
+ if cap(mb.literal_context_map) < int(mb.literal_context_map_size) {
476
+ mb.literal_context_map = make([]uint32, (mb.literal_context_map_size))
477
+ } else {
478
+ mb.literal_context_map = mb.literal_context_map[:mb.literal_context_map_size]
479
+ }
480
+
481
+ for i = 0; i < mb.literal_split.num_types; i++ {
482
+ var offset uint32 = uint32(i * num_contexts)
483
+ var j uint
484
+ for j = 0; j < 1<<literalContextBits; j++ {
485
+ mb.literal_context_map[(i<<literalContextBits)+j] = offset + static_context_map[j]
486
+ }
487
+ }
488
+ }
489
+
490
+ func buildMetaBlockGreedyInternal(ringbuffer []byte, pos uint, mask uint, prev_byte byte, prev_byte2 byte, literal_context_lut contextLUT, num_contexts uint, static_context_map []uint32, commands []command, mb *metaBlockSplit) {
491
+ var lit_blocks struct {
492
+ plain blockSplitterLiteral
493
+ ctx contextBlockSplitter
494
+ }
495
+ var cmd_blocks blockSplitterCommand
496
+ var dist_blocks blockSplitterDistance
497
+ var num_literals uint = 0
498
+ for i := range commands {
499
+ num_literals += uint(commands[i].insert_len_)
500
+ }
501
+
502
+ if num_contexts == 1 {
503
+ initBlockSplitterLiteral(&lit_blocks.plain, 256, 512, 400.0, num_literals, &mb.literal_split, &mb.literal_histograms, &mb.literal_histograms_size)
504
+ } else {
505
+ initContextBlockSplitter(&lit_blocks.ctx, 256, num_contexts, 512, 400.0, num_literals, &mb.literal_split, &mb.literal_histograms, &mb.literal_histograms_size)
506
+ }
507
+
508
+ initBlockSplitterCommand(&cmd_blocks, numCommandSymbols, 1024, 500.0, uint(len(commands)), &mb.command_split, &mb.command_histograms, &mb.command_histograms_size)
509
+ initBlockSplitterDistance(&dist_blocks, 64, 512, 100.0, uint(len(commands)), &mb.distance_split, &mb.distance_histograms, &mb.distance_histograms_size)
510
+
511
+ for _, cmd := range commands {
512
+ var j uint
513
+ blockSplitterAddSymbolCommand(&cmd_blocks, uint(cmd.cmd_prefix_))
514
+ for j = uint(cmd.insert_len_); j != 0; j-- {
515
+ var literal byte = ringbuffer[pos&mask]
516
+ if num_contexts == 1 {
517
+ blockSplitterAddSymbolLiteral(&lit_blocks.plain, uint(literal))
518
+ } else {
519
+ var context uint = uint(getContext(prev_byte, prev_byte2, literal_context_lut))
520
+ contextBlockSplitterAddSymbol(&lit_blocks.ctx, uint(literal), uint(static_context_map[context]))
521
+ }
522
+
523
+ prev_byte2 = prev_byte
524
+ prev_byte = literal
525
+ pos++
526
+ }
527
+
528
+ pos += uint(commandCopyLen(&cmd))
529
+ if commandCopyLen(&cmd) != 0 {
530
+ prev_byte2 = ringbuffer[(pos-2)&mask]
531
+ prev_byte = ringbuffer[(pos-1)&mask]
532
+ if cmd.cmd_prefix_ >= 128 {
533
+ blockSplitterAddSymbolDistance(&dist_blocks, uint(cmd.dist_prefix_)&0x3FF)
534
+ }
535
+ }
536
+ }
537
+
538
+ if num_contexts == 1 {
539
+ blockSplitterFinishBlockLiteral(&lit_blocks.plain, true) /* is_final = */
540
+ } else {
541
+ contextBlockSplitterFinishBlock(&lit_blocks.ctx, true) /* is_final = */
542
+ }
543
+
544
+ blockSplitterFinishBlockCommand(&cmd_blocks, true) /* is_final = */
545
+ blockSplitterFinishBlockDistance(&dist_blocks, true) /* is_final = */
546
+
547
+ if num_contexts > 1 {
548
+ mapStaticContexts(num_contexts, static_context_map, mb)
549
+ }
550
+ }
551
+
552
+ func buildMetaBlockGreedy(ringbuffer []byte, pos uint, mask uint, prev_byte byte, prev_byte2 byte, literal_context_lut contextLUT, num_contexts uint, static_context_map []uint32, commands []command, mb *metaBlockSplit) {
553
+ if num_contexts == 1 {
554
+ buildMetaBlockGreedyInternal(ringbuffer, pos, mask, prev_byte, prev_byte2, literal_context_lut, 1, nil, commands, mb)
555
+ } else {
556
+ buildMetaBlockGreedyInternal(ringbuffer, pos, mask, prev_byte, prev_byte2, literal_context_lut, num_contexts, static_context_map, commands, mb)
557
+ }
558
+ }
559
+
560
+ func optimizeHistograms(num_distance_codes uint32, mb *metaBlockSplit) {
561
+ var good_for_rle [numCommandSymbols]byte
562
+ var i uint
563
+ for i = 0; i < mb.literal_histograms_size; i++ {
564
+ optimizeHuffmanCountsForRLE(256, mb.literal_histograms[i].data_[:], good_for_rle[:])
565
+ }
566
+
567
+ for i = 0; i < mb.command_histograms_size; i++ {
568
+ optimizeHuffmanCountsForRLE(numCommandSymbols, mb.command_histograms[i].data_[:], good_for_rle[:])
569
+ }
570
+
571
+ for i = 0; i < mb.distance_histograms_size; i++ {
572
+ optimizeHuffmanCountsForRLE(uint(num_distance_codes), mb.distance_histograms[i].data_[:], good_for_rle[:])
573
+ }
574
+ }