typenative 0.3.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (148) hide show
  1. package/CHANGELOG.md +71 -0
  2. package/README.md +17 -1
  3. package/bin/cli.js +25 -11
  4. package/bin/go/convert.js +11 -1
  5. package/bin/index.js +221 -44
  6. package/bin/main.js +35 -14
  7. package/bin/node/assert.js +2 -2
  8. package/bin/node/buffer.js +9 -0
  9. package/bin/node/dns.js +1 -1
  10. package/bin/node/events.js +1 -1
  11. package/bin/node/fs.js +1 -1
  12. package/bin/node/fs_watch.js +407 -0
  13. package/bin/node/http2.js +33 -6
  14. package/bin/node/readline.js +1 -1
  15. package/bin/node/registry.js +2 -1
  16. package/bin/node/stream.js +6 -3
  17. package/bin/node/timers.js +1 -1
  18. package/bin/node/tls.js +3 -1
  19. package/bin/node/typed_arrays.js +608 -0
  20. package/bin/node/util.js +3 -0
  21. package/bin/node/zlib.js +131 -16
  22. package/bin/transpiler.js +684 -62
  23. package/bin/version.js +1 -1
  24. package/conformance.md +16 -4
  25. package/examples/README.md +54 -0
  26. package/examples/fibonacci/README.md +89 -0
  27. package/examples/fibonacci/bench.mjs +87 -0
  28. package/examples/fibonacci/main.ts +21 -0
  29. package/examples/hello-world/README.md +62 -0
  30. package/examples/hello-world/bench.mjs +66 -0
  31. package/examples/hello-world/main.ts +7 -0
  32. package/examples/json-server/README.md +125 -0
  33. package/examples/json-server/bench.mjs +171 -0
  34. package/examples/json-server/main.ts +67 -0
  35. package/govendor/github.com/andybalholm/brotli/LICENSE +19 -0
  36. package/govendor/github.com/andybalholm/brotli/README.md +19 -0
  37. package/govendor/github.com/andybalholm/brotli/backward_references.go +185 -0
  38. package/govendor/github.com/andybalholm/brotli/backward_references_hq.go +796 -0
  39. package/govendor/github.com/andybalholm/brotli/bit_cost.go +436 -0
  40. package/govendor/github.com/andybalholm/brotli/bit_reader.go +266 -0
  41. package/govendor/github.com/andybalholm/brotli/bitwriter.go +56 -0
  42. package/govendor/github.com/andybalholm/brotli/block_splitter.go +144 -0
  43. package/govendor/github.com/andybalholm/brotli/block_splitter_command.go +434 -0
  44. package/govendor/github.com/andybalholm/brotli/block_splitter_distance.go +433 -0
  45. package/govendor/github.com/andybalholm/brotli/block_splitter_literal.go +433 -0
  46. package/govendor/github.com/andybalholm/brotli/brotli_bit_stream.go +1435 -0
  47. package/govendor/github.com/andybalholm/brotli/cluster.go +30 -0
  48. package/govendor/github.com/andybalholm/brotli/cluster_command.go +164 -0
  49. package/govendor/github.com/andybalholm/brotli/cluster_distance.go +326 -0
  50. package/govendor/github.com/andybalholm/brotli/cluster_literal.go +326 -0
  51. package/govendor/github.com/andybalholm/brotli/command.go +254 -0
  52. package/govendor/github.com/andybalholm/brotli/compress_fragment.go +834 -0
  53. package/govendor/github.com/andybalholm/brotli/compress_fragment_two_pass.go +775 -0
  54. package/govendor/github.com/andybalholm/brotli/constants.go +77 -0
  55. package/govendor/github.com/andybalholm/brotli/context.go +2176 -0
  56. package/govendor/github.com/andybalholm/brotli/decode.go +2570 -0
  57. package/govendor/github.com/andybalholm/brotli/dictionary.go +122890 -0
  58. package/govendor/github.com/andybalholm/brotli/dictionary_hash.go +32779 -0
  59. package/govendor/github.com/andybalholm/brotli/encode.go +1220 -0
  60. package/govendor/github.com/andybalholm/brotli/encoder.go +177 -0
  61. package/govendor/github.com/andybalholm/brotli/encoder_dict.go +22 -0
  62. package/govendor/github.com/andybalholm/brotli/encoder_fast.go +201 -0
  63. package/govendor/github.com/andybalholm/brotli/entropy_encode.go +592 -0
  64. package/govendor/github.com/andybalholm/brotli/entropy_encode_static.go +4399 -0
  65. package/govendor/github.com/andybalholm/brotli/fast_log.go +290 -0
  66. package/govendor/github.com/andybalholm/brotli/find_match_length.go +45 -0
  67. package/govendor/github.com/andybalholm/brotli/go.mod +3 -0
  68. package/govendor/github.com/andybalholm/brotli/h10.go +287 -0
  69. package/govendor/github.com/andybalholm/brotli/h5.go +214 -0
  70. package/govendor/github.com/andybalholm/brotli/h6.go +216 -0
  71. package/govendor/github.com/andybalholm/brotli/hash.go +342 -0
  72. package/govendor/github.com/andybalholm/brotli/hash_composite.go +93 -0
  73. package/govendor/github.com/andybalholm/brotli/hash_forgetful_chain.go +252 -0
  74. package/govendor/github.com/andybalholm/brotli/hash_longest_match_quickly.go +214 -0
  75. package/govendor/github.com/andybalholm/brotli/hash_rolling.go +168 -0
  76. package/govendor/github.com/andybalholm/brotli/histogram.go +226 -0
  77. package/govendor/github.com/andybalholm/brotli/http.go +201 -0
  78. package/govendor/github.com/andybalholm/brotli/huffman.go +653 -0
  79. package/govendor/github.com/andybalholm/brotli/literal_cost.go +182 -0
  80. package/govendor/github.com/andybalholm/brotli/matchfinder/bargain1.go +212 -0
  81. package/govendor/github.com/andybalholm/brotli/matchfinder/bargain2.go +248 -0
  82. package/govendor/github.com/andybalholm/brotli/matchfinder/bargain3.go +278 -0
  83. package/govendor/github.com/andybalholm/brotli/matchfinder/emitter.go +34 -0
  84. package/govendor/github.com/andybalholm/brotli/matchfinder/m0.go +175 -0
  85. package/govendor/github.com/andybalholm/brotli/matchfinder/m4.go +339 -0
  86. package/govendor/github.com/andybalholm/brotli/matchfinder/matchfinder.go +103 -0
  87. package/govendor/github.com/andybalholm/brotli/matchfinder/pathfinder.go +328 -0
  88. package/govendor/github.com/andybalholm/brotli/matchfinder/textencoder.go +53 -0
  89. package/govendor/github.com/andybalholm/brotli/matchfinder/trio.go +327 -0
  90. package/govendor/github.com/andybalholm/brotli/matchfinder/zdfast.go +258 -0
  91. package/govendor/github.com/andybalholm/brotli/matchfinder/zfast.go +206 -0
  92. package/govendor/github.com/andybalholm/brotli/matchfinder/zm.go +302 -0
  93. package/govendor/github.com/andybalholm/brotli/memory.go +66 -0
  94. package/govendor/github.com/andybalholm/brotli/metablock.go +574 -0
  95. package/govendor/github.com/andybalholm/brotli/metablock_command.go +165 -0
  96. package/govendor/github.com/andybalholm/brotli/metablock_distance.go +165 -0
  97. package/govendor/github.com/andybalholm/brotli/metablock_literal.go +165 -0
  98. package/govendor/github.com/andybalholm/brotli/params.go +37 -0
  99. package/govendor/github.com/andybalholm/brotli/platform.go +103 -0
  100. package/govendor/github.com/andybalholm/brotli/prefix.go +30 -0
  101. package/govendor/github.com/andybalholm/brotli/prefix_dec.go +723 -0
  102. package/govendor/github.com/andybalholm/brotli/quality.go +196 -0
  103. package/govendor/github.com/andybalholm/brotli/reader.go +127 -0
  104. package/govendor/github.com/andybalholm/brotli/ringbuffer.go +134 -0
  105. package/govendor/github.com/andybalholm/brotli/state.go +313 -0
  106. package/govendor/github.com/andybalholm/brotli/static_dict.go +662 -0
  107. package/govendor/github.com/andybalholm/brotli/static_dict_lut.go +75094 -0
  108. package/govendor/github.com/andybalholm/brotli/symbol_list.go +22 -0
  109. package/govendor/github.com/andybalholm/brotli/transform.go +641 -0
  110. package/govendor/github.com/andybalholm/brotli/utf8_util.go +70 -0
  111. package/govendor/github.com/andybalholm/brotli/util.go +7 -0
  112. package/govendor/github.com/andybalholm/brotli/write_bits.go +52 -0
  113. package/govendor/github.com/andybalholm/brotli/writer.go +162 -0
  114. package/govendor/github.com/fsnotify/fsnotify/.mailmap +2 -0
  115. package/govendor/github.com/fsnotify/fsnotify/CHANGELOG.md +651 -0
  116. package/govendor/github.com/fsnotify/fsnotify/CONTRIBUTING.md +147 -0
  117. package/govendor/github.com/fsnotify/fsnotify/LICENSE +25 -0
  118. package/govendor/github.com/fsnotify/fsnotify/README.md +214 -0
  119. package/govendor/github.com/fsnotify/fsnotify/backend_fen.go +469 -0
  120. package/govendor/github.com/fsnotify/fsnotify/backend_inotify.go +597 -0
  121. package/govendor/github.com/fsnotify/fsnotify/backend_kqueue.go +735 -0
  122. package/govendor/github.com/fsnotify/fsnotify/backend_other.go +22 -0
  123. package/govendor/github.com/fsnotify/fsnotify/backend_windows.go +702 -0
  124. package/govendor/github.com/fsnotify/fsnotify/fsnotify.go +526 -0
  125. package/govendor/github.com/fsnotify/fsnotify/go.mod +3 -0
  126. package/govendor/github.com/fsnotify/fsnotify/internal/darwin.go +20 -0
  127. package/govendor/github.com/fsnotify/fsnotify/internal/debug_darwin.go +15 -0
  128. package/govendor/github.com/fsnotify/fsnotify/internal/debug_dragonfly.go +15 -0
  129. package/govendor/github.com/fsnotify/fsnotify/internal/debug_freebsd.go +20 -0
  130. package/govendor/github.com/fsnotify/fsnotify/internal/debug_kqueue.go +32 -0
  131. package/govendor/github.com/fsnotify/fsnotify/internal/debug_linux.go +56 -0
  132. package/govendor/github.com/fsnotify/fsnotify/internal/debug_netbsd.go +15 -0
  133. package/govendor/github.com/fsnotify/fsnotify/internal/debug_openbsd.go +16 -0
  134. package/govendor/github.com/fsnotify/fsnotify/internal/debug_solaris.go +45 -0
  135. package/govendor/github.com/fsnotify/fsnotify/internal/debug_windows.go +40 -0
  136. package/govendor/github.com/fsnotify/fsnotify/internal/freebsd.go +20 -0
  137. package/govendor/github.com/fsnotify/fsnotify/internal/internal.go +2 -0
  138. package/govendor/github.com/fsnotify/fsnotify/internal/unix.go +20 -0
  139. package/govendor/github.com/fsnotify/fsnotify/internal/unix2.go +25 -0
  140. package/govendor/github.com/fsnotify/fsnotify/internal/windows.go +40 -0
  141. package/govendor/github.com/fsnotify/fsnotify/shared.go +64 -0
  142. package/govendor/github.com/fsnotify/fsnotify/staticcheck.conf +3 -0
  143. package/govendor/github.com/fsnotify/fsnotify/system_bsd.go +7 -0
  144. package/govendor/github.com/fsnotify/fsnotify/system_darwin.go +8 -0
  145. package/govendor/manifest.txt +3 -1
  146. package/package.json +1 -1
  147. package/types/typenative-npm.d.ts +72 -0
  148. package/types/typenative.d.ts +117 -1
@@ -0,0 +1,1220 @@
1
+ package brotli
2
+
3
+ import (
4
+ "io"
5
+ "math"
6
+ )
7
+
8
+ /* Copyright 2016 Google Inc. All Rights Reserved.
9
+
10
+ Distributed under MIT license.
11
+ See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
12
+ */
13
+
14
+ /** Minimal value for ::BROTLI_PARAM_LGWIN parameter. */
15
+ const minWindowBits = 10
16
+
17
+ /**
18
+ * Maximal value for ::BROTLI_PARAM_LGWIN parameter.
19
+ *
20
+ * @note equal to @c BROTLI_MAX_DISTANCE_BITS constant.
21
+ */
22
+ const maxWindowBits = 24
23
+
24
+ /**
25
+ * Maximal value for ::BROTLI_PARAM_LGWIN parameter
26
+ * in "Large Window Brotli" (32-bit).
27
+ */
28
+ const largeMaxWindowBits = 30
29
+
30
+ /** Minimal value for ::BROTLI_PARAM_LGBLOCK parameter. */
31
+ const minInputBlockBits = 16
32
+
33
+ /** Maximal value for ::BROTLI_PARAM_LGBLOCK parameter. */
34
+ const maxInputBlockBits = 24
35
+
36
+ /** Minimal value for ::BROTLI_PARAM_QUALITY parameter. */
37
+ const minQuality = 0
38
+
39
+ /** Maximal value for ::BROTLI_PARAM_QUALITY parameter. */
40
+ const maxQuality = 11
41
+
42
+ /** Options for ::BROTLI_PARAM_MODE parameter. */
43
+ const (
44
+ modeGeneric = 0
45
+ modeText = 1
46
+ modeFont = 2
47
+ )
48
+
49
+ /** Default value for ::BROTLI_PARAM_QUALITY parameter. */
50
+ const defaultQuality = 11
51
+
52
+ /** Default value for ::BROTLI_PARAM_LGWIN parameter. */
53
+ const defaultWindow = 22
54
+
55
+ /** Default value for ::BROTLI_PARAM_MODE parameter. */
56
+ const defaultMode = modeGeneric
57
+
58
+ /** Operations that can be performed by streaming encoder. */
59
+ const (
60
+ operationProcess = 0
61
+ operationFlush = 1
62
+ operationFinish = 2
63
+ operationEmitMetadata = 3
64
+ )
65
+
66
+ const (
67
+ streamProcessing = 0
68
+ streamFlushRequested = 1
69
+ streamFinished = 2
70
+ streamMetadataHead = 3
71
+ streamMetadataBody = 4
72
+ )
73
+
74
+ type Writer struct {
75
+ dst io.Writer
76
+ options WriterOptions
77
+ err error
78
+
79
+ params encoderParams
80
+ hasher_ hasherHandle
81
+ input_pos_ uint64
82
+ ringbuffer_ ringBuffer
83
+ commands []command
84
+ num_literals_ uint
85
+ last_insert_len_ uint
86
+ last_flush_pos_ uint64
87
+ last_processed_pos_ uint64
88
+ dist_cache_ [numDistanceShortCodes]int
89
+ saved_dist_cache_ [4]int
90
+ last_bytes_ uint16
91
+ last_bytes_bits_ byte
92
+ prev_byte_ byte
93
+ prev_byte2_ byte
94
+ storage []byte
95
+ small_table_ [1 << 10]int
96
+ large_table_ []int
97
+ large_table_size_ uint
98
+ cmd_depths_ [128]byte
99
+ cmd_bits_ [128]uint16
100
+ cmd_code_ [512]byte
101
+ cmd_code_numbits_ uint
102
+ command_buf_ []uint32
103
+ literal_buf_ []byte
104
+ tiny_buf_ struct {
105
+ u64 [2]uint64
106
+ u8 [16]byte
107
+ }
108
+ remaining_metadata_bytes_ uint32
109
+ stream_state_ int
110
+ is_last_block_emitted_ bool
111
+ is_initialized_ bool
112
+ }
113
+
114
+ func inputBlockSize(s *Writer) uint {
115
+ return uint(1) << uint(s.params.lgblock)
116
+ }
117
+
118
+ func unprocessedInputSize(s *Writer) uint64 {
119
+ return s.input_pos_ - s.last_processed_pos_
120
+ }
121
+
122
+ func remainingInputBlockSize(s *Writer) uint {
123
+ var delta uint64 = unprocessedInputSize(s)
124
+ var block_size uint = inputBlockSize(s)
125
+ if delta >= uint64(block_size) {
126
+ return 0
127
+ }
128
+ return block_size - uint(delta)
129
+ }
130
+
131
+ /* Wraps 64-bit input position to 32-bit ring-buffer position preserving
132
+ "not-a-first-lap" feature. */
133
+ func wrapPosition(position uint64) uint32 {
134
+ var result uint32 = uint32(position)
135
+ var gb uint64 = position >> 30
136
+ if gb > 2 {
137
+ /* Wrap every 2GiB; The first 3GB are continuous. */
138
+ result = result&((1<<30)-1) | (uint32((gb-1)&1)+1)<<30
139
+ }
140
+
141
+ return result
142
+ }
143
+
144
+ func (s *Writer) getStorage(size int) []byte {
145
+ if len(s.storage) < size {
146
+ s.storage = make([]byte, size)
147
+ }
148
+
149
+ return s.storage
150
+ }
151
+
152
+ func hashTableSize(max_table_size uint, input_size uint) uint {
153
+ var htsize uint = 256
154
+ for htsize < max_table_size && htsize < input_size {
155
+ htsize <<= 1
156
+ }
157
+
158
+ return htsize
159
+ }
160
+
161
+ func getHashTable(s *Writer, quality int, input_size uint, table_size *uint) []int {
162
+ var max_table_size uint = maxHashTableSize(quality)
163
+ var htsize uint = hashTableSize(max_table_size, input_size)
164
+ /* Use smaller hash table when input.size() is smaller, since we
165
+ fill the table, incurring O(hash table size) overhead for
166
+ compression, and if the input is short, we won't need that
167
+ many hash table entries anyway. */
168
+
169
+ var table []int
170
+ assert(max_table_size >= 256)
171
+ if quality == fastOnePassCompressionQuality {
172
+ /* Only odd shifts are supported by fast-one-pass. */
173
+ if htsize&0xAAAAA == 0 {
174
+ htsize <<= 1
175
+ }
176
+ }
177
+
178
+ if htsize <= uint(len(s.small_table_)) {
179
+ table = s.small_table_[:]
180
+ } else {
181
+ if htsize > s.large_table_size_ {
182
+ s.large_table_size_ = htsize
183
+ s.large_table_ = nil
184
+ s.large_table_ = make([]int, htsize)
185
+ }
186
+
187
+ table = s.large_table_
188
+ }
189
+
190
+ *table_size = htsize
191
+ for i := 0; i < int(htsize); i++ {
192
+ table[i] = 0
193
+ }
194
+ return table
195
+ }
196
+
197
+ func encodeWindowBits(lgwin int, large_window bool, last_bytes *uint16, last_bytes_bits *byte) {
198
+ if large_window {
199
+ *last_bytes = uint16((lgwin&0x3F)<<8 | 0x11)
200
+ *last_bytes_bits = 14
201
+ } else {
202
+ if lgwin == 16 {
203
+ *last_bytes = 0
204
+ *last_bytes_bits = 1
205
+ } else if lgwin == 17 {
206
+ *last_bytes = 1
207
+ *last_bytes_bits = 7
208
+ } else if lgwin > 17 {
209
+ *last_bytes = uint16((lgwin-17)<<1 | 0x01)
210
+ *last_bytes_bits = 4
211
+ } else {
212
+ *last_bytes = uint16((lgwin-8)<<4 | 0x01)
213
+ *last_bytes_bits = 7
214
+ }
215
+ }
216
+ }
217
+
218
+ /* Decide about the context map based on the ability of the prediction
219
+ ability of the previous byte UTF8-prefix on the next byte. The
220
+ prediction ability is calculated as Shannon entropy. Here we need
221
+ Shannon entropy instead of 'BitsEntropy' since the prefix will be
222
+ encoded with the remaining 6 bits of the following byte, and
223
+ BitsEntropy will assume that symbol to be stored alone using Huffman
224
+ coding. */
225
+
226
+ var kStaticContextMapContinuation = [64]uint32{
227
+ 1, 1, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
228
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
229
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
230
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
231
+ }
232
+ var kStaticContextMapSimpleUTF8 = [64]uint32{
233
+ 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
234
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
235
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
236
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
237
+ }
238
+
239
+ func chooseContextMap(quality int, bigram_histo []uint32, num_literal_contexts *uint, literal_context_map *[]uint32) {
240
+ var monogram_histo = [3]uint32{0}
241
+ var two_prefix_histo = [6]uint32{0}
242
+ var total uint
243
+ var i uint
244
+ var dummy uint
245
+ var entropy [4]float64
246
+ for i = 0; i < 9; i++ {
247
+ monogram_histo[i%3] += bigram_histo[i]
248
+ two_prefix_histo[i%6] += bigram_histo[i]
249
+ }
250
+
251
+ entropy[1] = shannonEntropy(monogram_histo[:], 3, &dummy)
252
+ entropy[2] = (shannonEntropy(two_prefix_histo[:], 3, &dummy) + shannonEntropy(two_prefix_histo[3:], 3, &dummy))
253
+ entropy[3] = 0
254
+ for i = 0; i < 3; i++ {
255
+ entropy[3] += shannonEntropy(bigram_histo[3*i:], 3, &dummy)
256
+ }
257
+
258
+ total = uint(monogram_histo[0] + monogram_histo[1] + monogram_histo[2])
259
+ assert(total != 0)
260
+ entropy[0] = 1.0 / float64(total)
261
+ entropy[1] *= entropy[0]
262
+ entropy[2] *= entropy[0]
263
+ entropy[3] *= entropy[0]
264
+
265
+ if quality < minQualityForHqContextModeling {
266
+ /* 3 context models is a bit slower, don't use it at lower qualities. */
267
+ entropy[3] = entropy[1] * 10
268
+ }
269
+
270
+ /* If expected savings by symbol are less than 0.2 bits, skip the
271
+ context modeling -- in exchange for faster decoding speed. */
272
+ if entropy[1]-entropy[2] < 0.2 && entropy[1]-entropy[3] < 0.2 {
273
+ *num_literal_contexts = 1
274
+ } else if entropy[2]-entropy[3] < 0.02 {
275
+ *num_literal_contexts = 2
276
+ *literal_context_map = kStaticContextMapSimpleUTF8[:]
277
+ } else {
278
+ *num_literal_contexts = 3
279
+ *literal_context_map = kStaticContextMapContinuation[:]
280
+ }
281
+ }
282
+
283
+ /* Decide if we want to use a more complex static context map containing 13
284
+ context values, based on the entropy reduction of histograms over the
285
+ first 5 bits of literals. */
286
+
287
+ var kStaticContextMapComplexUTF8 = [64]uint32{
288
+ 11, 11, 12, 12, /* 0 special */
289
+ 0, 0, 0, 0, /* 4 lf */
290
+ 1, 1, 9, 9, /* 8 space */
291
+ 2, 2, 2, 2, /* !, first after space/lf and after something else. */
292
+ 1, 1, 1, 1, /* " */
293
+ 8, 3, 3, 3, /* % */
294
+ 1, 1, 1, 1, /* ({[ */
295
+ 2, 2, 2, 2, /* }]) */
296
+ 8, 4, 4, 4, /* :; */
297
+ 8, 7, 4, 4, /* . */
298
+ 8, 0, 0, 0, /* > */
299
+ 3, 3, 3, 3, /* [0..9] */
300
+ 5, 5, 10, 5, /* [A-Z] */
301
+ 5, 5, 10, 5,
302
+ 6, 6, 6, 6, /* [a-z] */
303
+ 6, 6, 6, 6,
304
+ }
305
+
306
+ func shouldUseComplexStaticContextMap(input []byte, start_pos uint, length uint, mask uint, quality int, size_hint uint, num_literal_contexts *uint, literal_context_map *[]uint32) bool {
307
+ /* Try the more complex static context map only for long data. */
308
+ if size_hint < 1<<20 {
309
+ return false
310
+ } else {
311
+ var end_pos uint = start_pos + length
312
+ var combined_histo = [32]uint32{0}
313
+ var context_histo = [13][32]uint32{[32]uint32{0}}
314
+ var total uint32 = 0
315
+ var entropy [3]float64
316
+ var dummy uint
317
+ var i uint
318
+ var utf8_lut contextLUT = getContextLUT(contextUTF8)
319
+ /* To make entropy calculations faster and to fit on the stack, we collect
320
+ histograms over the 5 most significant bits of literals. One histogram
321
+ without context and 13 additional histograms for each context value. */
322
+ for ; start_pos+64 <= end_pos; start_pos += 4096 {
323
+ var stride_end_pos uint = start_pos + 64
324
+ var prev2 byte = input[start_pos&mask]
325
+ var prev1 byte = input[(start_pos+1)&mask]
326
+ var pos uint
327
+
328
+ /* To make the analysis of the data faster we only examine 64 byte long
329
+ strides at every 4kB intervals. */
330
+ for pos = start_pos + 2; pos < stride_end_pos; pos++ {
331
+ var literal byte = input[pos&mask]
332
+ var context byte = byte(kStaticContextMapComplexUTF8[getContext(prev1, prev2, utf8_lut)])
333
+ total++
334
+ combined_histo[literal>>3]++
335
+ context_histo[context][literal>>3]++
336
+ prev2 = prev1
337
+ prev1 = literal
338
+ }
339
+ }
340
+
341
+ entropy[1] = shannonEntropy(combined_histo[:], 32, &dummy)
342
+ entropy[2] = 0
343
+ for i = 0; i < 13; i++ {
344
+ entropy[2] += shannonEntropy(context_histo[i][0:], 32, &dummy)
345
+ }
346
+
347
+ entropy[0] = 1.0 / float64(total)
348
+ entropy[1] *= entropy[0]
349
+ entropy[2] *= entropy[0]
350
+
351
+ /* The triggering heuristics below were tuned by compressing the individual
352
+ files of the silesia corpus. If we skip this kind of context modeling
353
+ for not very well compressible input (i.e. entropy using context modeling
354
+ is 60% of maximal entropy) or if expected savings by symbol are less
355
+ than 0.2 bits, then in every case when it triggers, the final compression
356
+ ratio is improved. Note however that this heuristics might be too strict
357
+ for some cases and could be tuned further. */
358
+ if entropy[2] > 3.0 || entropy[1]-entropy[2] < 0.2 {
359
+ return false
360
+ } else {
361
+ *num_literal_contexts = 13
362
+ *literal_context_map = kStaticContextMapComplexUTF8[:]
363
+ return true
364
+ }
365
+ }
366
+ }
367
+
368
+ func decideOverLiteralContextModeling(input []byte, start_pos uint, length uint, mask uint, quality int, size_hint uint, num_literal_contexts *uint, literal_context_map *[]uint32) {
369
+ if quality < minQualityForContextModeling || length < 64 {
370
+ return
371
+ } else if shouldUseComplexStaticContextMap(input, start_pos, length, mask, quality, size_hint, num_literal_contexts, literal_context_map) {
372
+ } else /* Context map was already set, nothing else to do. */
373
+ {
374
+ var end_pos uint = start_pos + length
375
+ /* Gather bi-gram data of the UTF8 byte prefixes. To make the analysis of
376
+ UTF8 data faster we only examine 64 byte long strides at every 4kB
377
+ intervals. */
378
+
379
+ var bigram_prefix_histo = [9]uint32{0}
380
+ for ; start_pos+64 <= end_pos; start_pos += 4096 {
381
+ var lut = [4]int{0, 0, 1, 2}
382
+ var stride_end_pos uint = start_pos + 64
383
+ var prev int = lut[input[start_pos&mask]>>6] * 3
384
+ var pos uint
385
+ for pos = start_pos + 1; pos < stride_end_pos; pos++ {
386
+ var literal byte = input[pos&mask]
387
+ bigram_prefix_histo[prev+lut[literal>>6]]++
388
+ prev = lut[literal>>6] * 3
389
+ }
390
+ }
391
+
392
+ chooseContextMap(quality, bigram_prefix_histo[0:], num_literal_contexts, literal_context_map)
393
+ }
394
+ }
395
+
396
+ func shouldCompress_encode(data []byte, mask uint, last_flush_pos uint64, bytes uint, num_literals uint, num_commands uint) bool {
397
+ /* TODO: find more precise minimal block overhead. */
398
+ if bytes <= 2 {
399
+ return false
400
+ }
401
+ if num_commands < (bytes>>8)+2 {
402
+ if float64(num_literals) > 0.99*float64(bytes) {
403
+ var literal_histo = [256]uint32{0}
404
+ const kSampleRate uint32 = 13
405
+ const kMinEntropy float64 = 7.92
406
+ var bit_cost_threshold float64 = float64(bytes) * kMinEntropy / float64(kSampleRate)
407
+ var t uint = uint((uint32(bytes) + kSampleRate - 1) / kSampleRate)
408
+ var pos uint32 = uint32(last_flush_pos)
409
+ var i uint
410
+ for i = 0; i < t; i++ {
411
+ literal_histo[data[pos&uint32(mask)]]++
412
+ pos += kSampleRate
413
+ }
414
+
415
+ if bitsEntropy(literal_histo[:], 256) > bit_cost_threshold {
416
+ return false
417
+ }
418
+ }
419
+ }
420
+
421
+ return true
422
+ }
423
+
424
+ /* Chooses the literal context mode for a metablock */
425
+ func chooseContextMode(params *encoderParams, data []byte, pos uint, mask uint, length uint) int {
426
+ /* We only do the computation for the option of something else than
427
+ CONTEXT_UTF8 for the highest qualities */
428
+ if params.quality >= minQualityForHqBlockSplitting && !isMostlyUTF8(data, pos, mask, length, kMinUTF8Ratio) {
429
+ return contextSigned
430
+ }
431
+
432
+ return contextUTF8
433
+ }
434
+
435
+ func writeMetaBlockInternal(data []byte, mask uint, last_flush_pos uint64, bytes uint, is_last bool, literal_context_mode int, params *encoderParams, prev_byte byte, prev_byte2 byte, num_literals uint, commands []command, saved_dist_cache []int, dist_cache []int, storage_ix *uint, storage []byte) {
436
+ var wrapped_last_flush_pos uint32 = wrapPosition(last_flush_pos)
437
+ var last_bytes uint16
438
+ var last_bytes_bits byte
439
+ var literal_context_lut contextLUT = getContextLUT(literal_context_mode)
440
+ var block_params encoderParams = *params
441
+
442
+ if bytes == 0 {
443
+ /* Write the ISLAST and ISEMPTY bits. */
444
+ writeBits(2, 3, storage_ix, storage)
445
+
446
+ *storage_ix = (*storage_ix + 7) &^ 7
447
+ return
448
+ }
449
+
450
+ if !shouldCompress_encode(data, mask, last_flush_pos, bytes, num_literals, uint(len(commands))) {
451
+ /* Restore the distance cache, as its last update by
452
+ CreateBackwardReferences is now unused. */
453
+ copy(dist_cache, saved_dist_cache[:4])
454
+
455
+ storeUncompressedMetaBlock(is_last, data, uint(wrapped_last_flush_pos), mask, bytes, storage_ix, storage)
456
+ return
457
+ }
458
+
459
+ assert(*storage_ix <= 14)
460
+ last_bytes = uint16(storage[1])<<8 | uint16(storage[0])
461
+ last_bytes_bits = byte(*storage_ix)
462
+ if params.quality <= maxQualityForStaticEntropyCodes {
463
+ storeMetaBlockFast(data, uint(wrapped_last_flush_pos), bytes, mask, is_last, params, commands, storage_ix, storage)
464
+ } else if params.quality < minQualityForBlockSplit {
465
+ storeMetaBlockTrivial(data, uint(wrapped_last_flush_pos), bytes, mask, is_last, params, commands, storage_ix, storage)
466
+ } else {
467
+ mb := getMetaBlockSplit()
468
+ if params.quality < minQualityForHqBlockSplitting {
469
+ var num_literal_contexts uint = 1
470
+ var literal_context_map []uint32 = nil
471
+ if !params.disable_literal_context_modeling {
472
+ decideOverLiteralContextModeling(data, uint(wrapped_last_flush_pos), bytes, mask, params.quality, params.size_hint, &num_literal_contexts, &literal_context_map)
473
+ }
474
+
475
+ buildMetaBlockGreedy(data, uint(wrapped_last_flush_pos), mask, prev_byte, prev_byte2, literal_context_lut, num_literal_contexts, literal_context_map, commands, mb)
476
+ } else {
477
+ buildMetaBlock(data, uint(wrapped_last_flush_pos), mask, &block_params, prev_byte, prev_byte2, commands, literal_context_mode, mb)
478
+ }
479
+
480
+ if params.quality >= minQualityForOptimizeHistograms {
481
+ /* The number of distance symbols effectively used for distance
482
+ histograms. It might be less than distance alphabet size
483
+ for "Large Window Brotli" (32-bit). */
484
+ var num_effective_dist_codes uint32 = block_params.dist.alphabet_size
485
+ if num_effective_dist_codes > numHistogramDistanceSymbols {
486
+ num_effective_dist_codes = numHistogramDistanceSymbols
487
+ }
488
+
489
+ optimizeHistograms(num_effective_dist_codes, mb)
490
+ }
491
+
492
+ storeMetaBlock(data, uint(wrapped_last_flush_pos), bytes, mask, prev_byte, prev_byte2, is_last, &block_params, literal_context_mode, commands, mb, storage_ix, storage)
493
+ freeMetaBlockSplit(mb)
494
+ }
495
+
496
+ if bytes+4 < *storage_ix>>3 {
497
+ /* Restore the distance cache and last byte. */
498
+ copy(dist_cache, saved_dist_cache[:4])
499
+
500
+ storage[0] = byte(last_bytes)
501
+ storage[1] = byte(last_bytes >> 8)
502
+ *storage_ix = uint(last_bytes_bits)
503
+ storeUncompressedMetaBlock(is_last, data, uint(wrapped_last_flush_pos), mask, bytes, storage_ix, storage)
504
+ }
505
+ }
506
+
507
+ func chooseDistanceParams(params *encoderParams) {
508
+ var distance_postfix_bits uint32 = 0
509
+ var num_direct_distance_codes uint32 = 0
510
+
511
+ if params.quality >= minQualityForNonzeroDistanceParams {
512
+ var ndirect_msb uint32
513
+ if params.mode == modeFont {
514
+ distance_postfix_bits = 1
515
+ num_direct_distance_codes = 12
516
+ } else {
517
+ distance_postfix_bits = params.dist.distance_postfix_bits
518
+ num_direct_distance_codes = params.dist.num_direct_distance_codes
519
+ }
520
+
521
+ ndirect_msb = (num_direct_distance_codes >> distance_postfix_bits) & 0x0F
522
+ if distance_postfix_bits > maxNpostfix || num_direct_distance_codes > maxNdirect || ndirect_msb<<distance_postfix_bits != num_direct_distance_codes {
523
+ distance_postfix_bits = 0
524
+ num_direct_distance_codes = 0
525
+ }
526
+ }
527
+
528
+ initDistanceParams(params, distance_postfix_bits, num_direct_distance_codes)
529
+ }
530
+
531
+ func ensureInitialized(s *Writer) bool {
532
+ if s.is_initialized_ {
533
+ return true
534
+ }
535
+
536
+ s.last_bytes_bits_ = 0
537
+ s.last_bytes_ = 0
538
+ s.remaining_metadata_bytes_ = math.MaxUint32
539
+
540
+ sanitizeParams(&s.params)
541
+ s.params.lgblock = computeLgBlock(&s.params)
542
+ chooseDistanceParams(&s.params)
543
+
544
+ ringBufferSetup(&s.params, &s.ringbuffer_)
545
+
546
+ /* Initialize last byte with stream header. */
547
+ {
548
+ var lgwin int = int(s.params.lgwin)
549
+ if s.params.quality == fastOnePassCompressionQuality || s.params.quality == fastTwoPassCompressionQuality {
550
+ lgwin = brotli_max_int(lgwin, 18)
551
+ }
552
+
553
+ encodeWindowBits(lgwin, s.params.large_window, &s.last_bytes_, &s.last_bytes_bits_)
554
+ }
555
+
556
+ if s.params.quality == fastOnePassCompressionQuality {
557
+ s.cmd_depths_ = [128]byte{
558
+ 0, 4, 4, 5, 6, 6, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8,
559
+ 0, 0, 0, 4, 4, 4, 4, 4, 5, 5, 6, 6, 6, 6, 7, 7,
560
+ 7, 7, 10, 10, 10, 10, 10, 10, 0, 4, 4, 5, 5, 5, 6, 6,
561
+ 7, 8, 8, 9, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10,
562
+ 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
563
+ 6, 6, 6, 6, 6, 6, 5, 5, 5, 5, 5, 5, 4, 4, 4, 4,
564
+ 4, 4, 4, 5, 5, 5, 5, 5, 5, 6, 6, 7, 7, 7, 8, 10,
565
+ 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12,
566
+ }
567
+ s.cmd_bits_ = [128]uint16{
568
+ 0, 0, 8, 9, 3, 35, 7, 71,
569
+ 39, 103, 23, 47, 175, 111, 239, 31,
570
+ 0, 0, 0, 4, 12, 2, 10, 6,
571
+ 13, 29, 11, 43, 27, 59, 87, 55,
572
+ 15, 79, 319, 831, 191, 703, 447, 959,
573
+ 0, 14, 1, 25, 5, 21, 19, 51,
574
+ 119, 159, 95, 223, 479, 991, 63, 575,
575
+ 127, 639, 383, 895, 255, 767, 511, 1023,
576
+ 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
577
+ 27, 59, 7, 39, 23, 55, 30, 1, 17, 9, 25, 5, 0, 8, 4, 12,
578
+ 2, 10, 6, 21, 13, 29, 3, 19, 11, 15, 47, 31, 95, 63, 127, 255,
579
+ 767, 2815, 1791, 3839, 511, 2559, 1535, 3583, 1023, 3071, 2047, 4095,
580
+ }
581
+ s.cmd_code_ = [512]byte{
582
+ 0xff, 0x77, 0xd5, 0xbf, 0xe7, 0xde, 0xea, 0x9e, 0x51, 0x5d, 0xde, 0xc6,
583
+ 0x70, 0x57, 0xbc, 0x58, 0x58, 0x58, 0xd8, 0xd8, 0x58, 0xd5, 0xcb, 0x8c,
584
+ 0xea, 0xe0, 0xc3, 0x87, 0x1f, 0x83, 0xc1, 0x60, 0x1c, 0x67, 0xb2, 0xaa,
585
+ 0x06, 0x83, 0xc1, 0x60, 0x30, 0x18, 0xcc, 0xa1, 0xce, 0x88, 0x54, 0x94,
586
+ 0x46, 0xe1, 0xb0, 0xd0, 0x4e, 0xb2, 0xf7, 0x04, 0x00,
587
+ }
588
+ s.cmd_code_numbits_ = 448
589
+ }
590
+
591
+ s.is_initialized_ = true
592
+ return true
593
+ }
594
+
595
+ func encoderInitParams(params *encoderParams) {
596
+ params.mode = defaultMode
597
+ params.large_window = false
598
+ params.quality = defaultQuality
599
+ params.lgwin = defaultWindow
600
+ params.lgblock = 0
601
+ params.size_hint = 0
602
+ params.disable_literal_context_modeling = false
603
+ initEncoderDictionary(&params.dictionary)
604
+ params.dist.distance_postfix_bits = 0
605
+ params.dist.num_direct_distance_codes = 0
606
+ params.dist.alphabet_size = uint32(distanceAlphabetSize(0, 0, maxDistanceBits))
607
+ params.dist.max_distance = maxDistance
608
+ }
609
+
610
+ func encoderInitState(s *Writer) {
611
+ encoderInitParams(&s.params)
612
+ s.input_pos_ = 0
613
+ s.commands = s.commands[:0]
614
+ s.num_literals_ = 0
615
+ s.last_insert_len_ = 0
616
+ s.last_flush_pos_ = 0
617
+ s.last_processed_pos_ = 0
618
+ s.prev_byte_ = 0
619
+ s.prev_byte2_ = 0
620
+ if s.hasher_ != nil {
621
+ s.hasher_.Common().is_prepared_ = false
622
+ }
623
+ s.cmd_code_numbits_ = 0
624
+ s.stream_state_ = streamProcessing
625
+ s.is_last_block_emitted_ = false
626
+ s.is_initialized_ = false
627
+
628
+ ringBufferInit(&s.ringbuffer_)
629
+
630
+ /* Initialize distance cache. */
631
+ s.dist_cache_[0] = 4
632
+
633
+ s.dist_cache_[1] = 11
634
+ s.dist_cache_[2] = 15
635
+ s.dist_cache_[3] = 16
636
+
637
+ /* Save the state of the distance cache in case we need to restore it for
638
+ emitting an uncompressed block. */
639
+ copy(s.saved_dist_cache_[:], s.dist_cache_[:])
640
+ }
641
+
642
+ /*
643
+ Copies the given input data to the internal ring buffer of the compressor.
644
+ No processing of the data occurs at this time and this function can be
645
+ called multiple times before calling WriteBrotliData() to process the
646
+ accumulated input. At most input_block_size() bytes of input data can be
647
+ copied to the ring buffer, otherwise the next WriteBrotliData() will fail.
648
+ */
649
+ func copyInputToRingBuffer(s *Writer, input_size uint, input_buffer []byte) {
650
+ var ringbuffer_ *ringBuffer = &s.ringbuffer_
651
+ ringBufferWrite(input_buffer, input_size, ringbuffer_)
652
+ s.input_pos_ += uint64(input_size)
653
+
654
+ /* TL;DR: If needed, initialize 7 more bytes in the ring buffer to make the
655
+ hashing not depend on uninitialized data. This makes compression
656
+ deterministic and it prevents uninitialized memory warnings in Valgrind.
657
+ Even without erasing, the output would be valid (but nondeterministic).
658
+
659
+ Background information: The compressor stores short (at most 8 bytes)
660
+ substrings of the input already read in a hash table, and detects
661
+ repetitions by looking up such substrings in the hash table. If it
662
+ can find a substring, it checks whether the substring is really there
663
+ in the ring buffer (or it's just a hash collision). Should the hash
664
+ table become corrupt, this check makes sure that the output is
665
+ still valid, albeit the compression ratio would be bad.
666
+
667
+ The compressor populates the hash table from the ring buffer as it's
668
+ reading new bytes from the input. However, at the last few indexes of
669
+ the ring buffer, there are not enough bytes to build full-length
670
+ substrings from. Since the hash table always contains full-length
671
+ substrings, we erase with dummy zeros here to make sure that those
672
+ substrings will contain zeros at the end instead of uninitialized
673
+ data.
674
+
675
+ Please note that erasing is not necessary (because the
676
+ memory region is already initialized since he ring buffer
677
+ has a `tail' that holds a copy of the beginning,) so we
678
+ skip erasing if we have already gone around at least once in
679
+ the ring buffer.
680
+
681
+ Only clear during the first round of ring-buffer writes. On
682
+ subsequent rounds data in the ring-buffer would be affected. */
683
+ if ringbuffer_.pos_ <= ringbuffer_.mask_ {
684
+ /* This is the first time when the ring buffer is being written.
685
+ We clear 7 bytes just after the bytes that have been copied from
686
+ the input buffer.
687
+
688
+ The ring-buffer has a "tail" that holds a copy of the beginning,
689
+ but only once the ring buffer has been fully written once, i.e.,
690
+ pos <= mask. For the first time, we need to write values
691
+ in this tail (where index may be larger than mask), so that
692
+ we have exactly defined behavior and don't read uninitialized
693
+ memory. Due to performance reasons, hashing reads data using a
694
+ LOAD64, which can go 7 bytes beyond the bytes written in the
695
+ ring-buffer. */
696
+ for i := 0; i < int(7); i++ {
697
+ ringbuffer_.buffer_[ringbuffer_.pos_:][i] = 0
698
+ }
699
+ }
700
+ }
701
+
702
+ /* Marks all input as processed.
703
+ Returns true if position wrapping occurs. */
704
+ func updateLastProcessedPos(s *Writer) bool {
705
+ var wrapped_last_processed_pos uint32 = wrapPosition(s.last_processed_pos_)
706
+ var wrapped_input_pos uint32 = wrapPosition(s.input_pos_)
707
+ s.last_processed_pos_ = s.input_pos_
708
+ return wrapped_input_pos < wrapped_last_processed_pos
709
+ }
710
+
711
+ func extendLastCommand(s *Writer, bytes *uint32, wrapped_last_processed_pos *uint32) {
712
+ var last_command *command = &s.commands[len(s.commands)-1]
713
+ var data []byte = s.ringbuffer_.buffer_
714
+ var mask uint32 = s.ringbuffer_.mask_
715
+ var max_backward_distance uint64 = ((uint64(1)) << s.params.lgwin) - windowGap
716
+ var last_copy_len uint64 = uint64(last_command.copy_len_) & 0x1FFFFFF
717
+ var last_processed_pos uint64 = s.last_processed_pos_ - last_copy_len
718
+ var max_distance uint64
719
+ if last_processed_pos < max_backward_distance {
720
+ max_distance = last_processed_pos
721
+ } else {
722
+ max_distance = max_backward_distance
723
+ }
724
+ var cmd_dist uint64 = uint64(s.dist_cache_[0])
725
+ var distance_code uint32 = commandRestoreDistanceCode(last_command, &s.params.dist)
726
+ if distance_code < numDistanceShortCodes || uint64(distance_code-(numDistanceShortCodes-1)) == cmd_dist {
727
+ if cmd_dist <= max_distance {
728
+ for *bytes != 0 && data[*wrapped_last_processed_pos&mask] == data[(uint64(*wrapped_last_processed_pos)-cmd_dist)&uint64(mask)] {
729
+ last_command.copy_len_++
730
+ (*bytes)--
731
+ (*wrapped_last_processed_pos)++
732
+ }
733
+ }
734
+
735
+ /* The copy length is at most the metablock size, and thus expressible. */
736
+ getLengthCode(uint(last_command.insert_len_), uint(int(last_command.copy_len_&0x1FFFFFF)+int(last_command.copy_len_>>25)), (last_command.dist_prefix_&0x3FF == 0), &last_command.cmd_prefix_)
737
+ }
738
+ }
739
+
740
+ /*
741
+ Processes the accumulated input data and writes
742
+ the new output meta-block to s.dest, if one has been
743
+ created (otherwise the processed input data is buffered internally).
744
+ If |is_last| or |force_flush| is true, an output meta-block is
745
+ always created. However, until |is_last| is true encoder may retain up
746
+ to 7 bits of the last byte of output. To force encoder to dump the remaining
747
+ bits use WriteMetadata() to append an empty meta-data block.
748
+ Returns false if the size of the input data is larger than
749
+ input_block_size().
750
+ */
751
+ func encodeData(s *Writer, is_last bool, force_flush bool) bool {
752
+ var delta uint64 = unprocessedInputSize(s)
753
+ var bytes uint32 = uint32(delta)
754
+ var wrapped_last_processed_pos uint32 = wrapPosition(s.last_processed_pos_)
755
+ var data []byte
756
+ var mask uint32
757
+ var literal_context_mode int
758
+
759
+ data = s.ringbuffer_.buffer_
760
+ mask = s.ringbuffer_.mask_
761
+
762
+ /* Adding more blocks after "last" block is forbidden. */
763
+ if s.is_last_block_emitted_ {
764
+ return false
765
+ }
766
+ if is_last {
767
+ s.is_last_block_emitted_ = true
768
+ }
769
+
770
+ if delta > uint64(inputBlockSize(s)) {
771
+ return false
772
+ }
773
+
774
+ if s.params.quality == fastTwoPassCompressionQuality {
775
+ if s.command_buf_ == nil || cap(s.command_buf_) < int(kCompressFragmentTwoPassBlockSize) {
776
+ s.command_buf_ = make([]uint32, kCompressFragmentTwoPassBlockSize)
777
+ s.literal_buf_ = make([]byte, kCompressFragmentTwoPassBlockSize)
778
+ } else {
779
+ s.command_buf_ = s.command_buf_[:kCompressFragmentTwoPassBlockSize]
780
+ s.literal_buf_ = s.literal_buf_[:kCompressFragmentTwoPassBlockSize]
781
+ }
782
+ }
783
+
784
+ if s.params.quality == fastOnePassCompressionQuality || s.params.quality == fastTwoPassCompressionQuality {
785
+ var storage []byte
786
+ var storage_ix uint = uint(s.last_bytes_bits_)
787
+ var table_size uint
788
+ var table []int
789
+
790
+ if delta == 0 && !is_last {
791
+ /* We have no new input data and we don't have to finish the stream, so
792
+ nothing to do. */
793
+ return true
794
+ }
795
+
796
+ storage = s.getStorage(int(2*bytes + 503))
797
+ storage[0] = byte(s.last_bytes_)
798
+ storage[1] = byte(s.last_bytes_ >> 8)
799
+ table = getHashTable(s, s.params.quality, uint(bytes), &table_size)
800
+ if s.params.quality == fastOnePassCompressionQuality {
801
+ compressFragmentFast(data[wrapped_last_processed_pos&mask:], uint(bytes), is_last, table, table_size, s.cmd_depths_[:], s.cmd_bits_[:], &s.cmd_code_numbits_, s.cmd_code_[:], &storage_ix, storage)
802
+ } else {
803
+ compressFragmentTwoPass(data[wrapped_last_processed_pos&mask:], uint(bytes), is_last, s.command_buf_, s.literal_buf_, table, table_size, &storage_ix, storage)
804
+ }
805
+
806
+ s.last_bytes_ = uint16(storage[storage_ix>>3])
807
+ s.last_bytes_bits_ = byte(storage_ix & 7)
808
+ updateLastProcessedPos(s)
809
+ s.writeOutput(storage[:storage_ix>>3])
810
+ return true
811
+ }
812
+ {
813
+ /* Theoretical max number of commands is 1 per 2 bytes. */
814
+ newsize := len(s.commands) + int(bytes)/2 + 1
815
+ if newsize > cap(s.commands) {
816
+ /* Reserve a bit more memory to allow merging with a next block
817
+ without reallocation: that would impact speed. */
818
+ newsize += int(bytes/4) + 16
819
+
820
+ new_commands := make([]command, len(s.commands), newsize)
821
+ if s.commands != nil {
822
+ copy(new_commands, s.commands)
823
+ }
824
+
825
+ s.commands = new_commands
826
+ }
827
+ }
828
+
829
+ initOrStitchToPreviousBlock(&s.hasher_, data, uint(mask), &s.params, uint(wrapped_last_processed_pos), uint(bytes), is_last)
830
+
831
+ literal_context_mode = chooseContextMode(&s.params, data, uint(wrapPosition(s.last_flush_pos_)), uint(mask), uint(s.input_pos_-s.last_flush_pos_))
832
+
833
+ if len(s.commands) != 0 && s.last_insert_len_ == 0 {
834
+ extendLastCommand(s, &bytes, &wrapped_last_processed_pos)
835
+ }
836
+
837
+ if s.params.quality == zopflificationQuality {
838
+ assert(s.params.hasher.type_ == 10)
839
+ createZopfliBackwardReferences(uint(bytes), uint(wrapped_last_processed_pos), data, uint(mask), &s.params, s.hasher_.(*h10), s.dist_cache_[:], &s.last_insert_len_, &s.commands, &s.num_literals_)
840
+ } else if s.params.quality == hqZopflificationQuality {
841
+ assert(s.params.hasher.type_ == 10)
842
+ createHqZopfliBackwardReferences(uint(bytes), uint(wrapped_last_processed_pos), data, uint(mask), &s.params, s.hasher_, s.dist_cache_[:], &s.last_insert_len_, &s.commands, &s.num_literals_)
843
+ } else {
844
+ createBackwardReferences(uint(bytes), uint(wrapped_last_processed_pos), data, uint(mask), &s.params, s.hasher_, s.dist_cache_[:], &s.last_insert_len_, &s.commands, &s.num_literals_)
845
+ }
846
+ {
847
+ var max_length uint = maxMetablockSize(&s.params)
848
+ var max_literals uint = max_length / 8
849
+ max_commands := int(max_length / 8)
850
+ var processed_bytes uint = uint(s.input_pos_ - s.last_flush_pos_)
851
+ var next_input_fits_metablock bool = (processed_bytes+inputBlockSize(s) <= max_length)
852
+ var should_flush bool = (s.params.quality < minQualityForBlockSplit && s.num_literals_+uint(len(s.commands)) >= maxNumDelayedSymbols)
853
+ /* If maximal possible additional block doesn't fit metablock, flush now. */
854
+ /* TODO: Postpone decision until next block arrives? */
855
+
856
+ /* If block splitting is not used, then flush as soon as there is some
857
+ amount of commands / literals produced. */
858
+ if !is_last && !force_flush && !should_flush && next_input_fits_metablock && s.num_literals_ < max_literals && len(s.commands) < max_commands {
859
+ /* Merge with next input block. Everything will happen later. */
860
+ if updateLastProcessedPos(s) {
861
+ hasherReset(s.hasher_)
862
+ }
863
+
864
+ return true
865
+ }
866
+ }
867
+
868
+ /* Create the last insert-only command. */
869
+ if s.last_insert_len_ > 0 {
870
+ s.commands = append(s.commands, makeInsertCommand(s.last_insert_len_))
871
+ s.num_literals_ += s.last_insert_len_
872
+ s.last_insert_len_ = 0
873
+ }
874
+
875
+ if !is_last && s.input_pos_ == s.last_flush_pos_ {
876
+ /* We have no new input data and we don't have to finish the stream, so
877
+ nothing to do. */
878
+ return true
879
+ }
880
+
881
+ assert(s.input_pos_ >= s.last_flush_pos_)
882
+ assert(s.input_pos_ > s.last_flush_pos_ || is_last)
883
+ assert(s.input_pos_-s.last_flush_pos_ <= 1<<24)
884
+ {
885
+ var metablock_size uint32 = uint32(s.input_pos_ - s.last_flush_pos_)
886
+ var storage []byte = s.getStorage(int(2*metablock_size + 503))
887
+ var storage_ix uint = uint(s.last_bytes_bits_)
888
+ storage[0] = byte(s.last_bytes_)
889
+ storage[1] = byte(s.last_bytes_ >> 8)
890
+ writeMetaBlockInternal(data, uint(mask), s.last_flush_pos_, uint(metablock_size), is_last, literal_context_mode, &s.params, s.prev_byte_, s.prev_byte2_, s.num_literals_, s.commands, s.saved_dist_cache_[:], s.dist_cache_[:], &storage_ix, storage)
891
+ s.last_bytes_ = uint16(storage[storage_ix>>3])
892
+ s.last_bytes_bits_ = byte(storage_ix & 7)
893
+ s.last_flush_pos_ = s.input_pos_
894
+ if updateLastProcessedPos(s) {
895
+ hasherReset(s.hasher_)
896
+ }
897
+
898
+ if s.last_flush_pos_ > 0 {
899
+ s.prev_byte_ = data[(uint32(s.last_flush_pos_)-1)&mask]
900
+ }
901
+
902
+ if s.last_flush_pos_ > 1 {
903
+ s.prev_byte2_ = data[uint32(s.last_flush_pos_-2)&mask]
904
+ }
905
+
906
+ s.commands = s.commands[:0]
907
+ s.num_literals_ = 0
908
+
909
+ /* Save the state of the distance cache in case we need to restore it for
910
+ emitting an uncompressed block. */
911
+ copy(s.saved_dist_cache_[:], s.dist_cache_[:])
912
+
913
+ s.writeOutput(storage[:storage_ix>>3])
914
+ return true
915
+ }
916
+ }
917
+
918
+ /* Dumps remaining output bits and metadata header to |header|.
919
+ Returns number of produced bytes.
920
+ REQUIRED: |header| should be 8-byte aligned and at least 16 bytes long.
921
+ REQUIRED: |block_size| <= (1 << 24). */
922
+ func writeMetadataHeader(s *Writer, block_size uint, header []byte) uint {
923
+ storage_ix := uint(s.last_bytes_bits_)
924
+ header[0] = byte(s.last_bytes_)
925
+ header[1] = byte(s.last_bytes_ >> 8)
926
+ s.last_bytes_ = 0
927
+ s.last_bytes_bits_ = 0
928
+
929
+ writeBits(1, 0, &storage_ix, header)
930
+ writeBits(2, 3, &storage_ix, header)
931
+ writeBits(1, 0, &storage_ix, header)
932
+ if block_size == 0 {
933
+ writeBits(2, 0, &storage_ix, header)
934
+ } else {
935
+ var nbits uint32
936
+ if block_size == 1 {
937
+ nbits = 0
938
+ } else {
939
+ nbits = log2FloorNonZero(uint(uint32(block_size)-1)) + 1
940
+ }
941
+ var nbytes uint32 = (nbits + 7) / 8
942
+ writeBits(2, uint64(nbytes), &storage_ix, header)
943
+ writeBits(uint(8*nbytes), uint64(block_size)-1, &storage_ix, header)
944
+ }
945
+
946
+ return (storage_ix + 7) >> 3
947
+ }
948
+
949
+ func injectBytePaddingBlock(s *Writer) {
950
+ var seal uint32 = uint32(s.last_bytes_)
951
+ var seal_bits uint = uint(s.last_bytes_bits_)
952
+ s.last_bytes_ = 0
953
+ s.last_bytes_bits_ = 0
954
+
955
+ /* is_last = 0, data_nibbles = 11, reserved = 0, meta_nibbles = 00 */
956
+ seal |= 0x6 << seal_bits
957
+
958
+ seal_bits += 6
959
+
960
+ destination := s.tiny_buf_.u8[:]
961
+
962
+ destination[0] = byte(seal)
963
+ if seal_bits > 8 {
964
+ destination[1] = byte(seal >> 8)
965
+ }
966
+ if seal_bits > 16 {
967
+ destination[2] = byte(seal >> 16)
968
+ }
969
+ s.writeOutput(destination[:(seal_bits+7)>>3])
970
+ }
971
+
972
+ func checkFlushComplete(s *Writer) {
973
+ if s.stream_state_ == streamFlushRequested && s.err == nil {
974
+ s.stream_state_ = streamProcessing
975
+ }
976
+ }
977
+
978
+ func encoderCompressStreamFast(s *Writer, op int, available_in *uint, next_in *[]byte) bool {
979
+ var block_size_limit uint = uint(1) << s.params.lgwin
980
+ var buf_size uint = brotli_min_size_t(kCompressFragmentTwoPassBlockSize, brotli_min_size_t(*available_in, block_size_limit))
981
+ var command_buf []uint32 = nil
982
+ var literal_buf []byte = nil
983
+ if s.params.quality != fastOnePassCompressionQuality && s.params.quality != fastTwoPassCompressionQuality {
984
+ return false
985
+ }
986
+
987
+ if s.params.quality == fastTwoPassCompressionQuality {
988
+ if s.command_buf_ == nil || cap(s.command_buf_) < int(buf_size) {
989
+ s.command_buf_ = make([]uint32, buf_size)
990
+ s.literal_buf_ = make([]byte, buf_size)
991
+ } else {
992
+ s.command_buf_ = s.command_buf_[:buf_size]
993
+ s.literal_buf_ = s.literal_buf_[:buf_size]
994
+ }
995
+
996
+ command_buf = s.command_buf_
997
+ literal_buf = s.literal_buf_
998
+ }
999
+
1000
+ for {
1001
+ if s.stream_state_ == streamFlushRequested && s.last_bytes_bits_ != 0 {
1002
+ injectBytePaddingBlock(s)
1003
+ continue
1004
+ }
1005
+
1006
+ /* Compress block only when stream is not
1007
+ finished, there is no pending flush request, and there is either
1008
+ additional input or pending operation. */
1009
+ if s.stream_state_ == streamProcessing && (*available_in != 0 || op != int(operationProcess)) {
1010
+ var block_size uint = brotli_min_size_t(block_size_limit, *available_in)
1011
+ var is_last bool = (*available_in == block_size) && (op == int(operationFinish))
1012
+ var force_flush bool = (*available_in == block_size) && (op == int(operationFlush))
1013
+ var max_out_size uint = 2*block_size + 503
1014
+ var storage []byte = nil
1015
+ var storage_ix uint = uint(s.last_bytes_bits_)
1016
+ var table_size uint
1017
+ var table []int
1018
+
1019
+ if force_flush && block_size == 0 {
1020
+ s.stream_state_ = streamFlushRequested
1021
+ continue
1022
+ }
1023
+
1024
+ storage = s.getStorage(int(max_out_size))
1025
+
1026
+ storage[0] = byte(s.last_bytes_)
1027
+ storage[1] = byte(s.last_bytes_ >> 8)
1028
+ table = getHashTable(s, s.params.quality, block_size, &table_size)
1029
+
1030
+ if s.params.quality == fastOnePassCompressionQuality {
1031
+ compressFragmentFast(*next_in, block_size, is_last, table, table_size, s.cmd_depths_[:], s.cmd_bits_[:], &s.cmd_code_numbits_, s.cmd_code_[:], &storage_ix, storage)
1032
+ } else {
1033
+ compressFragmentTwoPass(*next_in, block_size, is_last, command_buf, literal_buf, table, table_size, &storage_ix, storage)
1034
+ }
1035
+
1036
+ *next_in = (*next_in)[block_size:]
1037
+ *available_in -= block_size
1038
+ var out_bytes uint = storage_ix >> 3
1039
+ s.writeOutput(storage[:out_bytes])
1040
+
1041
+ s.last_bytes_ = uint16(storage[storage_ix>>3])
1042
+ s.last_bytes_bits_ = byte(storage_ix & 7)
1043
+
1044
+ if force_flush {
1045
+ s.stream_state_ = streamFlushRequested
1046
+ }
1047
+ if is_last {
1048
+ s.stream_state_ = streamFinished
1049
+ }
1050
+ continue
1051
+ }
1052
+
1053
+ break
1054
+ }
1055
+
1056
+ checkFlushComplete(s)
1057
+ return true
1058
+ }
1059
+
1060
+ func processMetadata(s *Writer, available_in *uint, next_in *[]byte) bool {
1061
+ if *available_in > 1<<24 {
1062
+ return false
1063
+ }
1064
+
1065
+ /* Switch to metadata block workflow, if required. */
1066
+ if s.stream_state_ == streamProcessing {
1067
+ s.remaining_metadata_bytes_ = uint32(*available_in)
1068
+ s.stream_state_ = streamMetadataHead
1069
+ }
1070
+
1071
+ if s.stream_state_ != streamMetadataHead && s.stream_state_ != streamMetadataBody {
1072
+ return false
1073
+ }
1074
+
1075
+ for {
1076
+ if s.stream_state_ == streamFlushRequested && s.last_bytes_bits_ != 0 {
1077
+ injectBytePaddingBlock(s)
1078
+ continue
1079
+ }
1080
+
1081
+ if s.input_pos_ != s.last_flush_pos_ {
1082
+ var result bool = encodeData(s, false, true)
1083
+ if !result {
1084
+ return false
1085
+ }
1086
+ continue
1087
+ }
1088
+
1089
+ if s.stream_state_ == streamMetadataHead {
1090
+ n := writeMetadataHeader(s, uint(s.remaining_metadata_bytes_), s.tiny_buf_.u8[:])
1091
+ s.writeOutput(s.tiny_buf_.u8[:n])
1092
+ s.stream_state_ = streamMetadataBody
1093
+ continue
1094
+ } else {
1095
+ /* Exit workflow only when there is no more input and no more output.
1096
+ Otherwise client may continue producing empty metadata blocks. */
1097
+ if s.remaining_metadata_bytes_ == 0 {
1098
+ s.remaining_metadata_bytes_ = math.MaxUint32
1099
+ s.stream_state_ = streamProcessing
1100
+ break
1101
+ }
1102
+
1103
+ /* This guarantees progress in "TakeOutput" workflow. */
1104
+ var c uint32 = brotli_min_uint32_t(s.remaining_metadata_bytes_, 16)
1105
+ copy(s.tiny_buf_.u8[:], (*next_in)[:c])
1106
+ *next_in = (*next_in)[c:]
1107
+ *available_in -= uint(c)
1108
+ s.remaining_metadata_bytes_ -= c
1109
+ s.writeOutput(s.tiny_buf_.u8[:c])
1110
+
1111
+ continue
1112
+ }
1113
+ }
1114
+
1115
+ return true
1116
+ }
1117
+
1118
+ func updateSizeHint(s *Writer, available_in uint) {
1119
+ if s.params.size_hint == 0 {
1120
+ var delta uint64 = unprocessedInputSize(s)
1121
+ var tail uint64 = uint64(available_in)
1122
+ var limit uint32 = 1 << 30
1123
+ var total uint32
1124
+ if (delta >= uint64(limit)) || (tail >= uint64(limit)) || ((delta + tail) >= uint64(limit)) {
1125
+ total = limit
1126
+ } else {
1127
+ total = uint32(delta + tail)
1128
+ }
1129
+
1130
+ s.params.size_hint = uint(total)
1131
+ }
1132
+ }
1133
+
1134
+ func encoderCompressStream(s *Writer, op int, available_in *uint, next_in *[]byte) bool {
1135
+ if !ensureInitialized(s) {
1136
+ return false
1137
+ }
1138
+
1139
+ /* Unfinished metadata block; check requirements. */
1140
+ if s.remaining_metadata_bytes_ != math.MaxUint32 {
1141
+ if uint32(*available_in) != s.remaining_metadata_bytes_ {
1142
+ return false
1143
+ }
1144
+ if op != int(operationEmitMetadata) {
1145
+ return false
1146
+ }
1147
+ }
1148
+
1149
+ if op == int(operationEmitMetadata) {
1150
+ updateSizeHint(s, 0) /* First data metablock might be emitted here. */
1151
+ return processMetadata(s, available_in, next_in)
1152
+ }
1153
+
1154
+ if s.stream_state_ == streamMetadataHead || s.stream_state_ == streamMetadataBody {
1155
+ return false
1156
+ }
1157
+
1158
+ if s.stream_state_ != streamProcessing && *available_in != 0 {
1159
+ return false
1160
+ }
1161
+
1162
+ if s.params.quality == fastOnePassCompressionQuality || s.params.quality == fastTwoPassCompressionQuality {
1163
+ return encoderCompressStreamFast(s, op, available_in, next_in)
1164
+ }
1165
+
1166
+ for {
1167
+ var remaining_block_size uint = remainingInputBlockSize(s)
1168
+
1169
+ if remaining_block_size != 0 && *available_in != 0 {
1170
+ var copy_input_size uint = brotli_min_size_t(remaining_block_size, *available_in)
1171
+ copyInputToRingBuffer(s, copy_input_size, *next_in)
1172
+ *next_in = (*next_in)[copy_input_size:]
1173
+ *available_in -= copy_input_size
1174
+ continue
1175
+ }
1176
+
1177
+ if s.stream_state_ == streamFlushRequested && s.last_bytes_bits_ != 0 {
1178
+ injectBytePaddingBlock(s)
1179
+ continue
1180
+ }
1181
+
1182
+ /* Compress data only when stream is not
1183
+ finished and there is no pending flush request. */
1184
+ if s.stream_state_ == streamProcessing {
1185
+ if remaining_block_size == 0 || op != int(operationProcess) {
1186
+ var is_last bool = ((*available_in == 0) && op == int(operationFinish))
1187
+ var force_flush bool = ((*available_in == 0) && op == int(operationFlush))
1188
+ var result bool
1189
+ updateSizeHint(s, *available_in)
1190
+ result = encodeData(s, is_last, force_flush)
1191
+ if !result {
1192
+ return false
1193
+ }
1194
+ if force_flush {
1195
+ s.stream_state_ = streamFlushRequested
1196
+ }
1197
+ if is_last {
1198
+ s.stream_state_ = streamFinished
1199
+ }
1200
+ continue
1201
+ }
1202
+ }
1203
+
1204
+ break
1205
+ }
1206
+
1207
+ checkFlushComplete(s)
1208
+ return true
1209
+ }
1210
+
1211
+ func (w *Writer) writeOutput(data []byte) {
1212
+ if w.err != nil {
1213
+ return
1214
+ }
1215
+
1216
+ _, w.err = w.dst.Write(data)
1217
+ if w.err == nil {
1218
+ checkFlushComplete(w)
1219
+ }
1220
+ }