typenative 0.3.1 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/ci.yml +2 -0
- package/CHANGELOG.md +258 -0
- package/CONTRIBUTION.md +1 -1
- package/README.md +17 -1
- package/bin/cli.js +26 -11
- package/bin/go/bridge.js +74 -14
- package/bin/go/convert.js +59 -1
- package/bin/go/metadata.js +13 -4
- package/bin/go/overrides.js +4 -6
- package/bin/go/stdlib.generated.js +2 -0
- package/bin/index.js +338 -47
- package/bin/main.js +36 -14
- package/bin/node/assert.js +2 -2
- package/bin/node/buffer.js +9 -0
- package/bin/node/child_process.js +51 -3
- package/bin/node/cluster.js +41 -3
- package/bin/node/core.js +1 -1
- package/bin/node/crypto.js +205 -4
- package/bin/node/dgram.js +230 -6
- package/bin/node/dns.js +471 -8
- package/bin/node/errno_tables.js +445 -0
- package/bin/node/events.js +1 -1
- package/bin/node/fs.js +22 -15
- package/bin/node/fs_watch.js +407 -0
- package/bin/node/http.js +128 -30
- package/bin/node/http2.js +37 -7
- package/bin/node/module_v8.js +142 -0
- package/bin/node/net.js +9 -6
- package/bin/node/process.js +242 -0
- package/bin/node/readline.js +272 -1
- package/bin/node/registry.js +3 -1
- package/bin/node/sqlite.js +60 -12
- package/bin/node/stream.js +145 -10
- package/bin/node/test.js +311 -19
- package/bin/node/timers.js +1 -1
- package/bin/node/typed_arrays.js +608 -0
- package/bin/node/url.js +23 -5
- package/bin/node/util.js +21 -1
- package/bin/node/util_extra.js +6 -30
- package/bin/node/worker_threads.js +79 -19
- package/bin/node/zlib.js +131 -16
- package/bin/transpiler.js +4729 -616
- package/bin/typecheck-node.js +1 -0
- package/bin/version.js +1 -1
- package/conformance.md +1030 -877
- package/examples/README.md +54 -0
- package/examples/fibonacci/README.md +89 -0
- package/examples/fibonacci/bench.mjs +87 -0
- package/examples/fibonacci/main.ts +21 -0
- package/examples/hello-world/README.md +62 -0
- package/examples/hello-world/bench.mjs +66 -0
- package/examples/hello-world/main.ts +7 -0
- package/examples/json-server/README.md +127 -0
- package/examples/json-server/bench.mjs +263 -0
- package/examples/json-server/main.ts +67 -0
- package/govendor/github.com/andybalholm/brotli/LICENSE +19 -0
- package/govendor/github.com/andybalholm/brotli/README.md +19 -0
- package/govendor/github.com/andybalholm/brotli/backward_references.go +185 -0
- package/govendor/github.com/andybalholm/brotli/backward_references_hq.go +796 -0
- package/govendor/github.com/andybalholm/brotli/bit_cost.go +436 -0
- package/govendor/github.com/andybalholm/brotli/bit_reader.go +266 -0
- package/govendor/github.com/andybalholm/brotli/bitwriter.go +56 -0
- package/govendor/github.com/andybalholm/brotli/block_splitter.go +144 -0
- package/govendor/github.com/andybalholm/brotli/block_splitter_command.go +434 -0
- package/govendor/github.com/andybalholm/brotli/block_splitter_distance.go +433 -0
- package/govendor/github.com/andybalholm/brotli/block_splitter_literal.go +433 -0
- package/govendor/github.com/andybalholm/brotli/brotli_bit_stream.go +1435 -0
- package/govendor/github.com/andybalholm/brotli/cluster.go +30 -0
- package/govendor/github.com/andybalholm/brotli/cluster_command.go +164 -0
- package/govendor/github.com/andybalholm/brotli/cluster_distance.go +326 -0
- package/govendor/github.com/andybalholm/brotli/cluster_literal.go +326 -0
- package/govendor/github.com/andybalholm/brotli/command.go +254 -0
- package/govendor/github.com/andybalholm/brotli/compress_fragment.go +834 -0
- package/govendor/github.com/andybalholm/brotli/compress_fragment_two_pass.go +775 -0
- package/govendor/github.com/andybalholm/brotli/constants.go +77 -0
- package/govendor/github.com/andybalholm/brotli/context.go +2176 -0
- package/govendor/github.com/andybalholm/brotli/decode.go +2570 -0
- package/govendor/github.com/andybalholm/brotli/dictionary.go +122890 -0
- package/govendor/github.com/andybalholm/brotli/dictionary_hash.go +32779 -0
- package/govendor/github.com/andybalholm/brotli/encode.go +1220 -0
- package/govendor/github.com/andybalholm/brotli/encoder.go +177 -0
- package/govendor/github.com/andybalholm/brotli/encoder_dict.go +22 -0
- package/govendor/github.com/andybalholm/brotli/encoder_fast.go +201 -0
- package/govendor/github.com/andybalholm/brotli/entropy_encode.go +592 -0
- package/govendor/github.com/andybalholm/brotli/entropy_encode_static.go +4399 -0
- package/govendor/github.com/andybalholm/brotli/fast_log.go +290 -0
- package/govendor/github.com/andybalholm/brotli/find_match_length.go +45 -0
- package/govendor/github.com/andybalholm/brotli/go.mod +3 -0
- package/govendor/github.com/andybalholm/brotli/h10.go +287 -0
- package/govendor/github.com/andybalholm/brotli/h5.go +214 -0
- package/govendor/github.com/andybalholm/brotli/h6.go +216 -0
- package/govendor/github.com/andybalholm/brotli/hash.go +342 -0
- package/govendor/github.com/andybalholm/brotli/hash_composite.go +93 -0
- package/govendor/github.com/andybalholm/brotli/hash_forgetful_chain.go +252 -0
- package/govendor/github.com/andybalholm/brotli/hash_longest_match_quickly.go +214 -0
- package/govendor/github.com/andybalholm/brotli/hash_rolling.go +168 -0
- package/govendor/github.com/andybalholm/brotli/histogram.go +226 -0
- package/govendor/github.com/andybalholm/brotli/http.go +201 -0
- package/govendor/github.com/andybalholm/brotli/huffman.go +653 -0
- package/govendor/github.com/andybalholm/brotli/literal_cost.go +182 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/bargain1.go +212 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/bargain2.go +248 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/bargain3.go +278 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/emitter.go +34 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/m0.go +175 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/m4.go +339 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/matchfinder.go +103 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/pathfinder.go +328 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/textencoder.go +53 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/trio.go +327 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/zdfast.go +258 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/zfast.go +206 -0
- package/govendor/github.com/andybalholm/brotli/matchfinder/zm.go +302 -0
- package/govendor/github.com/andybalholm/brotli/memory.go +66 -0
- package/govendor/github.com/andybalholm/brotli/metablock.go +574 -0
- package/govendor/github.com/andybalholm/brotli/metablock_command.go +165 -0
- package/govendor/github.com/andybalholm/brotli/metablock_distance.go +165 -0
- package/govendor/github.com/andybalholm/brotli/metablock_literal.go +165 -0
- package/govendor/github.com/andybalholm/brotli/params.go +37 -0
- package/govendor/github.com/andybalholm/brotli/platform.go +103 -0
- package/govendor/github.com/andybalholm/brotli/prefix.go +30 -0
- package/govendor/github.com/andybalholm/brotli/prefix_dec.go +723 -0
- package/govendor/github.com/andybalholm/brotli/quality.go +196 -0
- package/govendor/github.com/andybalholm/brotli/reader.go +127 -0
- package/govendor/github.com/andybalholm/brotli/ringbuffer.go +134 -0
- package/govendor/github.com/andybalholm/brotli/state.go +313 -0
- package/govendor/github.com/andybalholm/brotli/static_dict.go +662 -0
- package/govendor/github.com/andybalholm/brotli/static_dict_lut.go +75094 -0
- package/govendor/github.com/andybalholm/brotli/symbol_list.go +22 -0
- package/govendor/github.com/andybalholm/brotli/transform.go +641 -0
- package/govendor/github.com/andybalholm/brotli/utf8_util.go +70 -0
- package/govendor/github.com/andybalholm/brotli/util.go +7 -0
- package/govendor/github.com/andybalholm/brotli/write_bits.go +52 -0
- package/govendor/github.com/andybalholm/brotli/writer.go +162 -0
- package/govendor/github.com/fsnotify/fsnotify/.mailmap +2 -0
- package/govendor/github.com/fsnotify/fsnotify/CHANGELOG.md +651 -0
- package/govendor/github.com/fsnotify/fsnotify/CONTRIBUTING.md +147 -0
- package/govendor/github.com/fsnotify/fsnotify/LICENSE +25 -0
- package/govendor/github.com/fsnotify/fsnotify/README.md +214 -0
- package/govendor/github.com/fsnotify/fsnotify/backend_fen.go +469 -0
- package/govendor/github.com/fsnotify/fsnotify/backend_inotify.go +597 -0
- package/govendor/github.com/fsnotify/fsnotify/backend_kqueue.go +735 -0
- package/govendor/github.com/fsnotify/fsnotify/backend_other.go +22 -0
- package/govendor/github.com/fsnotify/fsnotify/backend_windows.go +702 -0
- package/govendor/github.com/fsnotify/fsnotify/fsnotify.go +526 -0
- package/govendor/github.com/fsnotify/fsnotify/go.mod +3 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/darwin.go +20 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_darwin.go +15 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_dragonfly.go +15 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_freebsd.go +20 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_kqueue.go +32 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_linux.go +56 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_netbsd.go +15 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_openbsd.go +16 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_solaris.go +45 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/debug_windows.go +40 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/freebsd.go +20 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/internal.go +2 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/unix.go +20 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/unix2.go +25 -0
- package/govendor/github.com/fsnotify/fsnotify/internal/windows.go +40 -0
- package/govendor/github.com/fsnotify/fsnotify/shared.go +64 -0
- package/govendor/github.com/fsnotify/fsnotify/staticcheck.conf +3 -0
- package/govendor/github.com/fsnotify/fsnotify/system_bsd.go +7 -0
- package/govendor/github.com/fsnotify/fsnotify/system_darwin.go +8 -0
- package/govendor/manifest.txt +3 -1
- package/package.json +1 -1
- package/types/typenative-go.d.ts +23 -0
- package/types/typenative-npm.d.ts +241 -5
- package/types/typenative.d.ts +302 -12
|
@@ -0,0 +1,775 @@
|
|
|
1
|
+
package brotli
|
|
2
|
+
|
|
3
|
+
import "encoding/binary"
|
|
4
|
+
|
|
5
|
+
/* Copyright 2015 Google Inc. All Rights Reserved.
|
|
6
|
+
|
|
7
|
+
Distributed under MIT license.
|
|
8
|
+
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
/* Function for fast encoding of an input fragment, independently from the input
|
|
12
|
+
history. This function uses two-pass processing: in the first pass we save
|
|
13
|
+
the found backward matches and literal bytes into a buffer, and in the
|
|
14
|
+
second pass we emit them into the bit stream using prefix codes built based
|
|
15
|
+
on the actual command and literal byte histograms. */
|
|
16
|
+
|
|
17
|
+
const kCompressFragmentTwoPassBlockSize uint = 1 << 17
|
|
18
|
+
|
|
19
|
+
func hash1(p []byte, shift uint, length uint) uint32 {
|
|
20
|
+
var h uint64 = (binary.LittleEndian.Uint64(p) << ((8 - length) * 8)) * uint64(kHashMul32)
|
|
21
|
+
return uint32(h >> shift)
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
func hashBytesAtOffset(v uint64, offset uint, shift uint, length uint) uint32 {
|
|
25
|
+
assert(offset <= 8-length)
|
|
26
|
+
{
|
|
27
|
+
var h uint64 = ((v >> (8 * offset)) << ((8 - length) * 8)) * uint64(kHashMul32)
|
|
28
|
+
return uint32(h >> shift)
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
func isMatch1(p1 []byte, p2 []byte, length uint) bool {
|
|
33
|
+
if binary.LittleEndian.Uint32(p1) != binary.LittleEndian.Uint32(p2) {
|
|
34
|
+
return false
|
|
35
|
+
}
|
|
36
|
+
if length == 4 {
|
|
37
|
+
return true
|
|
38
|
+
}
|
|
39
|
+
return p1[4] == p2[4] && p1[5] == p2[5]
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/*
|
|
43
|
+
Builds a command and distance prefix code (each 64 symbols) into "depth" and
|
|
44
|
+
|
|
45
|
+
"bits" based on "histogram" and stores it into the bit stream.
|
|
46
|
+
*/
|
|
47
|
+
func buildAndStoreCommandPrefixCode(histogram []uint32, depth []byte, bits []uint16, storage_ix *uint, storage []byte) {
|
|
48
|
+
var tree [129]huffmanTree
|
|
49
|
+
var cmd_depth = [numCommandSymbols]byte{0}
|
|
50
|
+
/* Tree size for building a tree over 64 symbols is 2 * 64 + 1. */
|
|
51
|
+
|
|
52
|
+
var cmd_bits [64]uint16
|
|
53
|
+
createHuffmanTree(histogram, 64, 15, tree[:], depth)
|
|
54
|
+
createHuffmanTree(histogram[64:], 64, 14, tree[:], depth[64:])
|
|
55
|
+
|
|
56
|
+
/* We have to jump through a few hoops here in order to compute
|
|
57
|
+
the command bits because the symbols are in a different order than in
|
|
58
|
+
the full alphabet. This looks complicated, but having the symbols
|
|
59
|
+
in this order in the command bits saves a few branches in the Emit*
|
|
60
|
+
functions. */
|
|
61
|
+
copy(cmd_depth[:], depth[24:][:24])
|
|
62
|
+
|
|
63
|
+
copy(cmd_depth[24:][:], depth[:8])
|
|
64
|
+
copy(cmd_depth[32:][:], depth[48:][:8])
|
|
65
|
+
copy(cmd_depth[40:][:], depth[8:][:8])
|
|
66
|
+
copy(cmd_depth[48:][:], depth[56:][:8])
|
|
67
|
+
copy(cmd_depth[56:][:], depth[16:][:8])
|
|
68
|
+
convertBitDepthsToSymbols(cmd_depth[:], 64, cmd_bits[:])
|
|
69
|
+
copy(bits, cmd_bits[24:][:8])
|
|
70
|
+
copy(bits[8:], cmd_bits[40:][:8])
|
|
71
|
+
copy(bits[16:], cmd_bits[56:][:8])
|
|
72
|
+
copy(bits[24:], cmd_bits[:24])
|
|
73
|
+
copy(bits[48:], cmd_bits[32:][:8])
|
|
74
|
+
copy(bits[56:], cmd_bits[48:][:8])
|
|
75
|
+
convertBitDepthsToSymbols(depth[64:], 64, bits[64:])
|
|
76
|
+
{
|
|
77
|
+
/* Create the bit length array for the full command alphabet. */
|
|
78
|
+
var i uint
|
|
79
|
+
for i := 0; i < int(64); i++ {
|
|
80
|
+
cmd_depth[i] = 0
|
|
81
|
+
} /* only 64 first values were used */
|
|
82
|
+
copy(cmd_depth[:], depth[24:][:8])
|
|
83
|
+
copy(cmd_depth[64:][:], depth[32:][:8])
|
|
84
|
+
copy(cmd_depth[128:][:], depth[40:][:8])
|
|
85
|
+
copy(cmd_depth[192:][:], depth[48:][:8])
|
|
86
|
+
copy(cmd_depth[384:][:], depth[56:][:8])
|
|
87
|
+
for i = 0; i < 8; i++ {
|
|
88
|
+
cmd_depth[128+8*i] = depth[i]
|
|
89
|
+
cmd_depth[256+8*i] = depth[8+i]
|
|
90
|
+
cmd_depth[448+8*i] = depth[16+i]
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
storeHuffmanTree(cmd_depth[:], numCommandSymbols, tree[:], storage_ix, storage)
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
storeHuffmanTree(depth[64:], 64, tree[:], storage_ix, storage)
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
func emitInsertLen(insertlen uint32, commands *[]uint32) {
|
|
100
|
+
if insertlen < 6 {
|
|
101
|
+
(*commands)[0] = insertlen
|
|
102
|
+
} else if insertlen < 130 {
|
|
103
|
+
var tail uint32 = insertlen - 2
|
|
104
|
+
var nbits uint32 = log2FloorNonZero(uint(tail)) - 1
|
|
105
|
+
var prefix uint32 = tail >> nbits
|
|
106
|
+
var inscode uint32 = (nbits << 1) + prefix + 2
|
|
107
|
+
var extra uint32 = tail - (prefix << nbits)
|
|
108
|
+
(*commands)[0] = inscode | extra<<8
|
|
109
|
+
} else if insertlen < 2114 {
|
|
110
|
+
var tail uint32 = insertlen - 66
|
|
111
|
+
var nbits uint32 = log2FloorNonZero(uint(tail))
|
|
112
|
+
var code uint32 = nbits + 10
|
|
113
|
+
var extra uint32 = tail - (1 << nbits)
|
|
114
|
+
(*commands)[0] = code | extra<<8
|
|
115
|
+
} else if insertlen < 6210 {
|
|
116
|
+
var extra uint32 = insertlen - 2114
|
|
117
|
+
(*commands)[0] = 21 | extra<<8
|
|
118
|
+
} else if insertlen < 22594 {
|
|
119
|
+
var extra uint32 = insertlen - 6210
|
|
120
|
+
(*commands)[0] = 22 | extra<<8
|
|
121
|
+
} else {
|
|
122
|
+
var extra uint32 = insertlen - 22594
|
|
123
|
+
(*commands)[0] = 23 | extra<<8
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
*commands = (*commands)[1:]
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
func emitCopyLen(copylen uint, commands *[]uint32) {
|
|
130
|
+
if copylen < 10 {
|
|
131
|
+
(*commands)[0] = uint32(copylen + 38)
|
|
132
|
+
} else if copylen < 134 {
|
|
133
|
+
var tail uint = copylen - 6
|
|
134
|
+
var nbits uint = uint(log2FloorNonZero(tail) - 1)
|
|
135
|
+
var prefix uint = tail >> nbits
|
|
136
|
+
var code uint = (nbits << 1) + prefix + 44
|
|
137
|
+
var extra uint = tail - (prefix << nbits)
|
|
138
|
+
(*commands)[0] = uint32(code | extra<<8)
|
|
139
|
+
} else if copylen < 2118 {
|
|
140
|
+
var tail uint = copylen - 70
|
|
141
|
+
var nbits uint = uint(log2FloorNonZero(tail))
|
|
142
|
+
var code uint = nbits + 52
|
|
143
|
+
var extra uint = tail - (uint(1) << nbits)
|
|
144
|
+
(*commands)[0] = uint32(code | extra<<8)
|
|
145
|
+
} else {
|
|
146
|
+
var extra uint = copylen - 2118
|
|
147
|
+
(*commands)[0] = uint32(63 | extra<<8)
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
*commands = (*commands)[1:]
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
func emitCopyLenLastDistance(copylen uint, commands *[]uint32) {
|
|
154
|
+
if copylen < 12 {
|
|
155
|
+
(*commands)[0] = uint32(copylen + 20)
|
|
156
|
+
*commands = (*commands)[1:]
|
|
157
|
+
} else if copylen < 72 {
|
|
158
|
+
var tail uint = copylen - 8
|
|
159
|
+
var nbits uint = uint(log2FloorNonZero(tail) - 1)
|
|
160
|
+
var prefix uint = tail >> nbits
|
|
161
|
+
var code uint = (nbits << 1) + prefix + 28
|
|
162
|
+
var extra uint = tail - (prefix << nbits)
|
|
163
|
+
(*commands)[0] = uint32(code | extra<<8)
|
|
164
|
+
*commands = (*commands)[1:]
|
|
165
|
+
} else if copylen < 136 {
|
|
166
|
+
var tail uint = copylen - 8
|
|
167
|
+
var code uint = (tail >> 5) + 54
|
|
168
|
+
var extra uint = tail & 31
|
|
169
|
+
(*commands)[0] = uint32(code | extra<<8)
|
|
170
|
+
*commands = (*commands)[1:]
|
|
171
|
+
(*commands)[0] = 64
|
|
172
|
+
*commands = (*commands)[1:]
|
|
173
|
+
} else if copylen < 2120 {
|
|
174
|
+
var tail uint = copylen - 72
|
|
175
|
+
var nbits uint = uint(log2FloorNonZero(tail))
|
|
176
|
+
var code uint = nbits + 52
|
|
177
|
+
var extra uint = tail - (uint(1) << nbits)
|
|
178
|
+
(*commands)[0] = uint32(code | extra<<8)
|
|
179
|
+
*commands = (*commands)[1:]
|
|
180
|
+
(*commands)[0] = 64
|
|
181
|
+
*commands = (*commands)[1:]
|
|
182
|
+
} else {
|
|
183
|
+
var extra uint = copylen - 2120
|
|
184
|
+
(*commands)[0] = uint32(63 | extra<<8)
|
|
185
|
+
*commands = (*commands)[1:]
|
|
186
|
+
(*commands)[0] = 64
|
|
187
|
+
*commands = (*commands)[1:]
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
func emitDistance(distance uint32, commands *[]uint32) {
|
|
192
|
+
var d uint32 = distance + 3
|
|
193
|
+
var nbits uint32 = log2FloorNonZero(uint(d)) - 1
|
|
194
|
+
var prefix uint32 = (d >> nbits) & 1
|
|
195
|
+
var offset uint32 = (2 + prefix) << nbits
|
|
196
|
+
var distcode uint32 = 2*(nbits-1) + prefix + 80
|
|
197
|
+
var extra uint32 = d - offset
|
|
198
|
+
(*commands)[0] = distcode | extra<<8
|
|
199
|
+
*commands = (*commands)[1:]
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
/* REQUIRES: len <= 1 << 24. */
|
|
203
|
+
func storeMetaBlockHeader(len uint, is_uncompressed bool, storage_ix *uint, storage []byte) {
|
|
204
|
+
var nibbles uint = 6
|
|
205
|
+
|
|
206
|
+
/* ISLAST */
|
|
207
|
+
writeBits(1, 0, storage_ix, storage)
|
|
208
|
+
|
|
209
|
+
if len <= 1<<16 {
|
|
210
|
+
nibbles = 4
|
|
211
|
+
} else if len <= 1<<20 {
|
|
212
|
+
nibbles = 5
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
writeBits(2, uint64(nibbles)-4, storage_ix, storage)
|
|
216
|
+
writeBits(nibbles*4, uint64(len)-1, storage_ix, storage)
|
|
217
|
+
|
|
218
|
+
/* ISUNCOMPRESSED */
|
|
219
|
+
writeSingleBit(is_uncompressed, storage_ix, storage)
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
func storeMetaBlockHeaderBW(len uint, is_uncompressed bool, bw *bitWriter) {
|
|
223
|
+
var nibbles uint = 6
|
|
224
|
+
|
|
225
|
+
/* ISLAST */
|
|
226
|
+
bw.writeBits(1, 0)
|
|
227
|
+
|
|
228
|
+
if len <= 1<<16 {
|
|
229
|
+
nibbles = 4
|
|
230
|
+
} else if len <= 1<<20 {
|
|
231
|
+
nibbles = 5
|
|
232
|
+
} else if len > 1<<24 {
|
|
233
|
+
panic("metablock too long")
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
bw.writeBits(2, uint64(nibbles)-4)
|
|
237
|
+
bw.writeBits(nibbles*4, uint64(len)-1)
|
|
238
|
+
|
|
239
|
+
/* ISUNCOMPRESSED */
|
|
240
|
+
bw.writeSingleBit(is_uncompressed)
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
func createCommands(input []byte, block_size uint, input_size uint, base_ip_ptr []byte, table []int, table_bits uint, min_match uint, literals *[]byte, commands *[]uint32) {
|
|
244
|
+
var ip int = 0
|
|
245
|
+
var shift uint = 64 - table_bits
|
|
246
|
+
var ip_end int = int(block_size)
|
|
247
|
+
var base_ip int = -cap(base_ip_ptr) + cap(input)
|
|
248
|
+
var next_emit int = 0
|
|
249
|
+
var last_distance int = -1
|
|
250
|
+
/* "ip" is the input pointer. */
|
|
251
|
+
|
|
252
|
+
const kInputMarginBytes uint = windowGap
|
|
253
|
+
|
|
254
|
+
/* "next_emit" is a pointer to the first byte that is not covered by a
|
|
255
|
+
previous copy. Bytes between "next_emit" and the start of the next copy or
|
|
256
|
+
the end of the input will be emitted as literal bytes. */
|
|
257
|
+
if block_size >= kInputMarginBytes {
|
|
258
|
+
var len_limit uint = brotli_min_size_t(block_size-min_match, input_size-kInputMarginBytes)
|
|
259
|
+
var ip_limit int = int(len_limit)
|
|
260
|
+
/* For the last block, we need to keep a 16 bytes margin so that we can be
|
|
261
|
+
sure that all distances are at most window size - 16.
|
|
262
|
+
For all other blocks, we only need to keep a margin of 5 bytes so that
|
|
263
|
+
we don't go over the block size with a copy. */
|
|
264
|
+
|
|
265
|
+
var next_hash uint32
|
|
266
|
+
ip++
|
|
267
|
+
for next_hash = hash1(input[ip:], shift, min_match); ; {
|
|
268
|
+
var skip uint32 = 32
|
|
269
|
+
var next_ip int = ip
|
|
270
|
+
/* Step 1: Scan forward in the input looking for a 6-byte-long match.
|
|
271
|
+
If we get close to exhausting the input then goto emit_remainder.
|
|
272
|
+
|
|
273
|
+
Heuristic match skipping: If 32 bytes are scanned with no matches
|
|
274
|
+
found, start looking only at every other byte. If 32 more bytes are
|
|
275
|
+
scanned, look at every third byte, etc.. When a match is found,
|
|
276
|
+
immediately go back to looking at every byte. This is a small loss
|
|
277
|
+
(~5% performance, ~0.1% density) for compressible data due to more
|
|
278
|
+
bookkeeping, but for non-compressible data (such as JPEG) it's a huge
|
|
279
|
+
win since the compressor quickly "realizes" the data is incompressible
|
|
280
|
+
and doesn't bother looking for matches everywhere.
|
|
281
|
+
|
|
282
|
+
The "skip" variable keeps track of how many bytes there are since the
|
|
283
|
+
last match; dividing it by 32 (ie. right-shifting by five) gives the
|
|
284
|
+
number of bytes to move ahead for each iteration. */
|
|
285
|
+
|
|
286
|
+
var candidate int
|
|
287
|
+
|
|
288
|
+
assert(next_emit < ip)
|
|
289
|
+
|
|
290
|
+
trawl:
|
|
291
|
+
for {
|
|
292
|
+
var hash uint32 = next_hash
|
|
293
|
+
var bytes_between_hash_lookups uint32 = skip >> 5
|
|
294
|
+
skip++
|
|
295
|
+
ip = next_ip
|
|
296
|
+
assert(hash == hash1(input[ip:], shift, min_match))
|
|
297
|
+
next_ip = int(uint32(ip) + bytes_between_hash_lookups)
|
|
298
|
+
if next_ip > ip_limit {
|
|
299
|
+
goto emit_remainder
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
next_hash = hash1(input[next_ip:], shift, min_match)
|
|
303
|
+
candidate = ip - last_distance
|
|
304
|
+
if isMatch1(input[ip:], base_ip_ptr[candidate-base_ip:], min_match) {
|
|
305
|
+
if candidate < ip {
|
|
306
|
+
table[hash] = int(ip - base_ip)
|
|
307
|
+
break
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
candidate = base_ip + table[hash]
|
|
312
|
+
assert(candidate >= base_ip)
|
|
313
|
+
assert(candidate < ip)
|
|
314
|
+
|
|
315
|
+
table[hash] = int(ip - base_ip)
|
|
316
|
+
if isMatch1(input[ip:], base_ip_ptr[candidate-base_ip:], min_match) {
|
|
317
|
+
break
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
/* Check copy distance. If candidate is not feasible, continue search.
|
|
322
|
+
Checking is done outside of hot loop to reduce overhead. */
|
|
323
|
+
if ip-candidate > maxDistance_compress_fragment {
|
|
324
|
+
goto trawl
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
/* Step 2: Emit the found match together with the literal bytes from
|
|
328
|
+
"next_emit", and then see if we can find a next match immediately
|
|
329
|
+
afterwards. Repeat until we find no match for the input
|
|
330
|
+
without emitting some literal bytes. */
|
|
331
|
+
{
|
|
332
|
+
var base int = ip
|
|
333
|
+
/* > 0 */
|
|
334
|
+
var matched uint = min_match + findMatchLengthWithLimit(base_ip_ptr[uint(candidate-base_ip)+min_match:], input[uint(ip)+min_match:], uint(ip_end-ip)-min_match)
|
|
335
|
+
var distance int = int(base - candidate)
|
|
336
|
+
/* We have a 6-byte match at ip, and we need to emit bytes in
|
|
337
|
+
[next_emit, ip). */
|
|
338
|
+
|
|
339
|
+
var insert int = int(base - next_emit)
|
|
340
|
+
ip += int(matched)
|
|
341
|
+
emitInsertLen(uint32(insert), commands)
|
|
342
|
+
copy(*literals, input[next_emit:][:uint(insert)])
|
|
343
|
+
*literals = (*literals)[insert:]
|
|
344
|
+
if distance == last_distance {
|
|
345
|
+
(*commands)[0] = 64
|
|
346
|
+
*commands = (*commands)[1:]
|
|
347
|
+
} else {
|
|
348
|
+
emitDistance(uint32(distance), commands)
|
|
349
|
+
last_distance = distance
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
emitCopyLenLastDistance(matched, commands)
|
|
353
|
+
|
|
354
|
+
next_emit = ip
|
|
355
|
+
if ip >= ip_limit {
|
|
356
|
+
goto emit_remainder
|
|
357
|
+
}
|
|
358
|
+
{
|
|
359
|
+
var input_bytes uint64
|
|
360
|
+
var cur_hash uint32
|
|
361
|
+
/* We could immediately start working at ip now, but to improve
|
|
362
|
+
compression we first update "table" with the hashes of some
|
|
363
|
+
positions within the last copy. */
|
|
364
|
+
|
|
365
|
+
var prev_hash uint32
|
|
366
|
+
if min_match == 4 {
|
|
367
|
+
input_bytes = binary.LittleEndian.Uint64(input[ip-3:])
|
|
368
|
+
cur_hash = hashBytesAtOffset(input_bytes, 3, shift, min_match)
|
|
369
|
+
prev_hash = hashBytesAtOffset(input_bytes, 0, shift, min_match)
|
|
370
|
+
table[prev_hash] = int(ip - base_ip - 3)
|
|
371
|
+
prev_hash = hashBytesAtOffset(input_bytes, 1, shift, min_match)
|
|
372
|
+
table[prev_hash] = int(ip - base_ip - 2)
|
|
373
|
+
prev_hash = hashBytesAtOffset(input_bytes, 0, shift, min_match)
|
|
374
|
+
table[prev_hash] = int(ip - base_ip - 1)
|
|
375
|
+
} else {
|
|
376
|
+
input_bytes = binary.LittleEndian.Uint64(input[ip-5:])
|
|
377
|
+
prev_hash = hashBytesAtOffset(input_bytes, 0, shift, min_match)
|
|
378
|
+
table[prev_hash] = int(ip - base_ip - 5)
|
|
379
|
+
prev_hash = hashBytesAtOffset(input_bytes, 1, shift, min_match)
|
|
380
|
+
table[prev_hash] = int(ip - base_ip - 4)
|
|
381
|
+
prev_hash = hashBytesAtOffset(input_bytes, 2, shift, min_match)
|
|
382
|
+
table[prev_hash] = int(ip - base_ip - 3)
|
|
383
|
+
input_bytes = binary.LittleEndian.Uint64(input[ip-2:])
|
|
384
|
+
cur_hash = hashBytesAtOffset(input_bytes, 2, shift, min_match)
|
|
385
|
+
prev_hash = hashBytesAtOffset(input_bytes, 0, shift, min_match)
|
|
386
|
+
table[prev_hash] = int(ip - base_ip - 2)
|
|
387
|
+
prev_hash = hashBytesAtOffset(input_bytes, 1, shift, min_match)
|
|
388
|
+
table[prev_hash] = int(ip - base_ip - 1)
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
candidate = base_ip + table[cur_hash]
|
|
392
|
+
table[cur_hash] = int(ip - base_ip)
|
|
393
|
+
}
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
for ip-candidate <= maxDistance_compress_fragment && isMatch1(input[ip:], base_ip_ptr[candidate-base_ip:], min_match) {
|
|
397
|
+
var base int = ip
|
|
398
|
+
/* We have a 6-byte match at ip, and no need to emit any
|
|
399
|
+
literal bytes prior to ip. */
|
|
400
|
+
|
|
401
|
+
var matched uint = min_match + findMatchLengthWithLimit(base_ip_ptr[uint(candidate-base_ip)+min_match:], input[uint(ip)+min_match:], uint(ip_end-ip)-min_match)
|
|
402
|
+
ip += int(matched)
|
|
403
|
+
last_distance = int(base - candidate) /* > 0 */
|
|
404
|
+
emitCopyLen(matched, commands)
|
|
405
|
+
emitDistance(uint32(last_distance), commands)
|
|
406
|
+
|
|
407
|
+
next_emit = ip
|
|
408
|
+
if ip >= ip_limit {
|
|
409
|
+
goto emit_remainder
|
|
410
|
+
}
|
|
411
|
+
{
|
|
412
|
+
var input_bytes uint64
|
|
413
|
+
var cur_hash uint32
|
|
414
|
+
/* We could immediately start working at ip now, but to improve
|
|
415
|
+
compression we first update "table" with the hashes of some
|
|
416
|
+
positions within the last copy. */
|
|
417
|
+
|
|
418
|
+
var prev_hash uint32
|
|
419
|
+
if min_match == 4 {
|
|
420
|
+
input_bytes = binary.LittleEndian.Uint64(input[ip-3:])
|
|
421
|
+
cur_hash = hashBytesAtOffset(input_bytes, 3, shift, min_match)
|
|
422
|
+
prev_hash = hashBytesAtOffset(input_bytes, 0, shift, min_match)
|
|
423
|
+
table[prev_hash] = int(ip - base_ip - 3)
|
|
424
|
+
prev_hash = hashBytesAtOffset(input_bytes, 1, shift, min_match)
|
|
425
|
+
table[prev_hash] = int(ip - base_ip - 2)
|
|
426
|
+
prev_hash = hashBytesAtOffset(input_bytes, 2, shift, min_match)
|
|
427
|
+
table[prev_hash] = int(ip - base_ip - 1)
|
|
428
|
+
} else {
|
|
429
|
+
input_bytes = binary.LittleEndian.Uint64(input[ip-5:])
|
|
430
|
+
prev_hash = hashBytesAtOffset(input_bytes, 0, shift, min_match)
|
|
431
|
+
table[prev_hash] = int(ip - base_ip - 5)
|
|
432
|
+
prev_hash = hashBytesAtOffset(input_bytes, 1, shift, min_match)
|
|
433
|
+
table[prev_hash] = int(ip - base_ip - 4)
|
|
434
|
+
prev_hash = hashBytesAtOffset(input_bytes, 2, shift, min_match)
|
|
435
|
+
table[prev_hash] = int(ip - base_ip - 3)
|
|
436
|
+
input_bytes = binary.LittleEndian.Uint64(input[ip-2:])
|
|
437
|
+
cur_hash = hashBytesAtOffset(input_bytes, 2, shift, min_match)
|
|
438
|
+
prev_hash = hashBytesAtOffset(input_bytes, 0, shift, min_match)
|
|
439
|
+
table[prev_hash] = int(ip - base_ip - 2)
|
|
440
|
+
prev_hash = hashBytesAtOffset(input_bytes, 1, shift, min_match)
|
|
441
|
+
table[prev_hash] = int(ip - base_ip - 1)
|
|
442
|
+
}
|
|
443
|
+
|
|
444
|
+
candidate = base_ip + table[cur_hash]
|
|
445
|
+
table[cur_hash] = int(ip - base_ip)
|
|
446
|
+
}
|
|
447
|
+
}
|
|
448
|
+
|
|
449
|
+
ip++
|
|
450
|
+
next_hash = hash1(input[ip:], shift, min_match)
|
|
451
|
+
}
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
emit_remainder:
|
|
455
|
+
assert(next_emit <= ip_end)
|
|
456
|
+
|
|
457
|
+
/* Emit the remaining bytes as literals. */
|
|
458
|
+
if next_emit < ip_end {
|
|
459
|
+
var insert uint32 = uint32(ip_end - next_emit)
|
|
460
|
+
emitInsertLen(insert, commands)
|
|
461
|
+
copy(*literals, input[next_emit:][:insert])
|
|
462
|
+
*literals = (*literals)[insert:]
|
|
463
|
+
}
|
|
464
|
+
}
|
|
465
|
+
|
|
466
|
+
var storeCommands_kNumExtraBits = [128]uint32{
|
|
467
|
+
0,
|
|
468
|
+
0,
|
|
469
|
+
0,
|
|
470
|
+
0,
|
|
471
|
+
0,
|
|
472
|
+
0,
|
|
473
|
+
1,
|
|
474
|
+
1,
|
|
475
|
+
2,
|
|
476
|
+
2,
|
|
477
|
+
3,
|
|
478
|
+
3,
|
|
479
|
+
4,
|
|
480
|
+
4,
|
|
481
|
+
5,
|
|
482
|
+
5,
|
|
483
|
+
6,
|
|
484
|
+
7,
|
|
485
|
+
8,
|
|
486
|
+
9,
|
|
487
|
+
10,
|
|
488
|
+
12,
|
|
489
|
+
14,
|
|
490
|
+
24,
|
|
491
|
+
0,
|
|
492
|
+
0,
|
|
493
|
+
0,
|
|
494
|
+
0,
|
|
495
|
+
0,
|
|
496
|
+
0,
|
|
497
|
+
0,
|
|
498
|
+
0,
|
|
499
|
+
1,
|
|
500
|
+
1,
|
|
501
|
+
2,
|
|
502
|
+
2,
|
|
503
|
+
3,
|
|
504
|
+
3,
|
|
505
|
+
4,
|
|
506
|
+
4,
|
|
507
|
+
0,
|
|
508
|
+
0,
|
|
509
|
+
0,
|
|
510
|
+
0,
|
|
511
|
+
0,
|
|
512
|
+
0,
|
|
513
|
+
0,
|
|
514
|
+
0,
|
|
515
|
+
1,
|
|
516
|
+
1,
|
|
517
|
+
2,
|
|
518
|
+
2,
|
|
519
|
+
3,
|
|
520
|
+
3,
|
|
521
|
+
4,
|
|
522
|
+
4,
|
|
523
|
+
5,
|
|
524
|
+
5,
|
|
525
|
+
6,
|
|
526
|
+
7,
|
|
527
|
+
8,
|
|
528
|
+
9,
|
|
529
|
+
10,
|
|
530
|
+
24,
|
|
531
|
+
0,
|
|
532
|
+
0,
|
|
533
|
+
0,
|
|
534
|
+
0,
|
|
535
|
+
0,
|
|
536
|
+
0,
|
|
537
|
+
0,
|
|
538
|
+
0,
|
|
539
|
+
0,
|
|
540
|
+
0,
|
|
541
|
+
0,
|
|
542
|
+
0,
|
|
543
|
+
0,
|
|
544
|
+
0,
|
|
545
|
+
0,
|
|
546
|
+
0,
|
|
547
|
+
1,
|
|
548
|
+
1,
|
|
549
|
+
2,
|
|
550
|
+
2,
|
|
551
|
+
3,
|
|
552
|
+
3,
|
|
553
|
+
4,
|
|
554
|
+
4,
|
|
555
|
+
5,
|
|
556
|
+
5,
|
|
557
|
+
6,
|
|
558
|
+
6,
|
|
559
|
+
7,
|
|
560
|
+
7,
|
|
561
|
+
8,
|
|
562
|
+
8,
|
|
563
|
+
9,
|
|
564
|
+
9,
|
|
565
|
+
10,
|
|
566
|
+
10,
|
|
567
|
+
11,
|
|
568
|
+
11,
|
|
569
|
+
12,
|
|
570
|
+
12,
|
|
571
|
+
13,
|
|
572
|
+
13,
|
|
573
|
+
14,
|
|
574
|
+
14,
|
|
575
|
+
15,
|
|
576
|
+
15,
|
|
577
|
+
16,
|
|
578
|
+
16,
|
|
579
|
+
17,
|
|
580
|
+
17,
|
|
581
|
+
18,
|
|
582
|
+
18,
|
|
583
|
+
19,
|
|
584
|
+
19,
|
|
585
|
+
20,
|
|
586
|
+
20,
|
|
587
|
+
21,
|
|
588
|
+
21,
|
|
589
|
+
22,
|
|
590
|
+
22,
|
|
591
|
+
23,
|
|
592
|
+
23,
|
|
593
|
+
24,
|
|
594
|
+
24,
|
|
595
|
+
}
|
|
596
|
+
var storeCommands_kInsertOffset = [24]uint32{
|
|
597
|
+
0,
|
|
598
|
+
1,
|
|
599
|
+
2,
|
|
600
|
+
3,
|
|
601
|
+
4,
|
|
602
|
+
5,
|
|
603
|
+
6,
|
|
604
|
+
8,
|
|
605
|
+
10,
|
|
606
|
+
14,
|
|
607
|
+
18,
|
|
608
|
+
26,
|
|
609
|
+
34,
|
|
610
|
+
50,
|
|
611
|
+
66,
|
|
612
|
+
98,
|
|
613
|
+
130,
|
|
614
|
+
194,
|
|
615
|
+
322,
|
|
616
|
+
578,
|
|
617
|
+
1090,
|
|
618
|
+
2114,
|
|
619
|
+
6210,
|
|
620
|
+
22594,
|
|
621
|
+
}
|
|
622
|
+
|
|
623
|
+
func storeCommands(literals []byte, num_literals uint, commands []uint32, num_commands uint, storage_ix *uint, storage []byte) {
|
|
624
|
+
var lit_depths [256]byte
|
|
625
|
+
var lit_bits [256]uint16
|
|
626
|
+
var lit_histo = [256]uint32{0}
|
|
627
|
+
var cmd_depths = [128]byte{0}
|
|
628
|
+
var cmd_bits = [128]uint16{0}
|
|
629
|
+
var cmd_histo = [128]uint32{0}
|
|
630
|
+
var i uint
|
|
631
|
+
for i = 0; i < num_literals; i++ {
|
|
632
|
+
lit_histo[literals[i]]++
|
|
633
|
+
}
|
|
634
|
+
|
|
635
|
+
buildAndStoreHuffmanTreeFast(lit_histo[:], num_literals, /* max_bits = */
|
|
636
|
+
8, lit_depths[:], lit_bits[:], storage_ix, storage)
|
|
637
|
+
|
|
638
|
+
for i = 0; i < num_commands; i++ {
|
|
639
|
+
var code uint32 = commands[i] & 0xFF
|
|
640
|
+
assert(code < 128)
|
|
641
|
+
cmd_histo[code]++
|
|
642
|
+
}
|
|
643
|
+
|
|
644
|
+
cmd_histo[1] += 1
|
|
645
|
+
cmd_histo[2] += 1
|
|
646
|
+
cmd_histo[64] += 1
|
|
647
|
+
cmd_histo[84] += 1
|
|
648
|
+
buildAndStoreCommandPrefixCode(cmd_histo[:], cmd_depths[:], cmd_bits[:], storage_ix, storage)
|
|
649
|
+
|
|
650
|
+
for i = 0; i < num_commands; i++ {
|
|
651
|
+
var cmd uint32 = commands[i]
|
|
652
|
+
var code uint32 = cmd & 0xFF
|
|
653
|
+
var extra uint32 = cmd >> 8
|
|
654
|
+
assert(code < 128)
|
|
655
|
+
writeBits(uint(cmd_depths[code]), uint64(cmd_bits[code]), storage_ix, storage)
|
|
656
|
+
writeBits(uint(storeCommands_kNumExtraBits[code]), uint64(extra), storage_ix, storage)
|
|
657
|
+
if code < 24 {
|
|
658
|
+
var insert uint32 = storeCommands_kInsertOffset[code] + extra
|
|
659
|
+
var j uint32
|
|
660
|
+
for j = 0; j < insert; j++ {
|
|
661
|
+
var lit byte = literals[0]
|
|
662
|
+
writeBits(uint(lit_depths[lit]), uint64(lit_bits[lit]), storage_ix, storage)
|
|
663
|
+
literals = literals[1:]
|
|
664
|
+
}
|
|
665
|
+
}
|
|
666
|
+
}
|
|
667
|
+
}
|
|
668
|
+
|
|
669
|
+
/* Acceptable loss for uncompressible speedup is 2% */
|
|
670
|
+
const minRatio = 0.98
|
|
671
|
+
|
|
672
|
+
const sampleRate = 43
|
|
673
|
+
|
|
674
|
+
func shouldCompress(input []byte, input_size uint, num_literals uint) bool {
|
|
675
|
+
var corpus_size float64 = float64(input_size)
|
|
676
|
+
if float64(num_literals) < minRatio*corpus_size {
|
|
677
|
+
return true
|
|
678
|
+
} else {
|
|
679
|
+
var literal_histo = [256]uint32{0}
|
|
680
|
+
var max_total_bit_cost float64 = corpus_size * 8 * minRatio / sampleRate
|
|
681
|
+
var i uint
|
|
682
|
+
for i = 0; i < input_size; i += sampleRate {
|
|
683
|
+
literal_histo[input[i]]++
|
|
684
|
+
}
|
|
685
|
+
|
|
686
|
+
return bitsEntropy(literal_histo[:], 256) < max_total_bit_cost
|
|
687
|
+
}
|
|
688
|
+
}
|
|
689
|
+
|
|
690
|
+
func rewindBitPosition(new_storage_ix uint, storage_ix *uint, storage []byte) {
|
|
691
|
+
var bitpos uint = new_storage_ix & 7
|
|
692
|
+
var mask uint = (1 << bitpos) - 1
|
|
693
|
+
storage[new_storage_ix>>3] &= byte(mask)
|
|
694
|
+
*storage_ix = new_storage_ix
|
|
695
|
+
}
|
|
696
|
+
|
|
697
|
+
func emitUncompressedMetaBlock(input []byte, input_size uint, storage_ix *uint, storage []byte) {
|
|
698
|
+
storeMetaBlockHeader(input_size, true, storage_ix, storage)
|
|
699
|
+
*storage_ix = (*storage_ix + 7) &^ 7
|
|
700
|
+
copy(storage[*storage_ix>>3:], input[:input_size])
|
|
701
|
+
*storage_ix += input_size << 3
|
|
702
|
+
storage[*storage_ix>>3] = 0
|
|
703
|
+
}
|
|
704
|
+
|
|
705
|
+
func compressFragmentTwoPassImpl(input []byte, input_size uint, is_last bool, command_buf []uint32, literal_buf []byte, table []int, table_bits uint, min_match uint, storage_ix *uint, storage []byte) {
|
|
706
|
+
/* Save the start of the first block for position and distance computations.
|
|
707
|
+
*/
|
|
708
|
+
var base_ip []byte = input
|
|
709
|
+
|
|
710
|
+
for input_size > 0 {
|
|
711
|
+
var block_size uint = brotli_min_size_t(input_size, kCompressFragmentTwoPassBlockSize)
|
|
712
|
+
var commands []uint32 = command_buf
|
|
713
|
+
var literals []byte = literal_buf
|
|
714
|
+
var num_literals uint
|
|
715
|
+
createCommands(input, block_size, input_size, base_ip, table, table_bits, min_match, &literals, &commands)
|
|
716
|
+
num_literals = uint(-cap(literals) + cap(literal_buf))
|
|
717
|
+
if shouldCompress(input, block_size, num_literals) {
|
|
718
|
+
var num_commands uint = uint(-cap(commands) + cap(command_buf))
|
|
719
|
+
storeMetaBlockHeader(block_size, false, storage_ix, storage)
|
|
720
|
+
|
|
721
|
+
/* No block splits, no contexts. */
|
|
722
|
+
writeBits(13, 0, storage_ix, storage)
|
|
723
|
+
|
|
724
|
+
storeCommands(literal_buf, num_literals, command_buf, num_commands, storage_ix, storage)
|
|
725
|
+
} else {
|
|
726
|
+
/* Since we did not find many backward references and the entropy of
|
|
727
|
+
the data is close to 8 bits, we can simply emit an uncompressed block.
|
|
728
|
+
This makes compression speed of uncompressible data about 3x faster. */
|
|
729
|
+
emitUncompressedMetaBlock(input, block_size, storage_ix, storage)
|
|
730
|
+
}
|
|
731
|
+
|
|
732
|
+
input = input[block_size:]
|
|
733
|
+
input_size -= block_size
|
|
734
|
+
}
|
|
735
|
+
}
|
|
736
|
+
|
|
737
|
+
/*
|
|
738
|
+
Compresses "input" string to the "*storage" buffer as one or more complete
|
|
739
|
+
|
|
740
|
+
meta-blocks, and updates the "*storage_ix" bit position.
|
|
741
|
+
|
|
742
|
+
If "is_last" is 1, emits an additional empty last meta-block.
|
|
743
|
+
|
|
744
|
+
REQUIRES: "input_size" is greater than zero, or "is_last" is 1.
|
|
745
|
+
REQUIRES: "input_size" is less or equal to maximal metablock size (1 << 24).
|
|
746
|
+
REQUIRES: "command_buf" and "literal_buf" point to at least
|
|
747
|
+
kCompressFragmentTwoPassBlockSize long arrays.
|
|
748
|
+
REQUIRES: All elements in "table[0..table_size-1]" are initialized to zero.
|
|
749
|
+
REQUIRES: "table_size" is a power of two
|
|
750
|
+
OUTPUT: maximal copy distance <= |input_size|
|
|
751
|
+
OUTPUT: maximal copy distance <= BROTLI_MAX_BACKWARD_LIMIT(18)
|
|
752
|
+
*/
|
|
753
|
+
func compressFragmentTwoPass(input []byte, input_size uint, is_last bool, command_buf []uint32, literal_buf []byte, table []int, table_size uint, storage_ix *uint, storage []byte) {
|
|
754
|
+
var initial_storage_ix uint = *storage_ix
|
|
755
|
+
var table_bits uint = uint(log2FloorNonZero(table_size))
|
|
756
|
+
var min_match uint
|
|
757
|
+
if table_bits <= 15 {
|
|
758
|
+
min_match = 4
|
|
759
|
+
} else {
|
|
760
|
+
min_match = 6
|
|
761
|
+
}
|
|
762
|
+
compressFragmentTwoPassImpl(input, input_size, is_last, command_buf, literal_buf, table, table_bits, min_match, storage_ix, storage)
|
|
763
|
+
|
|
764
|
+
/* If output is larger than single uncompressed block, rewrite it. */
|
|
765
|
+
if *storage_ix-initial_storage_ix > 31+(input_size<<3) {
|
|
766
|
+
rewindBitPosition(initial_storage_ix, storage_ix, storage)
|
|
767
|
+
emitUncompressedMetaBlock(input, input_size, storage_ix, storage)
|
|
768
|
+
}
|
|
769
|
+
|
|
770
|
+
if is_last {
|
|
771
|
+
writeBits(1, 1, storage_ix, storage) /* islast */
|
|
772
|
+
writeBits(1, 1, storage_ix, storage) /* isempty */
|
|
773
|
+
*storage_ix = (*storage_ix + 7) &^ 7
|
|
774
|
+
}
|
|
775
|
+
}
|