typenative 0.3.1 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (147) hide show
  1. package/CHANGELOG.md +57 -0
  2. package/README.md +17 -1
  3. package/bin/cli.js +25 -11
  4. package/bin/go/convert.js +11 -1
  5. package/bin/index.js +221 -47
  6. package/bin/main.js +35 -14
  7. package/bin/node/assert.js +2 -2
  8. package/bin/node/buffer.js +9 -0
  9. package/bin/node/dns.js +1 -1
  10. package/bin/node/events.js +1 -1
  11. package/bin/node/fs.js +1 -1
  12. package/bin/node/fs_watch.js +407 -0
  13. package/bin/node/http2.js +33 -6
  14. package/bin/node/readline.js +1 -1
  15. package/bin/node/registry.js +2 -1
  16. package/bin/node/stream.js +6 -3
  17. package/bin/node/timers.js +1 -1
  18. package/bin/node/typed_arrays.js +608 -0
  19. package/bin/node/util.js +3 -0
  20. package/bin/node/zlib.js +131 -16
  21. package/bin/transpiler.js +635 -44
  22. package/bin/version.js +1 -1
  23. package/conformance.md +16 -4
  24. package/examples/README.md +54 -0
  25. package/examples/fibonacci/README.md +89 -0
  26. package/examples/fibonacci/bench.mjs +87 -0
  27. package/examples/fibonacci/main.ts +21 -0
  28. package/examples/hello-world/README.md +62 -0
  29. package/examples/hello-world/bench.mjs +66 -0
  30. package/examples/hello-world/main.ts +7 -0
  31. package/examples/json-server/README.md +125 -0
  32. package/examples/json-server/bench.mjs +171 -0
  33. package/examples/json-server/main.ts +67 -0
  34. package/govendor/github.com/andybalholm/brotli/LICENSE +19 -0
  35. package/govendor/github.com/andybalholm/brotli/README.md +19 -0
  36. package/govendor/github.com/andybalholm/brotli/backward_references.go +185 -0
  37. package/govendor/github.com/andybalholm/brotli/backward_references_hq.go +796 -0
  38. package/govendor/github.com/andybalholm/brotli/bit_cost.go +436 -0
  39. package/govendor/github.com/andybalholm/brotli/bit_reader.go +266 -0
  40. package/govendor/github.com/andybalholm/brotli/bitwriter.go +56 -0
  41. package/govendor/github.com/andybalholm/brotli/block_splitter.go +144 -0
  42. package/govendor/github.com/andybalholm/brotli/block_splitter_command.go +434 -0
  43. package/govendor/github.com/andybalholm/brotli/block_splitter_distance.go +433 -0
  44. package/govendor/github.com/andybalholm/brotli/block_splitter_literal.go +433 -0
  45. package/govendor/github.com/andybalholm/brotli/brotli_bit_stream.go +1435 -0
  46. package/govendor/github.com/andybalholm/brotli/cluster.go +30 -0
  47. package/govendor/github.com/andybalholm/brotli/cluster_command.go +164 -0
  48. package/govendor/github.com/andybalholm/brotli/cluster_distance.go +326 -0
  49. package/govendor/github.com/andybalholm/brotli/cluster_literal.go +326 -0
  50. package/govendor/github.com/andybalholm/brotli/command.go +254 -0
  51. package/govendor/github.com/andybalholm/brotli/compress_fragment.go +834 -0
  52. package/govendor/github.com/andybalholm/brotli/compress_fragment_two_pass.go +775 -0
  53. package/govendor/github.com/andybalholm/brotli/constants.go +77 -0
  54. package/govendor/github.com/andybalholm/brotli/context.go +2176 -0
  55. package/govendor/github.com/andybalholm/brotli/decode.go +2570 -0
  56. package/govendor/github.com/andybalholm/brotli/dictionary.go +122890 -0
  57. package/govendor/github.com/andybalholm/brotli/dictionary_hash.go +32779 -0
  58. package/govendor/github.com/andybalholm/brotli/encode.go +1220 -0
  59. package/govendor/github.com/andybalholm/brotli/encoder.go +177 -0
  60. package/govendor/github.com/andybalholm/brotli/encoder_dict.go +22 -0
  61. package/govendor/github.com/andybalholm/brotli/encoder_fast.go +201 -0
  62. package/govendor/github.com/andybalholm/brotli/entropy_encode.go +592 -0
  63. package/govendor/github.com/andybalholm/brotli/entropy_encode_static.go +4399 -0
  64. package/govendor/github.com/andybalholm/brotli/fast_log.go +290 -0
  65. package/govendor/github.com/andybalholm/brotli/find_match_length.go +45 -0
  66. package/govendor/github.com/andybalholm/brotli/go.mod +3 -0
  67. package/govendor/github.com/andybalholm/brotli/h10.go +287 -0
  68. package/govendor/github.com/andybalholm/brotli/h5.go +214 -0
  69. package/govendor/github.com/andybalholm/brotli/h6.go +216 -0
  70. package/govendor/github.com/andybalholm/brotli/hash.go +342 -0
  71. package/govendor/github.com/andybalholm/brotli/hash_composite.go +93 -0
  72. package/govendor/github.com/andybalholm/brotli/hash_forgetful_chain.go +252 -0
  73. package/govendor/github.com/andybalholm/brotli/hash_longest_match_quickly.go +214 -0
  74. package/govendor/github.com/andybalholm/brotli/hash_rolling.go +168 -0
  75. package/govendor/github.com/andybalholm/brotli/histogram.go +226 -0
  76. package/govendor/github.com/andybalholm/brotli/http.go +201 -0
  77. package/govendor/github.com/andybalholm/brotli/huffman.go +653 -0
  78. package/govendor/github.com/andybalholm/brotli/literal_cost.go +182 -0
  79. package/govendor/github.com/andybalholm/brotli/matchfinder/bargain1.go +212 -0
  80. package/govendor/github.com/andybalholm/brotli/matchfinder/bargain2.go +248 -0
  81. package/govendor/github.com/andybalholm/brotli/matchfinder/bargain3.go +278 -0
  82. package/govendor/github.com/andybalholm/brotli/matchfinder/emitter.go +34 -0
  83. package/govendor/github.com/andybalholm/brotli/matchfinder/m0.go +175 -0
  84. package/govendor/github.com/andybalholm/brotli/matchfinder/m4.go +339 -0
  85. package/govendor/github.com/andybalholm/brotli/matchfinder/matchfinder.go +103 -0
  86. package/govendor/github.com/andybalholm/brotli/matchfinder/pathfinder.go +328 -0
  87. package/govendor/github.com/andybalholm/brotli/matchfinder/textencoder.go +53 -0
  88. package/govendor/github.com/andybalholm/brotli/matchfinder/trio.go +327 -0
  89. package/govendor/github.com/andybalholm/brotli/matchfinder/zdfast.go +258 -0
  90. package/govendor/github.com/andybalholm/brotli/matchfinder/zfast.go +206 -0
  91. package/govendor/github.com/andybalholm/brotli/matchfinder/zm.go +302 -0
  92. package/govendor/github.com/andybalholm/brotli/memory.go +66 -0
  93. package/govendor/github.com/andybalholm/brotli/metablock.go +574 -0
  94. package/govendor/github.com/andybalholm/brotli/metablock_command.go +165 -0
  95. package/govendor/github.com/andybalholm/brotli/metablock_distance.go +165 -0
  96. package/govendor/github.com/andybalholm/brotli/metablock_literal.go +165 -0
  97. package/govendor/github.com/andybalholm/brotli/params.go +37 -0
  98. package/govendor/github.com/andybalholm/brotli/platform.go +103 -0
  99. package/govendor/github.com/andybalholm/brotli/prefix.go +30 -0
  100. package/govendor/github.com/andybalholm/brotli/prefix_dec.go +723 -0
  101. package/govendor/github.com/andybalholm/brotli/quality.go +196 -0
  102. package/govendor/github.com/andybalholm/brotli/reader.go +127 -0
  103. package/govendor/github.com/andybalholm/brotli/ringbuffer.go +134 -0
  104. package/govendor/github.com/andybalholm/brotli/state.go +313 -0
  105. package/govendor/github.com/andybalholm/brotli/static_dict.go +662 -0
  106. package/govendor/github.com/andybalholm/brotli/static_dict_lut.go +75094 -0
  107. package/govendor/github.com/andybalholm/brotli/symbol_list.go +22 -0
  108. package/govendor/github.com/andybalholm/brotli/transform.go +641 -0
  109. package/govendor/github.com/andybalholm/brotli/utf8_util.go +70 -0
  110. package/govendor/github.com/andybalholm/brotli/util.go +7 -0
  111. package/govendor/github.com/andybalholm/brotli/write_bits.go +52 -0
  112. package/govendor/github.com/andybalholm/brotli/writer.go +162 -0
  113. package/govendor/github.com/fsnotify/fsnotify/.mailmap +2 -0
  114. package/govendor/github.com/fsnotify/fsnotify/CHANGELOG.md +651 -0
  115. package/govendor/github.com/fsnotify/fsnotify/CONTRIBUTING.md +147 -0
  116. package/govendor/github.com/fsnotify/fsnotify/LICENSE +25 -0
  117. package/govendor/github.com/fsnotify/fsnotify/README.md +214 -0
  118. package/govendor/github.com/fsnotify/fsnotify/backend_fen.go +469 -0
  119. package/govendor/github.com/fsnotify/fsnotify/backend_inotify.go +597 -0
  120. package/govendor/github.com/fsnotify/fsnotify/backend_kqueue.go +735 -0
  121. package/govendor/github.com/fsnotify/fsnotify/backend_other.go +22 -0
  122. package/govendor/github.com/fsnotify/fsnotify/backend_windows.go +702 -0
  123. package/govendor/github.com/fsnotify/fsnotify/fsnotify.go +526 -0
  124. package/govendor/github.com/fsnotify/fsnotify/go.mod +3 -0
  125. package/govendor/github.com/fsnotify/fsnotify/internal/darwin.go +20 -0
  126. package/govendor/github.com/fsnotify/fsnotify/internal/debug_darwin.go +15 -0
  127. package/govendor/github.com/fsnotify/fsnotify/internal/debug_dragonfly.go +15 -0
  128. package/govendor/github.com/fsnotify/fsnotify/internal/debug_freebsd.go +20 -0
  129. package/govendor/github.com/fsnotify/fsnotify/internal/debug_kqueue.go +32 -0
  130. package/govendor/github.com/fsnotify/fsnotify/internal/debug_linux.go +56 -0
  131. package/govendor/github.com/fsnotify/fsnotify/internal/debug_netbsd.go +15 -0
  132. package/govendor/github.com/fsnotify/fsnotify/internal/debug_openbsd.go +16 -0
  133. package/govendor/github.com/fsnotify/fsnotify/internal/debug_solaris.go +45 -0
  134. package/govendor/github.com/fsnotify/fsnotify/internal/debug_windows.go +40 -0
  135. package/govendor/github.com/fsnotify/fsnotify/internal/freebsd.go +20 -0
  136. package/govendor/github.com/fsnotify/fsnotify/internal/internal.go +2 -0
  137. package/govendor/github.com/fsnotify/fsnotify/internal/unix.go +20 -0
  138. package/govendor/github.com/fsnotify/fsnotify/internal/unix2.go +25 -0
  139. package/govendor/github.com/fsnotify/fsnotify/internal/windows.go +40 -0
  140. package/govendor/github.com/fsnotify/fsnotify/shared.go +64 -0
  141. package/govendor/github.com/fsnotify/fsnotify/staticcheck.conf +3 -0
  142. package/govendor/github.com/fsnotify/fsnotify/system_bsd.go +7 -0
  143. package/govendor/github.com/fsnotify/fsnotify/system_darwin.go +8 -0
  144. package/govendor/manifest.txt +3 -1
  145. package/package.json +1 -1
  146. package/types/typenative-npm.d.ts +72 -0
  147. package/types/typenative.d.ts +117 -1
@@ -0,0 +1,30 @@
1
+ package brotli
2
+
3
+ /* Copyright 2013 Google Inc. All Rights Reserved.
4
+
5
+ Distributed under MIT license.
6
+ See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
7
+ */
8
+
9
+ /* Functions for clustering similar histograms together. */
10
+
11
+ type histogramPair struct {
12
+ idx1 uint32
13
+ idx2 uint32
14
+ cost_combo float64
15
+ cost_diff float64
16
+ }
17
+
18
+ func histogramPairIsLess(p1 *histogramPair, p2 *histogramPair) bool {
19
+ if p1.cost_diff != p2.cost_diff {
20
+ return p1.cost_diff > p2.cost_diff
21
+ }
22
+
23
+ return (p1.idx2 - p1.idx1) > (p2.idx2 - p2.idx1)
24
+ }
25
+
26
+ /* Returns entropy reduction of the context map when we combine two clusters. */
27
+ func clusterCostDiff(size_a uint, size_b uint) float64 {
28
+ var size_c uint = size_a + size_b
29
+ return float64(size_a)*fastLog2(size_a) + float64(size_b)*fastLog2(size_b) - float64(size_c)*fastLog2(size_c)
30
+ }
@@ -0,0 +1,164 @@
1
+ package brotli
2
+
3
+ /* Copyright 2013 Google Inc. All Rights Reserved.
4
+
5
+ Distributed under MIT license.
6
+ See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
7
+ */
8
+
9
+ /* Computes the bit cost reduction by combining out[idx1] and out[idx2] and if
10
+ it is below a threshold, stores the pair (idx1, idx2) in the *pairs queue. */
11
+ func compareAndPushToQueueCommand(out []histogramCommand, cluster_size []uint32, idx1 uint32, idx2 uint32, max_num_pairs uint, pairs []histogramPair, num_pairs *uint) {
12
+ var is_good_pair bool = false
13
+ var p histogramPair
14
+ p.idx2 = 0
15
+ p.idx1 = p.idx2
16
+ p.cost_combo = 0
17
+ p.cost_diff = p.cost_combo
18
+ if idx1 == idx2 {
19
+ return
20
+ }
21
+
22
+ if idx2 < idx1 {
23
+ var t uint32 = idx2
24
+ idx2 = idx1
25
+ idx1 = t
26
+ }
27
+
28
+ p.idx1 = idx1
29
+ p.idx2 = idx2
30
+ p.cost_diff = 0.5 * clusterCostDiff(uint(cluster_size[idx1]), uint(cluster_size[idx2]))
31
+ p.cost_diff -= out[idx1].bit_cost_
32
+ p.cost_diff -= out[idx2].bit_cost_
33
+
34
+ if out[idx1].total_count_ == 0 {
35
+ p.cost_combo = out[idx2].bit_cost_
36
+ is_good_pair = true
37
+ } else if out[idx2].total_count_ == 0 {
38
+ p.cost_combo = out[idx1].bit_cost_
39
+ is_good_pair = true
40
+ } else {
41
+ var threshold float64
42
+ if *num_pairs == 0 {
43
+ threshold = 1e99
44
+ } else {
45
+ threshold = brotli_max_double(0.0, pairs[0].cost_diff)
46
+ }
47
+ var combo histogramCommand = out[idx1]
48
+ var cost_combo float64
49
+ histogramAddHistogramCommand(&combo, &out[idx2])
50
+ cost_combo = populationCostCommand(&combo)
51
+ if cost_combo < threshold-p.cost_diff {
52
+ p.cost_combo = cost_combo
53
+ is_good_pair = true
54
+ }
55
+ }
56
+
57
+ if is_good_pair {
58
+ p.cost_diff += p.cost_combo
59
+ if *num_pairs > 0 && histogramPairIsLess(&pairs[0], &p) {
60
+ /* Replace the top of the queue if needed. */
61
+ if *num_pairs < max_num_pairs {
62
+ pairs[*num_pairs] = pairs[0]
63
+ (*num_pairs)++
64
+ }
65
+
66
+ pairs[0] = p
67
+ } else if *num_pairs < max_num_pairs {
68
+ pairs[*num_pairs] = p
69
+ (*num_pairs)++
70
+ }
71
+ }
72
+ }
73
+
74
+ func histogramCombineCommand(out []histogramCommand, cluster_size []uint32, symbols []uint32, clusters []uint32, pairs []histogramPair, num_clusters uint, symbols_size uint, max_clusters uint, max_num_pairs uint) uint {
75
+ var cost_diff_threshold float64 = 0.0
76
+ var min_cluster_size uint = 1
77
+ var num_pairs uint = 0
78
+ {
79
+ /* We maintain a vector of histogram pairs, with the property that the pair
80
+ with the maximum bit cost reduction is the first. */
81
+ var idx1 uint
82
+ for idx1 = 0; idx1 < num_clusters; idx1++ {
83
+ var idx2 uint
84
+ for idx2 = idx1 + 1; idx2 < num_clusters; idx2++ {
85
+ compareAndPushToQueueCommand(out, cluster_size, clusters[idx1], clusters[idx2], max_num_pairs, pairs[0:], &num_pairs)
86
+ }
87
+ }
88
+ }
89
+
90
+ for num_clusters > min_cluster_size {
91
+ var best_idx1 uint32
92
+ var best_idx2 uint32
93
+ var i uint
94
+ if pairs[0].cost_diff >= cost_diff_threshold {
95
+ cost_diff_threshold = 1e99
96
+ min_cluster_size = max_clusters
97
+ continue
98
+ }
99
+
100
+ /* Take the best pair from the top of heap. */
101
+ best_idx1 = pairs[0].idx1
102
+
103
+ best_idx2 = pairs[0].idx2
104
+ histogramAddHistogramCommand(&out[best_idx1], &out[best_idx2])
105
+ out[best_idx1].bit_cost_ = pairs[0].cost_combo
106
+ cluster_size[best_idx1] += cluster_size[best_idx2]
107
+ for i = 0; i < symbols_size; i++ {
108
+ if symbols[i] == best_idx2 {
109
+ symbols[i] = best_idx1
110
+ }
111
+ }
112
+
113
+ for i = 0; i < num_clusters; i++ {
114
+ if clusters[i] == best_idx2 {
115
+ copy(clusters[i:], clusters[i+1:][:num_clusters-i-1])
116
+ break
117
+ }
118
+ }
119
+
120
+ num_clusters--
121
+ {
122
+ /* Remove pairs intersecting the just combined best pair. */
123
+ var copy_to_idx uint = 0
124
+ for i = 0; i < num_pairs; i++ {
125
+ var p *histogramPair = &pairs[i]
126
+ if p.idx1 == best_idx1 || p.idx2 == best_idx1 || p.idx1 == best_idx2 || p.idx2 == best_idx2 {
127
+ /* Remove invalid pair from the queue. */
128
+ continue
129
+ }
130
+
131
+ if histogramPairIsLess(&pairs[0], p) {
132
+ /* Replace the top of the queue if needed. */
133
+ var front histogramPair = pairs[0]
134
+ pairs[0] = *p
135
+ pairs[copy_to_idx] = front
136
+ } else {
137
+ pairs[copy_to_idx] = *p
138
+ }
139
+
140
+ copy_to_idx++
141
+ }
142
+
143
+ num_pairs = copy_to_idx
144
+ }
145
+
146
+ /* Push new pairs formed with the combined histogram to the heap. */
147
+ for i = 0; i < num_clusters; i++ {
148
+ compareAndPushToQueueCommand(out, cluster_size, best_idx1, clusters[i], max_num_pairs, pairs[0:], &num_pairs)
149
+ }
150
+ }
151
+
152
+ return num_clusters
153
+ }
154
+
155
+ /* What is the bit cost of moving histogram from cur_symbol to candidate. */
156
+ func histogramBitCostDistanceCommand(histogram *histogramCommand, candidate *histogramCommand) float64 {
157
+ if histogram.total_count_ == 0 {
158
+ return 0.0
159
+ } else {
160
+ var tmp histogramCommand = *histogram
161
+ histogramAddHistogramCommand(&tmp, candidate)
162
+ return populationCostCommand(&tmp) - candidate.bit_cost_
163
+ }
164
+ }
@@ -0,0 +1,326 @@
1
+ package brotli
2
+
3
+ import "math"
4
+
5
+ /* Copyright 2013 Google Inc. All Rights Reserved.
6
+
7
+ Distributed under MIT license.
8
+ See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
9
+ */
10
+
11
+ /* Computes the bit cost reduction by combining out[idx1] and out[idx2] and if
12
+ it is below a threshold, stores the pair (idx1, idx2) in the *pairs queue. */
13
+ func compareAndPushToQueueDistance(out []histogramDistance, cluster_size []uint32, idx1 uint32, idx2 uint32, max_num_pairs uint, pairs []histogramPair, num_pairs *uint) {
14
+ var is_good_pair bool = false
15
+ var p histogramPair
16
+ p.idx2 = 0
17
+ p.idx1 = p.idx2
18
+ p.cost_combo = 0
19
+ p.cost_diff = p.cost_combo
20
+ if idx1 == idx2 {
21
+ return
22
+ }
23
+
24
+ if idx2 < idx1 {
25
+ var t uint32 = idx2
26
+ idx2 = idx1
27
+ idx1 = t
28
+ }
29
+
30
+ p.idx1 = idx1
31
+ p.idx2 = idx2
32
+ p.cost_diff = 0.5 * clusterCostDiff(uint(cluster_size[idx1]), uint(cluster_size[idx2]))
33
+ p.cost_diff -= out[idx1].bit_cost_
34
+ p.cost_diff -= out[idx2].bit_cost_
35
+
36
+ if out[idx1].total_count_ == 0 {
37
+ p.cost_combo = out[idx2].bit_cost_
38
+ is_good_pair = true
39
+ } else if out[idx2].total_count_ == 0 {
40
+ p.cost_combo = out[idx1].bit_cost_
41
+ is_good_pair = true
42
+ } else {
43
+ var threshold float64
44
+ if *num_pairs == 0 {
45
+ threshold = 1e99
46
+ } else {
47
+ threshold = brotli_max_double(0.0, pairs[0].cost_diff)
48
+ }
49
+ var combo histogramDistance = out[idx1]
50
+ var cost_combo float64
51
+ histogramAddHistogramDistance(&combo, &out[idx2])
52
+ cost_combo = populationCostDistance(&combo)
53
+ if cost_combo < threshold-p.cost_diff {
54
+ p.cost_combo = cost_combo
55
+ is_good_pair = true
56
+ }
57
+ }
58
+
59
+ if is_good_pair {
60
+ p.cost_diff += p.cost_combo
61
+ if *num_pairs > 0 && histogramPairIsLess(&pairs[0], &p) {
62
+ /* Replace the top of the queue if needed. */
63
+ if *num_pairs < max_num_pairs {
64
+ pairs[*num_pairs] = pairs[0]
65
+ (*num_pairs)++
66
+ }
67
+
68
+ pairs[0] = p
69
+ } else if *num_pairs < max_num_pairs {
70
+ pairs[*num_pairs] = p
71
+ (*num_pairs)++
72
+ }
73
+ }
74
+ }
75
+
76
+ func histogramCombineDistance(out []histogramDistance, cluster_size []uint32, symbols []uint32, clusters []uint32, pairs []histogramPair, num_clusters uint, symbols_size uint, max_clusters uint, max_num_pairs uint) uint {
77
+ var cost_diff_threshold float64 = 0.0
78
+ var min_cluster_size uint = 1
79
+ var num_pairs uint = 0
80
+ {
81
+ /* We maintain a vector of histogram pairs, with the property that the pair
82
+ with the maximum bit cost reduction is the first. */
83
+ var idx1 uint
84
+ for idx1 = 0; idx1 < num_clusters; idx1++ {
85
+ var idx2 uint
86
+ for idx2 = idx1 + 1; idx2 < num_clusters; idx2++ {
87
+ compareAndPushToQueueDistance(out, cluster_size, clusters[idx1], clusters[idx2], max_num_pairs, pairs[0:], &num_pairs)
88
+ }
89
+ }
90
+ }
91
+
92
+ for num_clusters > min_cluster_size {
93
+ var best_idx1 uint32
94
+ var best_idx2 uint32
95
+ var i uint
96
+ if pairs[0].cost_diff >= cost_diff_threshold {
97
+ cost_diff_threshold = 1e99
98
+ min_cluster_size = max_clusters
99
+ continue
100
+ }
101
+
102
+ /* Take the best pair from the top of heap. */
103
+ best_idx1 = pairs[0].idx1
104
+
105
+ best_idx2 = pairs[0].idx2
106
+ histogramAddHistogramDistance(&out[best_idx1], &out[best_idx2])
107
+ out[best_idx1].bit_cost_ = pairs[0].cost_combo
108
+ cluster_size[best_idx1] += cluster_size[best_idx2]
109
+ for i = 0; i < symbols_size; i++ {
110
+ if symbols[i] == best_idx2 {
111
+ symbols[i] = best_idx1
112
+ }
113
+ }
114
+
115
+ for i = 0; i < num_clusters; i++ {
116
+ if clusters[i] == best_idx2 {
117
+ copy(clusters[i:], clusters[i+1:][:num_clusters-i-1])
118
+ break
119
+ }
120
+ }
121
+
122
+ num_clusters--
123
+ {
124
+ /* Remove pairs intersecting the just combined best pair. */
125
+ var copy_to_idx uint = 0
126
+ for i = 0; i < num_pairs; i++ {
127
+ var p *histogramPair = &pairs[i]
128
+ if p.idx1 == best_idx1 || p.idx2 == best_idx1 || p.idx1 == best_idx2 || p.idx2 == best_idx2 {
129
+ /* Remove invalid pair from the queue. */
130
+ continue
131
+ }
132
+
133
+ if histogramPairIsLess(&pairs[0], p) {
134
+ /* Replace the top of the queue if needed. */
135
+ var front histogramPair = pairs[0]
136
+ pairs[0] = *p
137
+ pairs[copy_to_idx] = front
138
+ } else {
139
+ pairs[copy_to_idx] = *p
140
+ }
141
+
142
+ copy_to_idx++
143
+ }
144
+
145
+ num_pairs = copy_to_idx
146
+ }
147
+
148
+ /* Push new pairs formed with the combined histogram to the heap. */
149
+ for i = 0; i < num_clusters; i++ {
150
+ compareAndPushToQueueDistance(out, cluster_size, best_idx1, clusters[i], max_num_pairs, pairs[0:], &num_pairs)
151
+ }
152
+ }
153
+
154
+ return num_clusters
155
+ }
156
+
157
+ /* What is the bit cost of moving histogram from cur_symbol to candidate. */
158
+ func histogramBitCostDistanceDistance(histogram *histogramDistance, candidate *histogramDistance) float64 {
159
+ if histogram.total_count_ == 0 {
160
+ return 0.0
161
+ } else {
162
+ var tmp histogramDistance = *histogram
163
+ histogramAddHistogramDistance(&tmp, candidate)
164
+ return populationCostDistance(&tmp) - candidate.bit_cost_
165
+ }
166
+ }
167
+
168
+ /* Find the best 'out' histogram for each of the 'in' histograms.
169
+ When called, clusters[0..num_clusters) contains the unique values from
170
+ symbols[0..in_size), but this property is not preserved in this function.
171
+ Note: we assume that out[]->bit_cost_ is already up-to-date. */
172
+ func histogramRemapDistance(in []histogramDistance, in_size uint, clusters []uint32, num_clusters uint, out []histogramDistance, symbols []uint32) {
173
+ var i uint
174
+ for i = 0; i < in_size; i++ {
175
+ var best_out uint32
176
+ if i == 0 {
177
+ best_out = symbols[0]
178
+ } else {
179
+ best_out = symbols[i-1]
180
+ }
181
+ var best_bits float64 = histogramBitCostDistanceDistance(&in[i], &out[best_out])
182
+ var j uint
183
+ for j = 0; j < num_clusters; j++ {
184
+ var cur_bits float64 = histogramBitCostDistanceDistance(&in[i], &out[clusters[j]])
185
+ if cur_bits < best_bits {
186
+ best_bits = cur_bits
187
+ best_out = clusters[j]
188
+ }
189
+ }
190
+
191
+ symbols[i] = best_out
192
+ }
193
+
194
+ /* Recompute each out based on raw and symbols. */
195
+ for i = 0; i < num_clusters; i++ {
196
+ histogramClearDistance(&out[clusters[i]])
197
+ }
198
+
199
+ for i = 0; i < in_size; i++ {
200
+ histogramAddHistogramDistance(&out[symbols[i]], &in[i])
201
+ }
202
+ }
203
+
204
+ /* Reorders elements of the out[0..length) array and changes values in
205
+ symbols[0..length) array in the following way:
206
+ * when called, symbols[] contains indexes into out[], and has N unique
207
+ values (possibly N < length)
208
+ * on return, symbols'[i] = f(symbols[i]) and
209
+ out'[symbols'[i]] = out[symbols[i]], for each 0 <= i < length,
210
+ where f is a bijection between the range of symbols[] and [0..N), and
211
+ the first occurrences of values in symbols'[i] come in consecutive
212
+ increasing order.
213
+ Returns N, the number of unique values in symbols[]. */
214
+
215
+ var histogramReindexDistance_kInvalidIndex uint32 = math.MaxUint32
216
+
217
+ func histogramReindexDistance(out []histogramDistance, symbols []uint32, length uint) uint {
218
+ var new_index []uint32 = make([]uint32, length)
219
+ var next_index uint32
220
+ var tmp []histogramDistance
221
+ var i uint
222
+ for i = 0; i < length; i++ {
223
+ new_index[i] = histogramReindexDistance_kInvalidIndex
224
+ }
225
+
226
+ next_index = 0
227
+ for i = 0; i < length; i++ {
228
+ if new_index[symbols[i]] == histogramReindexDistance_kInvalidIndex {
229
+ new_index[symbols[i]] = next_index
230
+ next_index++
231
+ }
232
+ }
233
+
234
+ /* TODO: by using idea of "cycle-sort" we can avoid allocation of
235
+ tmp and reduce the number of copying by the factor of 2. */
236
+ tmp = make([]histogramDistance, next_index)
237
+
238
+ next_index = 0
239
+ for i = 0; i < length; i++ {
240
+ if new_index[symbols[i]] == next_index {
241
+ tmp[next_index] = out[symbols[i]]
242
+ next_index++
243
+ }
244
+
245
+ symbols[i] = new_index[symbols[i]]
246
+ }
247
+
248
+ new_index = nil
249
+ for i = 0; uint32(i) < next_index; i++ {
250
+ out[i] = tmp[i]
251
+ }
252
+
253
+ tmp = nil
254
+ return uint(next_index)
255
+ }
256
+
257
+ func clusterHistogramsDistance(in []histogramDistance, in_size uint, max_histograms uint, out []histogramDistance, out_size *uint, histogram_symbols []uint32) {
258
+ var cluster_size []uint32 = make([]uint32, in_size)
259
+ var clusters []uint32 = make([]uint32, in_size)
260
+ var num_clusters uint = 0
261
+ var max_input_histograms uint = 64
262
+ var pairs_capacity uint = max_input_histograms * max_input_histograms / 2
263
+ var pairs []histogramPair = make([]histogramPair, (pairs_capacity + 1))
264
+ var i uint
265
+
266
+ /* For the first pass of clustering, we allow all pairs. */
267
+ for i = 0; i < in_size; i++ {
268
+ cluster_size[i] = 1
269
+ }
270
+
271
+ for i = 0; i < in_size; i++ {
272
+ out[i] = in[i]
273
+ out[i].bit_cost_ = populationCostDistance(&in[i])
274
+ histogram_symbols[i] = uint32(i)
275
+ }
276
+
277
+ for i = 0; i < in_size; i += max_input_histograms {
278
+ var num_to_combine uint = brotli_min_size_t(in_size-i, max_input_histograms)
279
+ var num_new_clusters uint
280
+ var j uint
281
+ for j = 0; j < num_to_combine; j++ {
282
+ clusters[num_clusters+j] = uint32(i + j)
283
+ }
284
+
285
+ num_new_clusters = histogramCombineDistance(out, cluster_size, histogram_symbols[i:], clusters[num_clusters:], pairs, num_to_combine, num_to_combine, max_histograms, pairs_capacity)
286
+ num_clusters += num_new_clusters
287
+ }
288
+ {
289
+ /* For the second pass, we limit the total number of histogram pairs.
290
+ After this limit is reached, we only keep searching for the best pair. */
291
+ var max_num_pairs uint = brotli_min_size_t(64*num_clusters, (num_clusters/2)*num_clusters)
292
+ if pairs_capacity < (max_num_pairs + 1) {
293
+ var _new_size uint
294
+ if pairs_capacity == 0 {
295
+ _new_size = max_num_pairs + 1
296
+ } else {
297
+ _new_size = pairs_capacity
298
+ }
299
+ var new_array []histogramPair
300
+ for _new_size < (max_num_pairs + 1) {
301
+ _new_size *= 2
302
+ }
303
+ new_array = make([]histogramPair, _new_size)
304
+ if pairs_capacity != 0 {
305
+ copy(new_array, pairs[:pairs_capacity])
306
+ }
307
+
308
+ pairs = new_array
309
+ pairs_capacity = _new_size
310
+ }
311
+
312
+ /* Collapse similar histograms. */
313
+ num_clusters = histogramCombineDistance(out, cluster_size, histogram_symbols, clusters, pairs, num_clusters, in_size, max_histograms, max_num_pairs)
314
+ }
315
+
316
+ pairs = nil
317
+ cluster_size = nil
318
+
319
+ /* Find the optimal map from original histograms to the final ones. */
320
+ histogramRemapDistance(in, in_size, clusters, num_clusters, out, histogram_symbols)
321
+
322
+ clusters = nil
323
+
324
+ /* Convert the context map to a canonical form. */
325
+ *out_size = histogramReindexDistance(out, histogram_symbols, in_size)
326
+ }