teeworlds 2.0.4 → 2.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/client.js +36 -61
- package/lib/client.ts +3 -19
- package/lib/huffman.js +236 -0
- package/lib/huffman.ts +342 -0
- package/package.json +1 -1
- package/lib/huffman.py +0 -224
package/lib/client.js
CHANGED
|
@@ -54,10 +54,11 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
54
54
|
var crypto_1 = require("crypto");
|
|
55
55
|
var dgram_1 = __importDefault(require("dgram"));
|
|
56
56
|
var stream_1 = require("stream");
|
|
57
|
-
var child_process_1 = require("child_process");
|
|
58
57
|
var MsgUnpacker_1 = __importDefault(require("./MsgUnpacker"));
|
|
59
58
|
var MsgPacker_1 = __importDefault(require("./MsgPacker"));
|
|
60
59
|
var snapshot_1 = require("./snapshot");
|
|
60
|
+
var huffman_1 = __importDefault(require("./huffman"));
|
|
61
|
+
var huff = new huffman_1.default();
|
|
61
62
|
var SnapUnpacker = new snapshot_1.Snapshot();
|
|
62
63
|
var NETMSGTYPE;
|
|
63
64
|
(function (NETMSGTYPE) {
|
|
@@ -122,26 +123,6 @@ var items;
|
|
|
122
123
|
function toHexStream(buff) {
|
|
123
124
|
return buff.toJSON().data.map(function (a) { return ('0' + (a & 0xff).toString(16)).slice(-2); }).join("");
|
|
124
125
|
}
|
|
125
|
-
function decompress(buff) {
|
|
126
|
-
return __awaiter(this, void 0, void 0, function () {
|
|
127
|
-
return __generator(this, function (_a) {
|
|
128
|
-
return [2 /*return*/, new Promise(function (resolve) {
|
|
129
|
-
// get hex stream
|
|
130
|
-
var hexStream = toHexStream(buff);
|
|
131
|
-
var ls = child_process_1.spawn('python', [__dirname + '\\huffman.py', hexStream, "-decompress"]);
|
|
132
|
-
ls.stdout.on('data', function (data) {
|
|
133
|
-
resolve(Buffer.from(eval(data.toString()))); // convert stdout array to actual array, then convert the array to Buffer & return it
|
|
134
|
-
});
|
|
135
|
-
setTimeout(function () {
|
|
136
|
-
ls.stdin.end();
|
|
137
|
-
ls.stdout.destroy();
|
|
138
|
-
ls.stderr.destroy();
|
|
139
|
-
resolve(Buffer.from([]));
|
|
140
|
-
}, 750);
|
|
141
|
-
})];
|
|
142
|
-
});
|
|
143
|
-
});
|
|
144
|
-
}
|
|
145
126
|
var messageTypes = [
|
|
146
127
|
["none, starts at 1", "SV_MOTD", "SV_BROADCAST", "SV_CHAT", "SV_KILL_MSG", "SV_SOUND_GLOBAL", "SV_TUNE_PARAMS", "SV_EXTRA_PROJECTILE", "SV_READY_TO_ENTER", "SV_WEAPON_PICKUP", "SV_EMOTICON", "SV_VOTE_CLEAR_OPTIONS", "SV_VOTE_OPTION_LIST_ADD", "SV_VOTE_OPTION_ADD", "SV_VOTE_OPTION_REMOVE", "SV_VOTE_SET", "SV_VOTE_STATUS", "CL_SAY", "CL_SET_TEAM", "CL_SET_SPECTATOR_MODE", "CL_START_INFO", "CL_CHANGE_INFO", "CL_KILL", "CL_EMOTICON", "CL_VOTE", "CL_CALL_VOTE", "CL_IS_DDNET", "SV_DDRACE_TIME", "SV_RECORD", "UNUSED", "SV_TEAMS_STATE", "CL_SHOW_OTHERS_LEGACY"],
|
|
147
128
|
["none, starts at 1", "INFO", "MAP_CHANGE", "MAP_DATA", "CON_READY", "SNAP", "SNAP_EMPTY", "SNAP_SINGLE", "INPUT_TIMING", "RCON_AUTH_STATUS", "RCON_LINE", "READY", "ENTER_GAME", "INPUT", "RCON_CMD", "RCON_AUTH", "REQUEST_MAP_DATA", "PING", "PING_REPLY", "RCON_CMD_ADD", "RCON_CMD_REMOVE"]
|
|
@@ -193,48 +174,42 @@ var Client = /** @class */ (function (_super) {
|
|
|
193
174
|
return __awaiter(this, void 0, void 0, function () {
|
|
194
175
|
var unpacked, i, chunk;
|
|
195
176
|
return __generator(this, function (_a) {
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
return [4 /*yield*/, decompress(packet)];
|
|
204
|
-
case 1:
|
|
205
|
-
packet = _a.sent();
|
|
206
|
-
if (packet.length == 1 && packet[0] == -1)
|
|
207
|
-
return [2 /*return*/, unpacked];
|
|
208
|
-
_a.label = 2;
|
|
209
|
-
case 2:
|
|
210
|
-
// return unpacked;
|
|
211
|
-
for (i = 0; i < unpacked.twprotocol.chunkAmount; i++) {
|
|
212
|
-
chunk = {};
|
|
213
|
-
chunk.bytes = ((packet[0] & 0x3f) << 4) | (packet[1] & ((1 << 4) - 1)); // idk what this shit is but it works
|
|
214
|
-
chunk.flags = (packet[0] >> 6) & 3;
|
|
215
|
-
chunk.sequence = -1;
|
|
216
|
-
if (chunk.flags & 1) {
|
|
217
|
-
chunk.seq = ((packet[1] & 0xf0) << 2) | packet[2];
|
|
218
|
-
packet = packet.slice(3); // remove flags & size
|
|
219
|
-
}
|
|
220
|
-
else
|
|
221
|
-
packet = packet.slice(2);
|
|
222
|
-
chunk.type = packet[0] & 1 ? "sys" : "game"; // & 1 = binary, ****_***1. e.g 0001_0111 sys, 0001_0110 game
|
|
223
|
-
chunk.msgid = (packet[0] - (packet[0] & 1)) / 2;
|
|
224
|
-
chunk.msg = messageTypes[packet[0] & 1][chunk.msgid];
|
|
225
|
-
chunk.raw = packet.slice(1, chunk.bytes);
|
|
226
|
-
Object.values(messageUUIDs).forEach(function (a, i) {
|
|
227
|
-
if (a.compare(packet.slice(0, 16)) == 0) {
|
|
228
|
-
chunk.extended_msgid = a;
|
|
229
|
-
// chunk.type = 'sys';
|
|
230
|
-
chunk.msg = Object.keys(messageUUIDs)[i];
|
|
231
|
-
}
|
|
232
|
-
});
|
|
233
|
-
packet = packet.slice(chunk.bytes); // +1 cuz it adds an extra \x00 for easier parsing i guess
|
|
234
|
-
unpacked.chunks.push(chunk);
|
|
235
|
-
}
|
|
177
|
+
unpacked = { twprotocol: { flags: packet[0], ack: packet[1], chunkAmount: packet[2], size: packet.byteLength - 3 }, chunks: [] };
|
|
178
|
+
if (packet.indexOf(Buffer.from([0xff, 0xff, 0xff, 0xff])) == 0 && !(unpacked.twprotocol.flags & 8) || unpacked.twprotocol.flags == 255) // flags == 255 is connectionless (used for sending usernames)
|
|
179
|
+
return [2 /*return*/, unpacked];
|
|
180
|
+
packet = packet.slice(3);
|
|
181
|
+
if (unpacked.twprotocol.flags & 128) {
|
|
182
|
+
packet = huff.decompress(packet);
|
|
183
|
+
if (packet.length == 1 && packet[0] == -1)
|
|
236
184
|
return [2 /*return*/, unpacked];
|
|
237
185
|
}
|
|
186
|
+
// return unpacked;
|
|
187
|
+
for (i = 0; i < unpacked.twprotocol.chunkAmount; i++) {
|
|
188
|
+
chunk = {};
|
|
189
|
+
chunk.bytes = ((packet[0] & 0x3f) << 4) | (packet[1] & ((1 << 4) - 1)); // idk what this shit is but it works
|
|
190
|
+
chunk.flags = (packet[0] >> 6) & 3;
|
|
191
|
+
chunk.sequence = -1;
|
|
192
|
+
if (chunk.flags & 1) {
|
|
193
|
+
chunk.seq = ((packet[1] & 0xf0) << 2) | packet[2];
|
|
194
|
+
packet = packet.slice(3); // remove flags & size
|
|
195
|
+
}
|
|
196
|
+
else
|
|
197
|
+
packet = packet.slice(2);
|
|
198
|
+
chunk.type = packet[0] & 1 ? "sys" : "game"; // & 1 = binary, ****_***1. e.g 0001_0111 sys, 0001_0110 game
|
|
199
|
+
chunk.msgid = (packet[0] - (packet[0] & 1)) / 2;
|
|
200
|
+
chunk.msg = messageTypes[packet[0] & 1][chunk.msgid];
|
|
201
|
+
chunk.raw = packet.slice(1, chunk.bytes);
|
|
202
|
+
Object.values(messageUUIDs).forEach(function (a, i) {
|
|
203
|
+
if (a.compare(packet.slice(0, 16)) == 0) {
|
|
204
|
+
chunk.extended_msgid = a;
|
|
205
|
+
// chunk.type = 'sys';
|
|
206
|
+
chunk.msg = Object.keys(messageUUIDs)[i];
|
|
207
|
+
}
|
|
208
|
+
});
|
|
209
|
+
packet = packet.slice(chunk.bytes); // +1 cuz it adds an extra \x00 for easier parsing i guess
|
|
210
|
+
unpacked.chunks.push(chunk);
|
|
211
|
+
}
|
|
212
|
+
return [2 /*return*/, unpacked];
|
|
238
213
|
});
|
|
239
214
|
});
|
|
240
215
|
};
|
package/lib/client.ts
CHANGED
|
@@ -7,7 +7,9 @@ import { spawn } from 'child_process';
|
|
|
7
7
|
import MsgUnpacker from './MsgUnpacker';
|
|
8
8
|
import MsgPacker from './MsgPacker';
|
|
9
9
|
import { Snapshot } from './snapshot';
|
|
10
|
+
import Huffman from "./huffman";
|
|
10
11
|
|
|
12
|
+
const huff = new Huffman();
|
|
11
13
|
const SnapUnpacker = new Snapshot();
|
|
12
14
|
|
|
13
15
|
enum NETMSGTYPE {
|
|
@@ -83,24 +85,6 @@ interface chunk {
|
|
|
83
85
|
function toHexStream(buff: Buffer): string {
|
|
84
86
|
return buff.toJSON().data.map(a => ('0' + (a & 0xff).toString(16)).slice(-2)).join("");
|
|
85
87
|
}
|
|
86
|
-
async function decompress(buff: Buffer): Promise<Buffer> {
|
|
87
|
-
return new Promise((resolve) => {
|
|
88
|
-
// get hex stream
|
|
89
|
-
var hexStream = toHexStream(buff)
|
|
90
|
-
var ls = spawn('python', [__dirname + '\\huffman.py', hexStream, "-decompress"])
|
|
91
|
-
ls.stdout.on('data', (data) => {
|
|
92
|
-
resolve(Buffer.from(eval(data.toString()))); // convert stdout array to actual array, then convert the array to Buffer & return it
|
|
93
|
-
});
|
|
94
|
-
setTimeout(() => {
|
|
95
|
-
ls.stdin.end();
|
|
96
|
-
ls.stdout.destroy();
|
|
97
|
-
ls.stderr.destroy();
|
|
98
|
-
|
|
99
|
-
resolve(Buffer.from([]));
|
|
100
|
-
}, 750)
|
|
101
|
-
|
|
102
|
-
})
|
|
103
|
-
}
|
|
104
88
|
interface _packet {
|
|
105
89
|
twprotocol: { flags: number, ack: number, chunkAmount: number, size: number },
|
|
106
90
|
chunks: chunk[]
|
|
@@ -207,7 +191,7 @@ class Client extends EventEmitter {
|
|
|
207
191
|
return unpacked;
|
|
208
192
|
packet = packet.slice(3)
|
|
209
193
|
if (unpacked.twprotocol.flags & 128) {
|
|
210
|
-
packet =
|
|
194
|
+
packet = huff.decompress(packet)
|
|
211
195
|
if (packet.length == 1 && packet[0] == -1)
|
|
212
196
|
return unpacked
|
|
213
197
|
}
|
package/lib/huffman.js
ADDED
|
@@ -0,0 +1,236 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// from Ryozuki, found on ddnet discord ( 11.10.2018 )
|
|
3
|
+
// from typing import List
|
|
4
|
+
// from sys import argv
|
|
5
|
+
var FREQ_TABLE = [
|
|
6
|
+
1 << 30, 4545, 2657, 431, 1950, 919, 444, 482, 2244, 617, 838, 542, 715, 1814, 304, 240, 754, 212, 647, 186,
|
|
7
|
+
283, 131, 146, 166, 543, 164, 167, 136, 179, 859, 363, 113, 157, 154, 204, 108, 137, 180, 202, 176,
|
|
8
|
+
872, 404, 168, 134, 151, 111, 113, 109, 120, 126, 129, 100, 41, 20, 16, 22, 18, 18, 17, 19,
|
|
9
|
+
16, 37, 13, 21, 362, 166, 99, 78, 95, 88, 81, 70, 83, 284, 91, 187, 77, 68, 52, 68,
|
|
10
|
+
59, 66, 61, 638, 71, 157, 50, 46, 69, 43, 11, 24, 13, 19, 10, 12, 12, 20, 14, 9,
|
|
11
|
+
20, 20, 10, 10, 15, 15, 12, 12, 7, 19, 15, 14, 13, 18, 35, 19, 17, 14, 8, 5,
|
|
12
|
+
15, 17, 9, 15, 14, 18, 8, 10, 2173, 134, 157, 68, 188, 60, 170, 60, 194, 62, 175, 71,
|
|
13
|
+
148, 67, 167, 78, 211, 67, 156, 69, 1674, 90, 174, 53, 147, 89, 181, 51, 174, 63, 163, 80,
|
|
14
|
+
167, 94, 128, 122, 223, 153, 218, 77, 200, 110, 190, 73, 174, 69, 145, 66, 277, 143, 141, 60,
|
|
15
|
+
136, 53, 180, 57, 142, 57, 158, 61, 166, 112, 152, 92, 26, 22, 21, 28, 20, 26, 30, 21,
|
|
16
|
+
32, 27, 20, 17, 23, 21, 30, 22, 22, 21, 27, 25, 17, 27, 23, 18, 39, 26, 15, 21,
|
|
17
|
+
12, 18, 18, 27, 20, 18, 15, 19, 11, 17, 33, 12, 18, 15, 19, 18, 16, 26, 17, 18,
|
|
18
|
+
9, 10, 25, 22, 22, 17, 20, 16, 6, 16, 15, 20, 14, 18, 24, 335, 1517
|
|
19
|
+
], HUFFMAN_EOF_SYMBOL = 256, HUFFMAN_MAX_SYMBOLS = HUFFMAN_EOF_SYMBOL + 1, HUFFMAN_MAX_NODES = HUFFMAN_MAX_SYMBOLS * 2 - 1, HUFFMAN_LUTBITS = 10, HUFFMAN_LUTSIZE = 1 << HUFFMAN_LUTBITS, HUFFMAN_LUTMASK = HUFFMAN_LUTSIZE - 1;
|
|
20
|
+
// class HuffmanConstructNode {
|
|
21
|
+
// node_id: number;
|
|
22
|
+
// frequency: number;
|
|
23
|
+
// constructor(node_id: number, frequency: number) {
|
|
24
|
+
// this.node_id = node_id;
|
|
25
|
+
// this.frequency = frequency;
|
|
26
|
+
// }
|
|
27
|
+
// }
|
|
28
|
+
// class HuffmanConstructNode:
|
|
29
|
+
// def __init__(self):
|
|
30
|
+
// self.node_id: int = None
|
|
31
|
+
// self.frequency: int = None
|
|
32
|
+
var Huffman = /** @class */ (function () {
|
|
33
|
+
function Huffman(frequencies) {
|
|
34
|
+
if (frequencies === void 0) { frequencies = FREQ_TABLE; }
|
|
35
|
+
// this.frequencies = frequencies;
|
|
36
|
+
this.nodes = new Array(HUFFMAN_MAX_NODES);
|
|
37
|
+
for (var i = 0; i < HUFFMAN_MAX_NODES; i++) {
|
|
38
|
+
this.nodes[i] = {};
|
|
39
|
+
}
|
|
40
|
+
this.decode_lut = new Array(HUFFMAN_LUTSIZE);
|
|
41
|
+
this.num_nodes = 0;
|
|
42
|
+
this.start_node_index = 0;
|
|
43
|
+
this.construct_tree(frequencies);
|
|
44
|
+
var xx = 0;
|
|
45
|
+
for (var i = 0; i < HUFFMAN_LUTSIZE; i++) {
|
|
46
|
+
var bits = i;
|
|
47
|
+
var broke = false;
|
|
48
|
+
var index = this.start_node_index;
|
|
49
|
+
for (var x = 0; x < HUFFMAN_LUTBITS; x++) {
|
|
50
|
+
if (bits & 1)
|
|
51
|
+
index = this.nodes[index].right;
|
|
52
|
+
else
|
|
53
|
+
index = this.nodes[index].left;
|
|
54
|
+
bits >>= 1;
|
|
55
|
+
if (this.nodes[index].numbits) {
|
|
56
|
+
this.decode_lut[i] = index;
|
|
57
|
+
broke = true;
|
|
58
|
+
break;
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
if (!broke) {
|
|
62
|
+
this.decode_lut[i] = index;
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
Huffman.prototype.set_bits_r = function (node_index, bits, depth) {
|
|
67
|
+
if (this.nodes[node_index].right != 0xffff)
|
|
68
|
+
this.set_bits_r(this.nodes[node_index].right, bits | (1 << depth), depth + 1);
|
|
69
|
+
if (this.nodes[node_index].left != 0xffff)
|
|
70
|
+
this.set_bits_r(this.nodes[node_index].left, bits, depth + 1);
|
|
71
|
+
if (this.nodes[node_index].numbits) {
|
|
72
|
+
this.nodes[node_index].bits = bits;
|
|
73
|
+
this.nodes[node_index].numbits = depth;
|
|
74
|
+
}
|
|
75
|
+
};
|
|
76
|
+
Huffman.prototype.bubble_sort = function (index_list, node_list, size) {
|
|
77
|
+
var changed = true;
|
|
78
|
+
while (changed) {
|
|
79
|
+
changed = false;
|
|
80
|
+
for (var i = 0; i < size - 1; i++) {
|
|
81
|
+
if (node_list[index_list[i]].frequency < node_list[index_list[i + 1]].frequency) {
|
|
82
|
+
var temp = index_list[i];
|
|
83
|
+
index_list[i] = index_list[i + 1];
|
|
84
|
+
index_list[i + 1] = temp;
|
|
85
|
+
changed = true;
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
size--;
|
|
89
|
+
}
|
|
90
|
+
return index_list;
|
|
91
|
+
};
|
|
92
|
+
Huffman.prototype.construct_tree = function (frequencies) {
|
|
93
|
+
if (frequencies === void 0) { frequencies = FREQ_TABLE; }
|
|
94
|
+
var nodes_left_storage = new Array(HUFFMAN_MAX_SYMBOLS);
|
|
95
|
+
for (var i = 0; i < HUFFMAN_MAX_SYMBOLS; i++) {
|
|
96
|
+
nodes_left_storage[i] = {};
|
|
97
|
+
}
|
|
98
|
+
var nodes_left = new Array(HUFFMAN_MAX_SYMBOLS);
|
|
99
|
+
var num_nodes_left = HUFFMAN_MAX_SYMBOLS;
|
|
100
|
+
for (var i = 0; i < HUFFMAN_MAX_SYMBOLS; i++) {
|
|
101
|
+
this.nodes[i].numbits = 0xFFFFFFFF;
|
|
102
|
+
this.nodes[i].symbol = i;
|
|
103
|
+
this.nodes[i].left = 0xFFFF;
|
|
104
|
+
this.nodes[i].right = 0xFFFF;
|
|
105
|
+
if (i == HUFFMAN_EOF_SYMBOL) {
|
|
106
|
+
nodes_left_storage[i].frequency = 1;
|
|
107
|
+
}
|
|
108
|
+
else
|
|
109
|
+
nodes_left_storage[i].frequency = frequencies[i];
|
|
110
|
+
nodes_left_storage[i].node_id = i;
|
|
111
|
+
nodes_left[i] = i;
|
|
112
|
+
}
|
|
113
|
+
this.num_nodes = HUFFMAN_MAX_SYMBOLS;
|
|
114
|
+
while (num_nodes_left > 1) {
|
|
115
|
+
nodes_left = this.bubble_sort(nodes_left, nodes_left_storage, num_nodes_left);
|
|
116
|
+
this.nodes[this.num_nodes].numbits = 0;
|
|
117
|
+
this.nodes[this.num_nodes].left = nodes_left_storage[nodes_left[num_nodes_left - 1]].node_id;
|
|
118
|
+
this.nodes[this.num_nodes].right = nodes_left_storage[nodes_left[num_nodes_left - 2]].node_id;
|
|
119
|
+
nodes_left_storage[nodes_left[num_nodes_left - 2]].node_id = this.num_nodes;
|
|
120
|
+
nodes_left_storage[nodes_left[num_nodes_left - 2]].frequency = nodes_left_storage[nodes_left[num_nodes_left - 1]].frequency
|
|
121
|
+
+ nodes_left_storage[nodes_left[num_nodes_left - 2]].frequency;
|
|
122
|
+
this.num_nodes++;
|
|
123
|
+
num_nodes_left--;
|
|
124
|
+
}
|
|
125
|
+
this.start_node_index = this.num_nodes - 1;
|
|
126
|
+
this.set_bits_r(this.start_node_index, 0, 0);
|
|
127
|
+
};
|
|
128
|
+
Huffman.prototype.compress = function (inp_buffer, start_index, size) {
|
|
129
|
+
if (start_index === void 0) { start_index = 0; }
|
|
130
|
+
if (size === void 0) { size = 0; }
|
|
131
|
+
var output = [];
|
|
132
|
+
var bits = 0;
|
|
133
|
+
var bitcount = 0;
|
|
134
|
+
if (size == 0)
|
|
135
|
+
size = inp_buffer.byteLength;
|
|
136
|
+
inp_buffer = inp_buffer.slice(start_index, start_index + size);
|
|
137
|
+
for (var i = 0; i < size; i++) {
|
|
138
|
+
var x = inp_buffer[i];
|
|
139
|
+
bits |= this.nodes[x].bits << bitcount;
|
|
140
|
+
bitcount += this.nodes[x].numbits;
|
|
141
|
+
while (bitcount >= 8) {
|
|
142
|
+
output.push(bits & 0xff);
|
|
143
|
+
bits >>= 8;
|
|
144
|
+
bitcount -= 8;
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
bits |= this.nodes[HUFFMAN_EOF_SYMBOL].bits << bitcount;
|
|
148
|
+
bitcount += this.nodes[HUFFMAN_EOF_SYMBOL].numbits;
|
|
149
|
+
while (bitcount >= 8) {
|
|
150
|
+
output.push(bits & 0xff);
|
|
151
|
+
bits >>= 8;
|
|
152
|
+
bitcount -= 8;
|
|
153
|
+
}
|
|
154
|
+
output.push(bits);
|
|
155
|
+
return Buffer.from(output);
|
|
156
|
+
/* output = bytearray()
|
|
157
|
+
bits = 0
|
|
158
|
+
bitcount = 0
|
|
159
|
+
|
|
160
|
+
if size is None:
|
|
161
|
+
size = len(inp_buffer)
|
|
162
|
+
|
|
163
|
+
for x in inp_buffer[start_index:size:]:
|
|
164
|
+
bits |= self.nodes[x].bits << bitcount
|
|
165
|
+
bitcount += self.nodes[x].numbits
|
|
166
|
+
|
|
167
|
+
while bitcount >= 8:
|
|
168
|
+
output.append(bits & 0xff)
|
|
169
|
+
bits >>= 8
|
|
170
|
+
bitcount -= 8
|
|
171
|
+
|
|
172
|
+
bits |= self.nodes[HUFFMAN_EOF_SYMBOL].bits << bitcount
|
|
173
|
+
bitcount += self.nodes[HUFFMAN_EOF_SYMBOL].numbits
|
|
174
|
+
|
|
175
|
+
while bitcount >= 8:
|
|
176
|
+
output.append(bits & 0xff)
|
|
177
|
+
bits >>= 8
|
|
178
|
+
bitcount -= 8
|
|
179
|
+
|
|
180
|
+
# write out last bits
|
|
181
|
+
output.append(bits)
|
|
182
|
+
return output
|
|
183
|
+
*/
|
|
184
|
+
};
|
|
185
|
+
Huffman.prototype.decompress = function (inp_buffer, size) {
|
|
186
|
+
if (size === void 0) { size = 0; }
|
|
187
|
+
var bits = 0;
|
|
188
|
+
var bitcount = 0;
|
|
189
|
+
var eof = this.nodes[HUFFMAN_EOF_SYMBOL];
|
|
190
|
+
var output = [];
|
|
191
|
+
if (size == 0)
|
|
192
|
+
size = inp_buffer.byteLength;
|
|
193
|
+
inp_buffer = inp_buffer.slice(0, size);
|
|
194
|
+
var src_index = 0;
|
|
195
|
+
while (true) {
|
|
196
|
+
var node_i = -1;
|
|
197
|
+
if (bitcount >= HUFFMAN_LUTBITS)
|
|
198
|
+
node_i = this.decode_lut[bits & HUFFMAN_LUTMASK];
|
|
199
|
+
while (bitcount < 24 && src_index != size) {
|
|
200
|
+
bits |= inp_buffer[src_index] << bitcount;
|
|
201
|
+
bitcount += 8;
|
|
202
|
+
src_index++;
|
|
203
|
+
}
|
|
204
|
+
if (node_i == -1)
|
|
205
|
+
node_i = this.decode_lut[bits & HUFFMAN_LUTMASK];
|
|
206
|
+
// console.log(node_i, bits & HUFFMAN_LUTMASK, this.decode_lut)
|
|
207
|
+
if (this.nodes[node_i].numbits) {
|
|
208
|
+
bits >>= this.nodes[node_i].numbits;
|
|
209
|
+
bitcount -= this.nodes[node_i].numbits;
|
|
210
|
+
}
|
|
211
|
+
else {
|
|
212
|
+
bits >>= HUFFMAN_LUTBITS;
|
|
213
|
+
bitcount -= HUFFMAN_LUTBITS;
|
|
214
|
+
while (true) {
|
|
215
|
+
if (bits & 1) {
|
|
216
|
+
node_i = this.nodes[node_i].right;
|
|
217
|
+
}
|
|
218
|
+
else
|
|
219
|
+
node_i = this.nodes[node_i].left;
|
|
220
|
+
bitcount -= 1;
|
|
221
|
+
bits >>= 1;
|
|
222
|
+
if (this.nodes[node_i].numbits)
|
|
223
|
+
break;
|
|
224
|
+
if (bitcount == 0)
|
|
225
|
+
throw new Error("No more bits, decoding error");
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
if (this.nodes[node_i] == eof)
|
|
229
|
+
break;
|
|
230
|
+
output.push(this.nodes[node_i].symbol);
|
|
231
|
+
}
|
|
232
|
+
return Buffer.from(output);
|
|
233
|
+
};
|
|
234
|
+
return Huffman;
|
|
235
|
+
}());
|
|
236
|
+
module.exports = Huffman;
|
package/lib/huffman.ts
ADDED
|
@@ -0,0 +1,342 @@
|
|
|
1
|
+
// from Ryozuki, found on ddnet discord ( 11.10.2018 )
|
|
2
|
+
// from typing import List
|
|
3
|
+
// from sys import argv
|
|
4
|
+
let FREQ_TABLE = [
|
|
5
|
+
1 << 30, 4545, 2657, 431, 1950, 919, 444, 482, 2244, 617, 838, 542, 715, 1814, 304, 240, 754, 212, 647, 186,
|
|
6
|
+
283, 131, 146, 166, 543, 164, 167, 136, 179, 859, 363, 113, 157, 154, 204, 108, 137, 180, 202, 176,
|
|
7
|
+
872, 404, 168, 134, 151, 111, 113, 109, 120, 126, 129, 100, 41, 20, 16, 22, 18, 18, 17, 19,
|
|
8
|
+
16, 37, 13, 21, 362, 166, 99, 78, 95, 88, 81, 70, 83, 284, 91, 187, 77, 68, 52, 68,
|
|
9
|
+
59, 66, 61, 638, 71, 157, 50, 46, 69, 43, 11, 24, 13, 19, 10, 12, 12, 20, 14, 9,
|
|
10
|
+
20, 20, 10, 10, 15, 15, 12, 12, 7, 19, 15, 14, 13, 18, 35, 19, 17, 14, 8, 5,
|
|
11
|
+
15, 17, 9, 15, 14, 18, 8, 10, 2173, 134, 157, 68, 188, 60, 170, 60, 194, 62, 175, 71,
|
|
12
|
+
148, 67, 167, 78, 211, 67, 156, 69, 1674, 90, 174, 53, 147, 89, 181, 51, 174, 63, 163, 80,
|
|
13
|
+
167, 94, 128, 122, 223, 153, 218, 77, 200, 110, 190, 73, 174, 69, 145, 66, 277, 143, 141, 60,
|
|
14
|
+
136, 53, 180, 57, 142, 57, 158, 61, 166, 112, 152, 92, 26, 22, 21, 28, 20, 26, 30, 21,
|
|
15
|
+
32, 27, 20, 17, 23, 21, 30, 22, 22, 21, 27, 25, 17, 27, 23, 18, 39, 26, 15, 21,
|
|
16
|
+
12, 18, 18, 27, 20, 18, 15, 19, 11, 17, 33, 12, 18, 15, 19, 18, 16, 26, 17, 18,
|
|
17
|
+
9, 10, 25, 22, 22, 17, 20, 16, 6, 16, 15, 20, 14, 18, 24, 335, 1517],
|
|
18
|
+
|
|
19
|
+
HUFFMAN_EOF_SYMBOL = 256,
|
|
20
|
+
HUFFMAN_MAX_SYMBOLS = HUFFMAN_EOF_SYMBOL + 1,
|
|
21
|
+
HUFFMAN_MAX_NODES = HUFFMAN_MAX_SYMBOLS * 2 - 1,
|
|
22
|
+
HUFFMAN_LUTBITS = 10,
|
|
23
|
+
HUFFMAN_LUTSIZE = 1 << HUFFMAN_LUTBITS,
|
|
24
|
+
HUFFMAN_LUTMASK = HUFFMAN_LUTSIZE - 1;
|
|
25
|
+
|
|
26
|
+
interface HuffmanNode {
|
|
27
|
+
bits: number;
|
|
28
|
+
numbits: number;
|
|
29
|
+
left: number;
|
|
30
|
+
right: number;
|
|
31
|
+
symbol: number;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
// class HuffmanNode {
|
|
35
|
+
// bits: number;
|
|
36
|
+
// numbits: number;
|
|
37
|
+
// left: number;
|
|
38
|
+
// right: number;
|
|
39
|
+
// symbol: number;
|
|
40
|
+
// constructor(/*bits: number, numbits: number, left: number, right: number, symbol: number*/) {
|
|
41
|
+
// this.bits = -1;
|
|
42
|
+
// this.numbits = -1;
|
|
43
|
+
// this.left = -1;
|
|
44
|
+
// this.right = -1;
|
|
45
|
+
// this.symbol = -1;
|
|
46
|
+
// }
|
|
47
|
+
|
|
48
|
+
// equal(node: HuffmanNode) {
|
|
49
|
+
// return this.bits == node.bits && this.numbits == node.numbits && this.left == node.left && this.right == node.right && this.symbol == node.symbol;
|
|
50
|
+
// }
|
|
51
|
+
|
|
52
|
+
// }
|
|
53
|
+
|
|
54
|
+
interface HuffmanConstructNode {
|
|
55
|
+
node_id: number;
|
|
56
|
+
frequency: number;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
// class HuffmanConstructNode {
|
|
60
|
+
// node_id: number;
|
|
61
|
+
// frequency: number;
|
|
62
|
+
// constructor(node_id: number, frequency: number) {
|
|
63
|
+
// this.node_id = node_id;
|
|
64
|
+
// this.frequency = frequency;
|
|
65
|
+
// }
|
|
66
|
+
// }
|
|
67
|
+
// class HuffmanConstructNode:
|
|
68
|
+
// def __init__(self):
|
|
69
|
+
// self.node_id: int = None
|
|
70
|
+
// self.frequency: int = None
|
|
71
|
+
|
|
72
|
+
class Huffman {
|
|
73
|
+
nodes: HuffmanNode[];
|
|
74
|
+
decode_lut: number[];
|
|
75
|
+
num_nodes: number;
|
|
76
|
+
start_node_index: number;
|
|
77
|
+
|
|
78
|
+
constructor(frequencies = FREQ_TABLE) {
|
|
79
|
+
// this.frequencies = frequencies;
|
|
80
|
+
this.nodes = new Array<HuffmanNode>(HUFFMAN_MAX_NODES);
|
|
81
|
+
for (let i = 0; i < HUFFMAN_MAX_NODES; i++) {
|
|
82
|
+
this.nodes[i] = {} as HuffmanNode;
|
|
83
|
+
}
|
|
84
|
+
this.decode_lut = new Array<number>(HUFFMAN_LUTSIZE);
|
|
85
|
+
this.num_nodes = 0;
|
|
86
|
+
this.start_node_index = 0;
|
|
87
|
+
|
|
88
|
+
this.construct_tree(frequencies);
|
|
89
|
+
|
|
90
|
+
let xx = 0;
|
|
91
|
+
|
|
92
|
+
for (let i = 0; i < HUFFMAN_LUTSIZE; i++) {
|
|
93
|
+
let bits = i;
|
|
94
|
+
let broke = false;
|
|
95
|
+
let index = this.start_node_index;
|
|
96
|
+
for (let x = 0; x < HUFFMAN_LUTBITS; x++) {
|
|
97
|
+
if (bits & 1)
|
|
98
|
+
index = this.nodes[index].right;
|
|
99
|
+
else
|
|
100
|
+
index = this.nodes[index].left;
|
|
101
|
+
bits >>= 1;
|
|
102
|
+
if (this.nodes[index].numbits) {
|
|
103
|
+
this.decode_lut[i] = index;
|
|
104
|
+
broke = true;
|
|
105
|
+
break;
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
if (!broke) {
|
|
109
|
+
this.decode_lut[i] = index;
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
set_bits_r(node_index: number, bits: number, depth: number) {
|
|
114
|
+
if (this.nodes[node_index].right != 0xffff)
|
|
115
|
+
this.set_bits_r(this.nodes[node_index].right, bits | (1 << depth), depth + 1)
|
|
116
|
+
if (this.nodes[node_index].left != 0xffff)
|
|
117
|
+
this.set_bits_r(this.nodes[node_index].left, bits, depth + 1)
|
|
118
|
+
if (this.nodes[node_index].numbits) {
|
|
119
|
+
this.nodes[node_index].bits = bits;
|
|
120
|
+
this.nodes[node_index].numbits = depth;
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
bubble_sort(index_list: number[], node_list: HuffmanConstructNode[], size: number) {
|
|
125
|
+
let changed = true;
|
|
126
|
+
while (changed) {
|
|
127
|
+
changed = false;
|
|
128
|
+
for (let i = 0; i < size-1; i++) {
|
|
129
|
+
if (node_list[index_list[i]].frequency < node_list[index_list[i + 1]].frequency) {
|
|
130
|
+
let temp = index_list[i];
|
|
131
|
+
index_list[i] = index_list[i + 1];
|
|
132
|
+
index_list[i + 1] = temp;
|
|
133
|
+
changed = true;
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
size--;
|
|
137
|
+
}
|
|
138
|
+
return index_list;
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
construct_tree(frequencies = FREQ_TABLE) {
|
|
142
|
+
let nodes_left_storage: HuffmanConstructNode[] = new Array<HuffmanConstructNode>(HUFFMAN_MAX_SYMBOLS);
|
|
143
|
+
for (let i = 0; i < HUFFMAN_MAX_SYMBOLS; i++) {
|
|
144
|
+
nodes_left_storage[i] = {} as HuffmanConstructNode;
|
|
145
|
+
}
|
|
146
|
+
let nodes_left: number[] = new Array<number>(HUFFMAN_MAX_SYMBOLS);
|
|
147
|
+
let num_nodes_left = HUFFMAN_MAX_SYMBOLS;
|
|
148
|
+
|
|
149
|
+
for (let i = 0; i < HUFFMAN_MAX_SYMBOLS; i++) {
|
|
150
|
+
this.nodes[i].numbits = 0xFFFFFFFF;
|
|
151
|
+
this.nodes[i].symbol = i;
|
|
152
|
+
this.nodes[i].left = 0xFFFF;
|
|
153
|
+
this.nodes[i].right = 0xFFFF;
|
|
154
|
+
|
|
155
|
+
if (i == HUFFMAN_EOF_SYMBOL) {
|
|
156
|
+
nodes_left_storage[i].frequency = 1;
|
|
157
|
+
} else
|
|
158
|
+
nodes_left_storage[i].frequency = frequencies[i];
|
|
159
|
+
nodes_left_storage[i].node_id = i;
|
|
160
|
+
nodes_left[i] = i;
|
|
161
|
+
}
|
|
162
|
+
this.num_nodes = HUFFMAN_MAX_SYMBOLS;
|
|
163
|
+
|
|
164
|
+
while (num_nodes_left > 1) {
|
|
165
|
+
nodes_left = this.bubble_sort(nodes_left, nodes_left_storage, num_nodes_left);
|
|
166
|
+
|
|
167
|
+
this.nodes[this.num_nodes].numbits = 0;
|
|
168
|
+
this.nodes[this.num_nodes].left = nodes_left_storage[nodes_left[num_nodes_left - 1]].node_id
|
|
169
|
+
this.nodes[this.num_nodes].right = nodes_left_storage[nodes_left[num_nodes_left - 2]].node_id
|
|
170
|
+
|
|
171
|
+
nodes_left_storage[nodes_left[num_nodes_left - 2]].node_id = this.num_nodes;
|
|
172
|
+
nodes_left_storage[nodes_left[num_nodes_left - 2]].frequency = nodes_left_storage[nodes_left[num_nodes_left - 1]].frequency
|
|
173
|
+
+ nodes_left_storage[nodes_left[num_nodes_left - 2]].frequency
|
|
174
|
+
this.num_nodes++;
|
|
175
|
+
num_nodes_left--;
|
|
176
|
+
}
|
|
177
|
+
this.start_node_index = this.num_nodes-1;
|
|
178
|
+
this.set_bits_r(this.start_node_index, 0, 0);
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
compress(inp_buffer: Buffer, start_index = 0, size: number = 0): Buffer {
|
|
182
|
+
let output = [];
|
|
183
|
+
let bits = 0;
|
|
184
|
+
let bitcount = 0;
|
|
185
|
+
|
|
186
|
+
if (size == 0)
|
|
187
|
+
size = inp_buffer.byteLength;
|
|
188
|
+
inp_buffer = inp_buffer.slice(start_index, start_index + size);
|
|
189
|
+
for (let i = 0; i < size; i++) {
|
|
190
|
+
let x = inp_buffer[i];
|
|
191
|
+
bits |= this.nodes[x].bits << bitcount;
|
|
192
|
+
bitcount += this.nodes[x].numbits;
|
|
193
|
+
|
|
194
|
+
while (bitcount >= 8) {
|
|
195
|
+
output.push(bits & 0xff);
|
|
196
|
+
bits >>= 8;
|
|
197
|
+
bitcount -= 8;
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
bits |= this.nodes[HUFFMAN_EOF_SYMBOL].bits << bitcount;
|
|
201
|
+
bitcount += this.nodes[HUFFMAN_EOF_SYMBOL].numbits;
|
|
202
|
+
|
|
203
|
+
while (bitcount >= 8) {
|
|
204
|
+
output.push(bits & 0xff);
|
|
205
|
+
bits >>= 8;
|
|
206
|
+
bitcount -= 8;
|
|
207
|
+
}
|
|
208
|
+
output.push(bits);
|
|
209
|
+
return Buffer.from(output);
|
|
210
|
+
|
|
211
|
+
/* output = bytearray()
|
|
212
|
+
bits = 0
|
|
213
|
+
bitcount = 0
|
|
214
|
+
|
|
215
|
+
if size is None:
|
|
216
|
+
size = len(inp_buffer)
|
|
217
|
+
|
|
218
|
+
for x in inp_buffer[start_index:size:]:
|
|
219
|
+
bits |= self.nodes[x].bits << bitcount
|
|
220
|
+
bitcount += self.nodes[x].numbits
|
|
221
|
+
|
|
222
|
+
while bitcount >= 8:
|
|
223
|
+
output.append(bits & 0xff)
|
|
224
|
+
bits >>= 8
|
|
225
|
+
bitcount -= 8
|
|
226
|
+
|
|
227
|
+
bits |= self.nodes[HUFFMAN_EOF_SYMBOL].bits << bitcount
|
|
228
|
+
bitcount += self.nodes[HUFFMAN_EOF_SYMBOL].numbits
|
|
229
|
+
|
|
230
|
+
while bitcount >= 8:
|
|
231
|
+
output.append(bits & 0xff)
|
|
232
|
+
bits >>= 8
|
|
233
|
+
bitcount -= 8
|
|
234
|
+
|
|
235
|
+
# write out last bits
|
|
236
|
+
output.append(bits)
|
|
237
|
+
return output
|
|
238
|
+
*/
|
|
239
|
+
}
|
|
240
|
+
decompress(inp_buffer: Buffer, size = 0): Buffer {
|
|
241
|
+
|
|
242
|
+
let bits = 0;
|
|
243
|
+
let bitcount = 0;
|
|
244
|
+
let eof = this.nodes[HUFFMAN_EOF_SYMBOL];
|
|
245
|
+
let output = [];
|
|
246
|
+
|
|
247
|
+
if (size == 0)
|
|
248
|
+
size = inp_buffer.byteLength;
|
|
249
|
+
inp_buffer = inp_buffer.slice(0, size);
|
|
250
|
+
let src_index = 0;
|
|
251
|
+
while (true) {
|
|
252
|
+
let node_i = -1;
|
|
253
|
+
if (bitcount >= HUFFMAN_LUTBITS)
|
|
254
|
+
node_i = this.decode_lut[bits & HUFFMAN_LUTMASK];
|
|
255
|
+
while (bitcount < 24 && src_index != size) {
|
|
256
|
+
bits |= inp_buffer[src_index] << bitcount;
|
|
257
|
+
bitcount += 8;
|
|
258
|
+
src_index++;
|
|
259
|
+
}
|
|
260
|
+
if (node_i == -1)
|
|
261
|
+
node_i = this.decode_lut[bits & HUFFMAN_LUTMASK];
|
|
262
|
+
// console.log(node_i, bits & HUFFMAN_LUTMASK, this.decode_lut)
|
|
263
|
+
if (this.nodes[node_i].numbits) {
|
|
264
|
+
bits >>= this.nodes[node_i].numbits;
|
|
265
|
+
bitcount -= this.nodes[node_i].numbits;
|
|
266
|
+
} else {
|
|
267
|
+
bits >>= HUFFMAN_LUTBITS;
|
|
268
|
+
bitcount -= HUFFMAN_LUTBITS;
|
|
269
|
+
|
|
270
|
+
while (true) {
|
|
271
|
+
if (bits & 1) {
|
|
272
|
+
node_i = this.nodes[node_i].right;
|
|
273
|
+
} else
|
|
274
|
+
node_i = this.nodes[node_i].left;
|
|
275
|
+
bitcount -= 1;
|
|
276
|
+
bits >>= 1;
|
|
277
|
+
|
|
278
|
+
if (this.nodes[node_i].numbits)
|
|
279
|
+
break;
|
|
280
|
+
if (bitcount == 0)
|
|
281
|
+
throw new Error("No more bits, decoding error")
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
}
|
|
285
|
+
if (this.nodes[node_i] == eof)
|
|
286
|
+
break;
|
|
287
|
+
output.push(this.nodes[node_i].symbol);
|
|
288
|
+
}
|
|
289
|
+
return Buffer.from(output);
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
/*bits = 0
|
|
293
|
+
bitcount = 0
|
|
294
|
+
eof = self.nodes[HUFFMAN_EOF_SYMBOL]
|
|
295
|
+
output = bytearray()
|
|
296
|
+
|
|
297
|
+
if size is None:
|
|
298
|
+
size = len(inp_buffer)
|
|
299
|
+
|
|
300
|
+
src_index = 0
|
|
301
|
+
while True:
|
|
302
|
+
node_i = None
|
|
303
|
+
if bitcount >= HUFFMAN_LUTBITS:
|
|
304
|
+
node_i = self.decode_lut[bits & HUFFMAN_LUTMASK]
|
|
305
|
+
|
|
306
|
+
while bitcount < 24 and src_index != size:
|
|
307
|
+
bits |= inp_buffer[src_index] << bitcount
|
|
308
|
+
src_index += 1
|
|
309
|
+
bitcount += 8
|
|
310
|
+
|
|
311
|
+
if node_i is None:
|
|
312
|
+
node_i = self.decode_lut[bits & HUFFMAN_LUTMASK]
|
|
313
|
+
if self.nodes[node_i].numbits:
|
|
314
|
+
bits >>= self.nodes[node_i].numbits
|
|
315
|
+
bitcount -= self.nodes[node_i].numbits
|
|
316
|
+
else:
|
|
317
|
+
bits >>= HUFFMAN_LUTBITS
|
|
318
|
+
bitcount -= HUFFMAN_LUTBITS
|
|
319
|
+
|
|
320
|
+
while True:
|
|
321
|
+
if bits & 1:
|
|
322
|
+
node_i = self.nodes[node_i].right
|
|
323
|
+
else:
|
|
324
|
+
node_i = self.nodes[node_i].left
|
|
325
|
+
|
|
326
|
+
bitcount -= 1
|
|
327
|
+
bits >>= 1
|
|
328
|
+
|
|
329
|
+
if self.nodes[node_i].numbits:
|
|
330
|
+
break
|
|
331
|
+
|
|
332
|
+
if bitcount === 0:
|
|
333
|
+
raise ValueError("No more bits, decoding error")
|
|
334
|
+
|
|
335
|
+
if self.nodes[node_i] === eof:
|
|
336
|
+
break
|
|
337
|
+
output.append(self.nodes[node_i].symbol)
|
|
338
|
+
|
|
339
|
+
return output */
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
export = Huffman;
|
package/package.json
CHANGED
package/lib/huffman.py
DELETED
|
@@ -1,224 +0,0 @@
|
|
|
1
|
-
# from Ryozuki, found on ddnet discord ( 11.10.2018 )
|
|
2
|
-
from typing import List
|
|
3
|
-
from sys import argv
|
|
4
|
-
FREQ_TABLE = [
|
|
5
|
-
1 << 30, 4545, 2657, 431, 1950, 919, 444, 482, 2244, 617, 838, 542, 715, 1814, 304, 240, 754, 212, 647, 186,
|
|
6
|
-
283, 131, 146, 166, 543, 164, 167, 136, 179, 859, 363, 113, 157, 154, 204, 108, 137, 180, 202, 176,
|
|
7
|
-
872, 404, 168, 134, 151, 111, 113, 109, 120, 126, 129, 100, 41, 20, 16, 22, 18, 18, 17, 19,
|
|
8
|
-
16, 37, 13, 21, 362, 166, 99, 78, 95, 88, 81, 70, 83, 284, 91, 187, 77, 68, 52, 68,
|
|
9
|
-
59, 66, 61, 638, 71, 157, 50, 46, 69, 43, 11, 24, 13, 19, 10, 12, 12, 20, 14, 9,
|
|
10
|
-
20, 20, 10, 10, 15, 15, 12, 12, 7, 19, 15, 14, 13, 18, 35, 19, 17, 14, 8, 5,
|
|
11
|
-
15, 17, 9, 15, 14, 18, 8, 10, 2173, 134, 157, 68, 188, 60, 170, 60, 194, 62, 175, 71,
|
|
12
|
-
148, 67, 167, 78, 211, 67, 156, 69, 1674, 90, 174, 53, 147, 89, 181, 51, 174, 63, 163, 80,
|
|
13
|
-
167, 94, 128, 122, 223, 153, 218, 77, 200, 110, 190, 73, 174, 69, 145, 66, 277, 143, 141, 60,
|
|
14
|
-
136, 53, 180, 57, 142, 57, 158, 61, 166, 112, 152, 92, 26, 22, 21, 28, 20, 26, 30, 21,
|
|
15
|
-
32, 27, 20, 17, 23, 21, 30, 22, 22, 21, 27, 25, 17, 27, 23, 18, 39, 26, 15, 21,
|
|
16
|
-
12, 18, 18, 27, 20, 18, 15, 19, 11, 17, 33, 12, 18, 15, 19, 18, 16, 26, 17, 18,
|
|
17
|
-
9, 10, 25, 22, 22, 17, 20, 16, 6, 16, 15, 20, 14, 18, 24, 335, 1517]
|
|
18
|
-
|
|
19
|
-
HUFFMAN_EOF_SYMBOL = 256
|
|
20
|
-
HUFFMAN_MAX_SYMBOLS = HUFFMAN_EOF_SYMBOL + 1
|
|
21
|
-
HUFFMAN_MAX_NODES = HUFFMAN_MAX_SYMBOLS * 2 - 1
|
|
22
|
-
HUFFMAN_LUTBITS = 10
|
|
23
|
-
HUFFMAN_LUTSIZE = 1 << HUFFMAN_LUTBITS
|
|
24
|
-
HUFFMAN_LUTMASK = HUFFMAN_LUTSIZE - 1
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
class Node:
|
|
28
|
-
def __init__(self):
|
|
29
|
-
self.bits: int = None
|
|
30
|
-
self.numbits: int = None
|
|
31
|
-
# Leafs
|
|
32
|
-
self.left: int = None
|
|
33
|
-
self.right: int = None
|
|
34
|
-
|
|
35
|
-
self.symbol: int = None
|
|
36
|
-
|
|
37
|
-
def __eq__(self, other):
|
|
38
|
-
return self.symbol == other.symbol
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
class HuffmanConstructNode:
|
|
42
|
-
def __init__(self):
|
|
43
|
-
self.node_id: int = None
|
|
44
|
-
self.frequency: int = None
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
class Huffman:
|
|
48
|
-
def __init__(self, frequencies: List[int]):
|
|
49
|
-
self.nodes: List[Node] = [Node() for _ in range(HUFFMAN_MAX_NODES)]
|
|
50
|
-
# list of index of nodes
|
|
51
|
-
self.decode_lut: List[int] = [None for _ in range(HUFFMAN_LUTSIZE)]
|
|
52
|
-
self.num_nodes: int = None
|
|
53
|
-
self.start_node_index: int = None
|
|
54
|
-
|
|
55
|
-
self.construct_tree(frequencies)
|
|
56
|
-
# print(self.num_nodes, "numnodes")
|
|
57
|
-
# print(self.start_node_index, "nodeindex")
|
|
58
|
-
# print(len(self.decode_lut), "dcodelut")
|
|
59
|
-
xx = 0
|
|
60
|
-
for i in range(HUFFMAN_LUTSIZE):
|
|
61
|
-
bits = i
|
|
62
|
-
broke = False
|
|
63
|
-
index = self.start_node_index
|
|
64
|
-
for x in range(HUFFMAN_LUTBITS):
|
|
65
|
-
if bits & 1:
|
|
66
|
-
index = self.nodes[index].right
|
|
67
|
-
else:
|
|
68
|
-
index = self.nodes[index].left
|
|
69
|
-
# print(index)
|
|
70
|
-
bits >>= 1
|
|
71
|
-
|
|
72
|
-
if self.nodes[index].numbits:
|
|
73
|
-
# print(i, bits)
|
|
74
|
-
self.decode_lut[i] = index
|
|
75
|
-
xx += 1
|
|
76
|
-
broke = True
|
|
77
|
-
break
|
|
78
|
-
if not broke:
|
|
79
|
-
self.decode_lut[i] = index
|
|
80
|
-
# print(xx, HUFFMAN_LUTSIZE, HUFFMAN_LUTBITS)
|
|
81
|
-
def set_bits_r(self, node_index: int, bits: int, depth: int):
|
|
82
|
-
if self.nodes[node_index].right != 0xffff:
|
|
83
|
-
self.set_bits_r(self.nodes[node_index].right, bits | (1 << depth), depth + 1)
|
|
84
|
-
if self.nodes[node_index].left != 0xffff:
|
|
85
|
-
self.set_bits_r(self.nodes[node_index].left, bits, depth + 1)
|
|
86
|
-
|
|
87
|
-
if self.nodes[node_index].numbits:
|
|
88
|
-
self.nodes[node_index].bits = bits
|
|
89
|
-
self.nodes[node_index].numbits = depth
|
|
90
|
-
|
|
91
|
-
@staticmethod
|
|
92
|
-
def bubble_sort(index_list: List[int], node_list: List[HuffmanConstructNode], size: int):
|
|
93
|
-
changed = True
|
|
94
|
-
while changed:
|
|
95
|
-
changed = False
|
|
96
|
-
for i in range(size - 1):
|
|
97
|
-
if node_list[index_list[i]].frequency < node_list[index_list[i + 1]].frequency:
|
|
98
|
-
index_list[i], index_list[i + 1] = index_list[i + 1], index_list[i]
|
|
99
|
-
changed = True
|
|
100
|
-
size -= 1
|
|
101
|
-
return index_list
|
|
102
|
-
|
|
103
|
-
def construct_tree(self, frequencies: List[int]):
|
|
104
|
-
nodes_left_storage: List[HuffmanConstructNode] = [HuffmanConstructNode() for _ in range(HUFFMAN_MAX_SYMBOLS)]
|
|
105
|
-
nodes_left: List[int] = [None for _ in range(HUFFMAN_MAX_SYMBOLS)]
|
|
106
|
-
num_nodes_left = HUFFMAN_MAX_SYMBOLS
|
|
107
|
-
|
|
108
|
-
for i in range(HUFFMAN_MAX_SYMBOLS):
|
|
109
|
-
self.nodes[i].numbits = 0xFFFFFFFF
|
|
110
|
-
self.nodes[i].symbol = i
|
|
111
|
-
self.nodes[i].left = 0xffff
|
|
112
|
-
self.nodes[i].right = 0xffff
|
|
113
|
-
|
|
114
|
-
if i == HUFFMAN_EOF_SYMBOL:
|
|
115
|
-
nodes_left_storage[i].frequency = 1
|
|
116
|
-
else:
|
|
117
|
-
nodes_left_storage[i].frequency = frequencies[i]
|
|
118
|
-
nodes_left_storage[i].node_id = i
|
|
119
|
-
nodes_left[i] = i
|
|
120
|
-
|
|
121
|
-
self.num_nodes = HUFFMAN_MAX_SYMBOLS
|
|
122
|
-
|
|
123
|
-
while num_nodes_left > 1:
|
|
124
|
-
nodes_left = Huffman.bubble_sort(nodes_left, nodes_left_storage, num_nodes_left)
|
|
125
|
-
|
|
126
|
-
self.nodes[self.num_nodes].numbits = 0
|
|
127
|
-
self.nodes[self.num_nodes].left = nodes_left_storage[nodes_left[num_nodes_left - 1]].node_id
|
|
128
|
-
self.nodes[self.num_nodes].right = nodes_left_storage[nodes_left[num_nodes_left - 2]].node_id
|
|
129
|
-
nodes_left_storage[nodes_left[num_nodes_left - 2]].node_id = self.num_nodes
|
|
130
|
-
nodes_left_storage[nodes_left[num_nodes_left - 2]].frequency = \
|
|
131
|
-
nodes_left_storage[nodes_left[num_nodes_left - 1]].frequency \
|
|
132
|
-
+ nodes_left_storage[nodes_left[num_nodes_left - 2]].frequency
|
|
133
|
-
self.num_nodes += 1
|
|
134
|
-
num_nodes_left -= 1
|
|
135
|
-
self.start_node_index = self.num_nodes - 1
|
|
136
|
-
self.set_bits_r(self.start_node_index, 0, 0)
|
|
137
|
-
|
|
138
|
-
def compress(self, inp_buffer: bytearray, start_index: int = 0, size: int = None):
|
|
139
|
-
output = bytearray()
|
|
140
|
-
bits = 0
|
|
141
|
-
bitcount = 0
|
|
142
|
-
|
|
143
|
-
if size is None:
|
|
144
|
-
size = len(inp_buffer)
|
|
145
|
-
|
|
146
|
-
for x in inp_buffer[start_index:size:]:
|
|
147
|
-
# print(x)
|
|
148
|
-
bits |= self.nodes[x].bits << bitcount
|
|
149
|
-
bitcount += self.nodes[x].numbits
|
|
150
|
-
|
|
151
|
-
while bitcount >= 8:
|
|
152
|
-
output.append(bits & 0xff)
|
|
153
|
-
bits >>= 8
|
|
154
|
-
bitcount -= 8
|
|
155
|
-
|
|
156
|
-
bits |= self.nodes[HUFFMAN_EOF_SYMBOL].bits << bitcount
|
|
157
|
-
bitcount += self.nodes[HUFFMAN_EOF_SYMBOL].numbits
|
|
158
|
-
|
|
159
|
-
while bitcount >= 8:
|
|
160
|
-
output.append(bits & 0xff)
|
|
161
|
-
bits >>= 8
|
|
162
|
-
bitcount -= 8
|
|
163
|
-
|
|
164
|
-
# write out last bits
|
|
165
|
-
output.append(bits)
|
|
166
|
-
return output
|
|
167
|
-
|
|
168
|
-
def decompress(self, inp_buffer: bytearray, size: int = None):
|
|
169
|
-
bits = 0
|
|
170
|
-
bitcount = 0
|
|
171
|
-
eof = self.nodes[HUFFMAN_EOF_SYMBOL]
|
|
172
|
-
output = bytearray()
|
|
173
|
-
|
|
174
|
-
if size is None:
|
|
175
|
-
size = len(inp_buffer)
|
|
176
|
-
|
|
177
|
-
src_index = 0
|
|
178
|
-
while True:
|
|
179
|
-
node_i = None
|
|
180
|
-
if bitcount >= HUFFMAN_LUTBITS:
|
|
181
|
-
node_i = self.decode_lut[bits & HUFFMAN_LUTMASK]
|
|
182
|
-
# print("yes im here!", output, node_i, bits & HUFFMAN_LUTMASK)
|
|
183
|
-
|
|
184
|
-
while bitcount < 24 and src_index != size:
|
|
185
|
-
bits |= inp_buffer[src_index] << bitcount
|
|
186
|
-
src_index += 1
|
|
187
|
-
bitcount += 8
|
|
188
|
-
|
|
189
|
-
# print(self.decode_lut)
|
|
190
|
-
if node_i is None:
|
|
191
|
-
node_i = self.decode_lut[bits & HUFFMAN_LUTMASK]
|
|
192
|
-
if self.nodes[node_i].numbits:
|
|
193
|
-
bits >>= self.nodes[node_i].numbits
|
|
194
|
-
bitcount -= self.nodes[node_i].numbits
|
|
195
|
-
else:
|
|
196
|
-
bits >>= HUFFMAN_LUTBITS
|
|
197
|
-
bitcount -= HUFFMAN_LUTBITS
|
|
198
|
-
# print("yes im here!", output)
|
|
199
|
-
|
|
200
|
-
while True:
|
|
201
|
-
if bits & 1:
|
|
202
|
-
node_i = self.nodes[node_i].right
|
|
203
|
-
else:
|
|
204
|
-
node_i = self.nodes[node_i].left
|
|
205
|
-
|
|
206
|
-
bitcount -= 1
|
|
207
|
-
bits >>= 1
|
|
208
|
-
|
|
209
|
-
if self.nodes[node_i].numbits:
|
|
210
|
-
break
|
|
211
|
-
|
|
212
|
-
if bitcount == 0:
|
|
213
|
-
raise ValueError("No more bits, decoding error")
|
|
214
|
-
|
|
215
|
-
if self.nodes[node_i] == eof:
|
|
216
|
-
break
|
|
217
|
-
output.append(self.nodes[node_i].symbol)
|
|
218
|
-
|
|
219
|
-
return output
|
|
220
|
-
huff = Huffman(frequencies=FREQ_TABLE)
|
|
221
|
-
if (argv[1] and argv[2] == "-decompress"):
|
|
222
|
-
print(list(huff.decompress(bytearray.fromhex(argv[1]))))
|
|
223
|
-
elif (argv[1] and argv[2] == "-compress"):
|
|
224
|
-
print(list(huff.compress(bytearray.fromhex(argv[1]))))
|