hacktools 0.40.1__tar.gz → 1.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {hacktools-0.40.1 → hacktools-1.0.1}/LICENSE +1 -1
- {hacktools-0.40.1/hacktools.egg-info → hacktools-1.0.1}/PKG-INFO +5 -4
- {hacktools-0.40.1 → hacktools-1.0.1}/README.md +2 -0
- hacktools-1.0.1/hacktools/__init__.py +1 -0
- {hacktools-0.40.1 → hacktools-1.0.1}/hacktools/arch.py +94 -18
- {hacktools-0.40.1 → hacktools-1.0.1}/hacktools/c_ext/cmp_cri.c +32 -5
- hacktools-1.0.1/hacktools/c_ext/cmp_huff.c +405 -0
- {hacktools-0.40.1 → hacktools-1.0.1}/hacktools/c_ext/cmp_lzss.c +87 -29
- hacktools-1.0.1/hacktools/c_ext/cmp_misc.c +163 -0
- hacktools-1.0.1/hacktools/c_ext/cmp_prs.c +370 -0
- {hacktools-0.40.1 → hacktools-1.0.1}/hacktools/c_ext/cmp_racjin.c +32 -12
- hacktools-1.0.1/hacktools/c_ext/inc.h +26 -0
- {hacktools-0.40.1 → hacktools-1.0.1}/hacktools/common.py +963 -161
- {hacktools-0.40.1 → hacktools-1.0.1}/hacktools/cpk.py +378 -71
- {hacktools-0.40.1 → hacktools-1.0.1}/hacktools/gb.py +44 -3
- {hacktools-0.40.1 → hacktools-1.0.1}/hacktools/gui.py +403 -304
- {hacktools-0.40.1 → hacktools-1.0.1}/hacktools/nasm.py +20 -11
- {hacktools-0.40.1 → hacktools-1.0.1}/hacktools/nds.py +219 -30
- {hacktools-0.40.1 → hacktools-1.0.1}/hacktools/nitro.py +826 -165
- hacktools-1.0.1/hacktools/ps2.py +204 -0
- {hacktools-0.40.1 → hacktools-1.0.1}/hacktools/psp.py +602 -104
- {hacktools-0.40.1 → hacktools-1.0.1}/hacktools/psx.py +455 -295
- hacktools-1.0.1/hacktools/wii.py +756 -0
- {hacktools-0.40.1 → hacktools-1.0.1}/hacktools/ws.py +310 -40
- {hacktools-0.40.1 → hacktools-1.0.1/hacktools.egg-info}/PKG-INFO +5 -4
- {hacktools-0.40.1 → hacktools-1.0.1}/hacktools.egg-info/SOURCES.txt +4 -1
- hacktools-1.0.1/pyproject.toml +25 -0
- {hacktools-0.40.1 → hacktools-1.0.1}/setup.py +6 -6
- hacktools-1.0.1/tests/test_cmp.py +103 -0
- hacktools-0.40.1/hacktools/__init__.py +0 -1
- hacktools-0.40.1/hacktools/c_ext/cmp_misc.c +0 -60
- hacktools-0.40.1/hacktools/c_ext/inc.h +0 -11
- hacktools-0.40.1/hacktools/compression.py +0 -216
- hacktools-0.40.1/hacktools/wii.py +0 -285
- hacktools-0.40.1/tests/test_cmp.py +0 -37
- {hacktools-0.40.1 → hacktools-1.0.1}/MANIFEST.in +0 -0
- {hacktools-0.40.1 → hacktools-1.0.1}/hacktools/assets/icon.png +0 -0
- {hacktools-0.40.1 → hacktools-1.0.1}/hacktools.egg-info/dependency_links.txt +0 -0
- {hacktools-0.40.1 → hacktools-1.0.1}/hacktools.egg-info/requires.txt +0 -0
- {hacktools-0.40.1 → hacktools-1.0.1}/hacktools.egg-info/top_level.txt +0 -0
- {hacktools-0.40.1 → hacktools-1.0.1}/setup.cfg +0 -0
|
@@ -1,19 +1,18 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: hacktools
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 1.0.1
|
|
4
4
|
Summary: A set of utilities and tools for rom hacking and translations.
|
|
5
5
|
Home-page: https://github.com/Illidanz/hacktools
|
|
6
6
|
Author: Illidan
|
|
7
7
|
License: MIT
|
|
8
|
-
Classifier: Programming Language :: Python :: 3.8
|
|
9
|
-
Classifier: Programming Language :: Python :: 3.9
|
|
10
8
|
Classifier: Programming Language :: Python :: 3.10
|
|
11
9
|
Classifier: Programming Language :: Python :: 3.11
|
|
12
10
|
Classifier: Programming Language :: Python :: 3.12
|
|
13
11
|
Classifier: Programming Language :: Python :: 3.13
|
|
14
12
|
Classifier: Programming Language :: Python :: 3.14
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.15
|
|
15
14
|
Classifier: Operating System :: OS Independent
|
|
16
|
-
Requires-Python: >=3.
|
|
15
|
+
Requires-Python: >=3.10
|
|
17
16
|
Description-Content-Type: text/markdown
|
|
18
17
|
License-File: LICENSE
|
|
19
18
|
Provides-Extra: nds
|
|
@@ -61,6 +60,8 @@ Dynamic: summary
|
|
|
61
60
|
# hacktools
|
|
62
61
|
A set of utilities and tools for rom hacking and translations.
|
|
63
62
|
|
|
63
|
+
Documentation is available at [illidanz.github.io/hacktools](https://illidanz.github.io/hacktools/).
|
|
64
|
+
|
|
64
65
|
## Installing dependencies
|
|
65
66
|
Most dependencies are optional, and can be installed with `pip install 'hacktools[name1,name2]'` or `pip install 'hacktools[all]'` to install all of them.
|
|
66
67
|
### Dependencies list
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
# hacktools
|
|
2
2
|
A set of utilities and tools for rom hacking and translations.
|
|
3
3
|
|
|
4
|
+
Documentation is available at [illidanz.github.io/hacktools](https://illidanz.github.io/hacktools/).
|
|
5
|
+
|
|
4
6
|
## Installing dependencies
|
|
5
7
|
Most dependencies are optional, and can be installed with `pip install 'hacktools[name1,name2]'` or `pip install 'hacktools[all]'` to install all of them.
|
|
6
8
|
### Dependencies list
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "1.0.1"
|
|
@@ -1,29 +1,63 @@
|
|
|
1
|
+
"""Support for ARCH archives, including compression and decompression.
|
|
2
|
+
|
|
3
|
+
An ARCH archive starts with an "ARCH" magic followed by a header, a name
|
|
4
|
+
table, a file allocation table and the file data. Files can optionally be
|
|
5
|
+
compressed with a byte-pair encoding scheme, based on Tinke's implementation.
|
|
6
|
+
"""
|
|
1
7
|
import os
|
|
2
8
|
from collections import Counter
|
|
3
9
|
from hacktools import common
|
|
4
10
|
|
|
5
11
|
|
|
6
12
|
class ARCHArchive:
|
|
13
|
+
"""Structure of an ARCH archive header.
|
|
14
|
+
|
|
15
|
+
Attributes:
|
|
16
|
+
filenum: Number of files in the archive.
|
|
17
|
+
tableoff: Offset of the name table.
|
|
18
|
+
fatoff: Offset of the file allocation table.
|
|
19
|
+
nameindexoff: Offset of the name index.
|
|
20
|
+
dataoff: Offset of the file data.
|
|
21
|
+
files: List of files in the archive.
|
|
22
|
+
"""
|
|
7
23
|
def __init__(self):
|
|
8
|
-
self.filenum = 0
|
|
9
|
-
self.tableoff = 0
|
|
10
|
-
self.fatoff = 0
|
|
11
|
-
self.nameindexoff = 0
|
|
12
|
-
self.dataoff = 0
|
|
13
|
-
self.files = []
|
|
24
|
+
self.filenum: int = 0
|
|
25
|
+
self.tableoff: int = 0
|
|
26
|
+
self.fatoff: int = 0
|
|
27
|
+
self.nameindexoff: int = 0
|
|
28
|
+
self.dataoff: int = 0
|
|
29
|
+
self.files: list[ARCHFile] = []
|
|
14
30
|
|
|
15
31
|
|
|
16
32
|
class ARCHFile:
|
|
33
|
+
"""Structure of a single file entry in an ARCH archive.
|
|
34
|
+
|
|
35
|
+
Attributes:
|
|
36
|
+
name: File name, read from the archive name table.
|
|
37
|
+
length: Length of the file data as stored in the archive.
|
|
38
|
+
declength: Decompressed length, equal to length if not encoded.
|
|
39
|
+
offset: Offset of the file data, relative to the archive dataoff.
|
|
40
|
+
nameoffset: Offset of the file name, relative to the archive tableoff.
|
|
41
|
+
encoded: Whether the file data is compressed.
|
|
42
|
+
"""
|
|
17
43
|
def __init__(self):
|
|
18
|
-
self.name = ""
|
|
19
|
-
self.length = 0
|
|
20
|
-
self.declength = 0
|
|
21
|
-
self.offset = 0
|
|
22
|
-
self.nameoffset = 0
|
|
23
|
-
self.encoded = False
|
|
44
|
+
self.name: str = ""
|
|
45
|
+
self.length: int = 0
|
|
46
|
+
self.declength: int = 0
|
|
47
|
+
self.offset: int = 0
|
|
48
|
+
self.nameoffset: int = 0
|
|
49
|
+
self.encoded: bool = False
|
|
50
|
+
|
|
24
51
|
|
|
52
|
+
def read(f: common.Stream) -> ARCHArchive:
|
|
53
|
+
"""Read the header and file table of an ARCH archive.
|
|
25
54
|
|
|
26
|
-
|
|
55
|
+
Args:
|
|
56
|
+
f: Stream opened on the archive file.
|
|
57
|
+
|
|
58
|
+
Returns:
|
|
59
|
+
The parsed archive structure.
|
|
60
|
+
"""
|
|
27
61
|
f.seek(4) # Magic: ARCH
|
|
28
62
|
archive = ARCHArchive()
|
|
29
63
|
archive.filenum = f.readUInt()
|
|
@@ -47,7 +81,19 @@ def read(f):
|
|
|
47
81
|
return archive
|
|
48
82
|
|
|
49
83
|
|
|
50
|
-
def repack(fin, f, archive, infolder):
|
|
84
|
+
def repack(fin: common.Stream, f: common.Stream, archive: ARCHArchive, infolder: str) -> None:
|
|
85
|
+
"""Repack an ARCH archive, replacing the files found in a folder.
|
|
86
|
+
|
|
87
|
+
Files that exist in infolder are read from there, compressing the ones
|
|
88
|
+
marked as encoded in the archive, while the others are copied from fin.
|
|
89
|
+
File data is aligned to 16 bytes with zeros.
|
|
90
|
+
|
|
91
|
+
Args:
|
|
92
|
+
fin: Stream opened on the original archive file.
|
|
93
|
+
f: Stream opened on the output archive file.
|
|
94
|
+
archive: Archive structure returned by :func:`read`.
|
|
95
|
+
infolder: Path of the folder containing the replacement files.
|
|
96
|
+
"""
|
|
51
97
|
# Copy everything up to dataoff
|
|
52
98
|
fin.seek(0)
|
|
53
99
|
f.seek(0)
|
|
@@ -96,7 +142,16 @@ def repack(fin, f, archive, infolder):
|
|
|
96
142
|
dataoff = f.tell() - archive.dataoff
|
|
97
143
|
|
|
98
144
|
|
|
99
|
-
def extract(f, archive, outfolder):
|
|
145
|
+
def extract(f: common.Stream, archive: ARCHArchive, outfolder: str) -> None:
|
|
146
|
+
"""Extract all the files of an ARCH archive to a folder.
|
|
147
|
+
|
|
148
|
+
Files marked as encoded are decompressed.
|
|
149
|
+
|
|
150
|
+
Args:
|
|
151
|
+
f: Stream opened on the archive file.
|
|
152
|
+
archive: Archive structure returned by :func:`read`.
|
|
153
|
+
outfolder: Path of the folder to extract the files to.
|
|
154
|
+
"""
|
|
100
155
|
for subfile in archive.files:
|
|
101
156
|
with common.Stream(outfolder + subfile.name, "wb") as fout:
|
|
102
157
|
f.seek(archive.dataoff + subfile.offset)
|
|
@@ -106,7 +161,19 @@ def extract(f, archive, outfolder):
|
|
|
106
161
|
fout.write(decompress(f.read(subfile.length), subfile.declength))
|
|
107
162
|
|
|
108
163
|
|
|
109
|
-
def compress(data):
|
|
164
|
+
def compress(data: bytes) -> bytes:
|
|
165
|
+
"""Compress data with the byte-pair encoding used by ARCH archives.
|
|
166
|
+
|
|
167
|
+
The most common byte pairs are recursively replaced with single bytes
|
|
168
|
+
that don't occur in the data, and a dictionary of these replacements is
|
|
169
|
+
written before the encoded content.
|
|
170
|
+
|
|
171
|
+
Args:
|
|
172
|
+
data: Data to compress.
|
|
173
|
+
|
|
174
|
+
Returns:
|
|
175
|
+
The compressed data.
|
|
176
|
+
"""
|
|
110
177
|
# Find unused bytes in the data
|
|
111
178
|
dictkeys = []
|
|
112
179
|
for i in range(1, 0x100):
|
|
@@ -199,7 +266,16 @@ def compress(data):
|
|
|
199
266
|
return f.read()
|
|
200
267
|
|
|
201
268
|
|
|
202
|
-
def decompress(data, declen):
|
|
269
|
+
def decompress(data: bytes, declen: int) -> bytes:
|
|
270
|
+
"""Decompress byte-pair encoded data from an ARCH archive.
|
|
271
|
+
|
|
272
|
+
Args:
|
|
273
|
+
data: Data to decompress.
|
|
274
|
+
declen: Expected length of the decompressed data, currently unused.
|
|
275
|
+
|
|
276
|
+
Returns:
|
|
277
|
+
The decompressed data.
|
|
278
|
+
"""
|
|
203
279
|
with common.Stream() as f:
|
|
204
280
|
with common.Stream() as fout:
|
|
205
281
|
f.write(data)
|
|
@@ -235,7 +311,7 @@ def decompress(data, declen):
|
|
|
235
311
|
# Process
|
|
236
312
|
numloops = (f.readByte() << 8) + f.readByte()
|
|
237
313
|
common.logDebug("Decompressing with", common.toHex(numloops), "loops starting at", common.toHex(f.tell()))
|
|
238
|
-
nextsamples = []
|
|
314
|
+
nextsamples: list[int] = []
|
|
239
315
|
while True:
|
|
240
316
|
if len(nextsamples) == 0:
|
|
241
317
|
if numloops == 0:
|
|
@@ -1,5 +1,20 @@
|
|
|
1
1
|
#include "inc.h"
|
|
2
2
|
|
|
3
|
+
PyDoc_STRVAR(compressCRILAYLA_doc,
|
|
4
|
+
"compressCRILAYLA($module, /, indata)\n"
|
|
5
|
+
"--\n"
|
|
6
|
+
"\n"
|
|
7
|
+
"Compress data with the CRILAYLA scheme used by CPK archives.\n"
|
|
8
|
+
"\n"
|
|
9
|
+
"The first 0x100 bytes of the input are stored uncompressed after the\n"
|
|
10
|
+
"compressed stream, so the input must be at least 0x100 bytes long.\n"
|
|
11
|
+
"\n"
|
|
12
|
+
"Args:\n"
|
|
13
|
+
" indata (bytes): Data to compress.\n"
|
|
14
|
+
"\n"
|
|
15
|
+
"Returns:\n"
|
|
16
|
+
" bytes: The compressed data, starting with the CRILAYLA signature.");
|
|
17
|
+
|
|
3
18
|
// https://github.com/ConnorKrammer/cpk-tools/blob/master/LibCRIComp/LibCRIComp.cpp
|
|
4
19
|
static PyObject* compressCRILAYLA(PyObject* module, PyObject* args, PyObject* kwargs)
|
|
5
20
|
{
|
|
@@ -23,7 +38,7 @@ static PyObject* compressCRILAYLA(PyObject* module, PyObject* args, PyObject* kw
|
|
|
23
38
|
for (; n >= 0x100;)
|
|
24
39
|
{
|
|
25
40
|
j = n + 3 + 0x2000;
|
|
26
|
-
if (j > srclen)
|
|
41
|
+
if (j > (int)srclen)
|
|
27
42
|
j = (int)srclen;
|
|
28
43
|
for (i = n + 3, p = 0; i < j; i++)
|
|
29
44
|
{
|
|
@@ -160,6 +175,18 @@ static inline uint16_t get_next_bits(unsigned char* input_buffer, long* const of
|
|
|
160
175
|
return out_bits;
|
|
161
176
|
}
|
|
162
177
|
|
|
178
|
+
PyDoc_STRVAR(decompressCRILAYLA_doc,
|
|
179
|
+
"decompressCRILAYLA($module, /, indata)\n"
|
|
180
|
+
"--\n"
|
|
181
|
+
"\n"
|
|
182
|
+
"Decompress CRILAYLA-compressed data.\n"
|
|
183
|
+
"\n"
|
|
184
|
+
"Args:\n"
|
|
185
|
+
" indata (bytes): Compressed data, starting with the CRILAYLA signature.\n"
|
|
186
|
+
"\n"
|
|
187
|
+
"Returns:\n"
|
|
188
|
+
" bytes: The decompressed data.");
|
|
189
|
+
|
|
163
190
|
// https://github.com/hcs64/vgm_ripping/blob/master/multi/utf_tab/cpk_uncompress.c
|
|
164
191
|
static PyObject* decompressCRILAYLA(PyObject* m, PyObject* args, PyObject* kwargs)
|
|
165
192
|
{
|
|
@@ -244,20 +271,20 @@ static PyObject* decompressCRILAYLA(PyObject* m, PyObject* args, PyObject* kwarg
|
|
|
244
271
|
}
|
|
245
272
|
|
|
246
273
|
static PyMethodDef Cmp_criMethods[] = {
|
|
247
|
-
{"compressCRILAYLA", (PyCFunction)compressCRILAYLA, METH_VARARGS | METH_KEYWORDS,
|
|
248
|
-
{"decompressCRILAYLA", (PyCFunction)decompressCRILAYLA, METH_VARARGS | METH_KEYWORDS,
|
|
274
|
+
{"compressCRILAYLA", (PyCFunction)compressCRILAYLA, METH_VARARGS | METH_KEYWORDS, compressCRILAYLA_doc},
|
|
275
|
+
{"decompressCRILAYLA", (PyCFunction)decompressCRILAYLA, METH_VARARGS | METH_KEYWORDS, decompressCRILAYLA_doc},
|
|
249
276
|
{NULL, NULL, 0, NULL}
|
|
250
277
|
};
|
|
251
278
|
|
|
252
279
|
static struct PyModuleDef cmp_crimodule = {
|
|
253
280
|
PyModuleDef_HEAD_INIT,
|
|
254
281
|
"cmp_cri",
|
|
255
|
-
"CRILAYLA
|
|
282
|
+
"C implementation of the CRILAYLA compression scheme used by CPK archives.",
|
|
256
283
|
-1,
|
|
257
284
|
Cmp_criMethods
|
|
258
285
|
};
|
|
259
286
|
|
|
260
287
|
PyMODINIT_FUNC PyInit_cmp_cri(void)
|
|
261
288
|
{
|
|
262
|
-
return
|
|
289
|
+
return moduleCreate(&cmp_crimodule);
|
|
263
290
|
}
|
|
@@ -0,0 +1,405 @@
|
|
|
1
|
+
#include "inc.h"
|
|
2
|
+
|
|
3
|
+
// Free the buffer before raising, as the checks happen after allocating it
|
|
4
|
+
#define ERROR_CHECK_FREE(cond, error, ptr) if (cond) { PyMem_Free(ptr); PyErr_SetString(PyExc_ValueError, error); return NULL; }
|
|
5
|
+
|
|
6
|
+
// A tree can hold at most 256 leaves, one per possible value, plus the 255 branches joining them
|
|
7
|
+
#define MAX_NODES 512
|
|
8
|
+
// With at most 256 leaves the tree is at most 255 levels deep, so codes are at most 255 bits long
|
|
9
|
+
#define MAX_CODE_BITS 256
|
|
10
|
+
|
|
11
|
+
PyDoc_STRVAR(decompressHuffman_doc,
|
|
12
|
+
"decompressHuffman($module, /, rawdata, decomplength, numbits=8, little=True)\n"
|
|
13
|
+
"--\n"
|
|
14
|
+
"\n"
|
|
15
|
+
"Decompress Huffman-coded data, as used by the GBA and NDS BIOS.\n"
|
|
16
|
+
"\n"
|
|
17
|
+
"The data is expected to start with the tree size and the tree itself,\n"
|
|
18
|
+
"followed by the 32-bit codeword stream, without the 4-byte header with\n"
|
|
19
|
+
"compression type and length.\n"
|
|
20
|
+
"\n"
|
|
21
|
+
"Args:\n"
|
|
22
|
+
" rawdata (bytes): Compressed data to read.\n"
|
|
23
|
+
" decomplength (int): Length of the decompressed data.\n"
|
|
24
|
+
" numbits (int): Data size in bits, 8 or 4.\n"
|
|
25
|
+
" little (bool): Whether nibbles are ordered low-first when numbits is 4.\n"
|
|
26
|
+
"\n"
|
|
27
|
+
"Returns:\n"
|
|
28
|
+
" bytes: The decompressed data.");
|
|
29
|
+
|
|
30
|
+
static PyObject* decompressHuffman(PyObject* m, PyObject* args, PyObject* kwargs)
|
|
31
|
+
{
|
|
32
|
+
static char *kwlist[] = { "rawdata", "decomplength", "numbits", "little", NULL };
|
|
33
|
+
|
|
34
|
+
unsigned char* rawdata;
|
|
35
|
+
size_t datalength;
|
|
36
|
+
unsigned int decomplength;
|
|
37
|
+
int numbits = 8;
|
|
38
|
+
int little = 1;
|
|
39
|
+
|
|
40
|
+
if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s#I|ip", kwlist, &rawdata, &datalength, &decomplength, &numbits, &little))
|
|
41
|
+
return NULL;
|
|
42
|
+
ERROR_CHECK(numbits != 8 && numbits != 4, "Data size must be 8 or 4 bits.");
|
|
43
|
+
ERROR_CHECK(datalength < 2, "Not enough data.");
|
|
44
|
+
|
|
45
|
+
unsigned int treesize = rawdata[0];
|
|
46
|
+
unsigned int treeroot = rawdata[1];
|
|
47
|
+
unsigned char* treebuffer = rawdata + 2;
|
|
48
|
+
size_t treelength = (size_t)treesize * 2;
|
|
49
|
+
ERROR_CHECK(datalength < 2 + treelength, "Not enough data.");
|
|
50
|
+
|
|
51
|
+
// Every written byte holds numbits bits of the decompressed data
|
|
52
|
+
size_t outlength = (size_t)decomplength * (8 / numbits);
|
|
53
|
+
unsigned char* out = PyMem_Malloc(outlength > 0 ? outlength : 1);
|
|
54
|
+
MALLOC_CHECK(out);
|
|
55
|
+
|
|
56
|
+
size_t readbytes = 2 + treelength;
|
|
57
|
+
ERROR_CHECK_FREE(readbytes + 4 > datalength, "Not enough data.", out);
|
|
58
|
+
unsigned int code = READ_32(rawdata, readbytes);
|
|
59
|
+
readbytes += 4;
|
|
60
|
+
|
|
61
|
+
size_t writebytes = 0;
|
|
62
|
+
unsigned int i = 0;
|
|
63
|
+
unsigned int next = 0;
|
|
64
|
+
unsigned int pos = treeroot;
|
|
65
|
+
while (writebytes < outlength)
|
|
66
|
+
{
|
|
67
|
+
// All the bits of the current codeword have been read, get a new one
|
|
68
|
+
if (i == 32)
|
|
69
|
+
{
|
|
70
|
+
ERROR_CHECK_FREE(readbytes + 4 > datalength, "Not enough data.", out);
|
|
71
|
+
code = READ_32(rawdata, readbytes);
|
|
72
|
+
readbytes += 4;
|
|
73
|
+
i = 0;
|
|
74
|
+
}
|
|
75
|
+
next += (pos & 0x3f) * 2 + 2;
|
|
76
|
+
unsigned int direction = ((code >> (31 - i)) & 1) == 0 ? 2 : 1;
|
|
77
|
+
int leaf = (((pos >> 5) >> direction) & 1) != 0;
|
|
78
|
+
ERROR_CHECK_FREE(next - direction >= treelength, "Tree offset out of range.", out);
|
|
79
|
+
pos = treebuffer[next - direction];
|
|
80
|
+
if (leaf)
|
|
81
|
+
{
|
|
82
|
+
out[writebytes++] = (unsigned char)(pos & 0xff);
|
|
83
|
+
pos = treeroot;
|
|
84
|
+
next = 0;
|
|
85
|
+
}
|
|
86
|
+
++i;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
// Pack the nibbles back into bytes
|
|
90
|
+
if (numbits == 4)
|
|
91
|
+
{
|
|
92
|
+
for (unsigned int j = 0; j < decomplength; ++j)
|
|
93
|
+
{
|
|
94
|
+
unsigned char b1 = out[2 * j + 1];
|
|
95
|
+
unsigned char b2 = out[2 * j];
|
|
96
|
+
if (little)
|
|
97
|
+
out[j] = (unsigned char)((b1 * 16 + b2) & 0xff);
|
|
98
|
+
else
|
|
99
|
+
out[j] = (unsigned char)((b2 * 16 + b1) & 0xff);
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
PyObject *output = PyBytes_FromStringAndSize(out, decomplength);
|
|
104
|
+
PyMem_Free(out);
|
|
105
|
+
return output;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
// Node of the binary tree built by compressHuffman
|
|
109
|
+
typedef struct HuffNode
|
|
110
|
+
{
|
|
111
|
+
unsigned int freqcount;
|
|
112
|
+
int code;
|
|
113
|
+
int score;
|
|
114
|
+
int haschildren;
|
|
115
|
+
struct HuffNode* children[2];
|
|
116
|
+
} HuffNode;
|
|
117
|
+
|
|
118
|
+
// Sort the nodes by frequency, keeping the order of equal elements.
|
|
119
|
+
// The list is always sorted except for the node appended by the last iteration,
|
|
120
|
+
// so the insertion sort only has to move that one back into place.
|
|
121
|
+
static void sortByFreq(HuffNode** freq, int freqlen)
|
|
122
|
+
{
|
|
123
|
+
for (int i = 1; i < freqlen; ++i)
|
|
124
|
+
{
|
|
125
|
+
HuffNode* node = freq[i];
|
|
126
|
+
int j = i - 1;
|
|
127
|
+
while (j >= 0 && freq[j]->freqcount > node->freqcount)
|
|
128
|
+
{
|
|
129
|
+
freq[j + 1] = freq[j];
|
|
130
|
+
--j;
|
|
131
|
+
}
|
|
132
|
+
freq[j + 1] = node;
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
// Store the code of every leaf node under this node as a string of 0 and 1 bits
|
|
137
|
+
static void getHuffCodes(HuffNode* node, unsigned char* seed, int seedlen, unsigned char* codebits, int* codelengths)
|
|
138
|
+
{
|
|
139
|
+
if (!node->haschildren)
|
|
140
|
+
{
|
|
141
|
+
for (int i = 0; i < seedlen; ++i)
|
|
142
|
+
codebits[node->code * MAX_CODE_BITS + i] = seed[i];
|
|
143
|
+
codelengths[node->code] = seedlen;
|
|
144
|
+
return;
|
|
145
|
+
}
|
|
146
|
+
for (int i = 0; i < 2; ++i)
|
|
147
|
+
{
|
|
148
|
+
seed[seedlen] = (unsigned char)i;
|
|
149
|
+
getHuffCodes(node->children[i], seed, seedlen + 1, codebits, codelengths);
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
PyDoc_STRVAR(compressHuffman_doc,
|
|
154
|
+
"compressHuffman($module, /, indata, numbits=8, little=True)\n"
|
|
155
|
+
"--\n"
|
|
156
|
+
"\n"
|
|
157
|
+
"Compress data with Huffman coding, as used by the GBA and NDS BIOS.\n"
|
|
158
|
+
"\n"
|
|
159
|
+
"The output starts with the tree size and the tree itself, followed by\n"
|
|
160
|
+
"the 32-bit codeword stream. The 4-byte header with compression type and\n"
|
|
161
|
+
"length is not included and should be written separately.\n"
|
|
162
|
+
"\n"
|
|
163
|
+
"Args:\n"
|
|
164
|
+
" indata (bytes): Data to compress.\n"
|
|
165
|
+
" numbits (int): Data size in bits, 8 or 4.\n"
|
|
166
|
+
" little (bool): Whether nibbles are ordered low-first when numbits is 4.\n"
|
|
167
|
+
"\n"
|
|
168
|
+
"Returns:\n"
|
|
169
|
+
" bytes: The compressed data.");
|
|
170
|
+
|
|
171
|
+
static PyObject* compressHuffman(PyObject* m, PyObject* args, PyObject* kwargs)
|
|
172
|
+
{
|
|
173
|
+
static char *kwlist[] = { "indata", "numbits", "little", NULL };
|
|
174
|
+
|
|
175
|
+
unsigned char* indata;
|
|
176
|
+
size_t inlength;
|
|
177
|
+
int numbits = 8;
|
|
178
|
+
int little = 1;
|
|
179
|
+
|
|
180
|
+
if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s#|ip", kwlist, &indata, &inlength, &numbits, &little))
|
|
181
|
+
return NULL;
|
|
182
|
+
ERROR_CHECK(numbits != 8 && numbits != 4, "Data size must be 8 or 4 bits.");
|
|
183
|
+
ERROR_CHECK(inlength == 0, "No data to compress.");
|
|
184
|
+
|
|
185
|
+
// Read indata as nibbles if numbits is 4
|
|
186
|
+
unsigned char* workdata = indata;
|
|
187
|
+
size_t worklength = inlength;
|
|
188
|
+
unsigned char* nibbles = NULL;
|
|
189
|
+
if (numbits == 4)
|
|
190
|
+
{
|
|
191
|
+
nibbles = PyMem_Malloc(inlength * 2);
|
|
192
|
+
MALLOC_CHECK(nibbles);
|
|
193
|
+
for (size_t i = 0; i < inlength; ++i)
|
|
194
|
+
{
|
|
195
|
+
unsigned char b1 = indata[i] % 16;
|
|
196
|
+
unsigned char b2 = indata[i] / 16;
|
|
197
|
+
nibbles[2 * i] = little ? b1 : b2;
|
|
198
|
+
nibbles[2 * i + 1] = little ? b2 : b1;
|
|
199
|
+
}
|
|
200
|
+
workdata = nibbles;
|
|
201
|
+
worklength = inlength * 2;
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
// Get frequencies
|
|
205
|
+
unsigned int freqcount[256] = { 0 };
|
|
206
|
+
for (size_t i = 0; i < worklength; ++i)
|
|
207
|
+
++freqcount[workdata[i]];
|
|
208
|
+
|
|
209
|
+
HuffNode* nodes = PyMem_Malloc(MAX_NODES * sizeof(HuffNode));
|
|
210
|
+
HuffNode** freq = PyMem_Malloc(MAX_NODES * sizeof(HuffNode*));
|
|
211
|
+
HuffNode** lst = PyMem_Malloc(MAX_NODES * sizeof(HuffNode*));
|
|
212
|
+
unsigned char* codebits = PyMem_Malloc(256 * MAX_CODE_BITS);
|
|
213
|
+
if (nodes == NULL || freq == NULL || lst == NULL || codebits == NULL)
|
|
214
|
+
{
|
|
215
|
+
PyMem_Free(nibbles);
|
|
216
|
+
PyMem_Free(nodes);
|
|
217
|
+
PyMem_Free(freq);
|
|
218
|
+
PyMem_Free(lst);
|
|
219
|
+
PyMem_Free(codebits);
|
|
220
|
+
return PyErr_NoMemory();
|
|
221
|
+
}
|
|
222
|
+
int nodecount = 0;
|
|
223
|
+
int freqlen = 0;
|
|
224
|
+
for (int i = 0; i < 256; ++i)
|
|
225
|
+
{
|
|
226
|
+
if (freqcount[i] > 0)
|
|
227
|
+
{
|
|
228
|
+
HuffNode* node = &nodes[nodecount++];
|
|
229
|
+
node->freqcount = freqcount[i];
|
|
230
|
+
node->code = i;
|
|
231
|
+
node->score = 0;
|
|
232
|
+
node->haschildren = 0;
|
|
233
|
+
freq[freqlen++] = node;
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
// Add a stub entry in the special case that there's only one item
|
|
238
|
+
if (freqlen == 1)
|
|
239
|
+
{
|
|
240
|
+
HuffNode* node = &nodes[nodecount++];
|
|
241
|
+
node->freqcount = 0;
|
|
242
|
+
node->code = (workdata[0] + 1) & 0xff;
|
|
243
|
+
node->score = 0;
|
|
244
|
+
node->haschildren = 0;
|
|
245
|
+
freq[freqlen++] = node;
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
// Sort and create the tree
|
|
249
|
+
while (freqlen > 1)
|
|
250
|
+
{
|
|
251
|
+
sortByFreq(freq, freqlen);
|
|
252
|
+
HuffNode* node = &nodes[nodecount++];
|
|
253
|
+
node->freqcount = freq[0]->freqcount + freq[1]->freqcount;
|
|
254
|
+
node->code = 0;
|
|
255
|
+
node->score = 0;
|
|
256
|
+
node->haschildren = 1;
|
|
257
|
+
node->children[0] = freq[0];
|
|
258
|
+
node->children[1] = freq[1];
|
|
259
|
+
for (int i = 2; i < freqlen; ++i)
|
|
260
|
+
freq[i - 2] = freq[i];
|
|
261
|
+
freqlen -= 2;
|
|
262
|
+
freq[freqlen++] = node;
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
// Label nodes to keep bandwidth small
|
|
266
|
+
int lstlen = 0;
|
|
267
|
+
while (freqlen > 0)
|
|
268
|
+
{
|
|
269
|
+
// Take the node with the lowest score, the first one if there are ties
|
|
270
|
+
int best = 0;
|
|
271
|
+
for (int i = 0; i < freqlen; ++i)
|
|
272
|
+
{
|
|
273
|
+
freq[i]->score = freq[i]->code - i;
|
|
274
|
+
if (freq[i]->score < freq[best]->score)
|
|
275
|
+
best = i;
|
|
276
|
+
}
|
|
277
|
+
HuffNode* node = freq[best];
|
|
278
|
+
for (int i = best; i < freqlen - 1; ++i)
|
|
279
|
+
freq[i] = freq[i + 1];
|
|
280
|
+
--freqlen;
|
|
281
|
+
node->code = (lstlen - node->code) & 0xff;
|
|
282
|
+
lst[lstlen++] = node;
|
|
283
|
+
if (node->haschildren)
|
|
284
|
+
{
|
|
285
|
+
for (int i = 1; i >= 0; --i)
|
|
286
|
+
{
|
|
287
|
+
if (node->children[i]->haschildren)
|
|
288
|
+
{
|
|
289
|
+
node->children[i]->code = lstlen & 0xff;
|
|
290
|
+
freq[freqlen++] = node->children[i];
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
// Convert our list of nodes to a table of bytes -> huffman codes
|
|
297
|
+
int codelengths[256] = { 0 };
|
|
298
|
+
unsigned char seed[MAX_CODE_BITS];
|
|
299
|
+
getHuffCodes(lst[0], seed, 0, codebits, codelengths);
|
|
300
|
+
|
|
301
|
+
// The output holds the tree, then the codes padded to a multiple of 32 bits
|
|
302
|
+
size_t totalbits = 0;
|
|
303
|
+
for (int i = 0; i < 256; ++i)
|
|
304
|
+
totalbits += (size_t)freqcount[i] * codelengths[i];
|
|
305
|
+
size_t outcap = 2 + 2 * (size_t)lstlen + 2 + (totalbits + 31) / 32 * 4 + 4;
|
|
306
|
+
unsigned char* out = PyMem_Malloc(outcap);
|
|
307
|
+
if (out == NULL)
|
|
308
|
+
{
|
|
309
|
+
PyMem_Free(nibbles);
|
|
310
|
+
PyMem_Free(nodes);
|
|
311
|
+
PyMem_Free(freq);
|
|
312
|
+
PyMem_Free(lst);
|
|
313
|
+
PyMem_Free(codebits);
|
|
314
|
+
return PyErr_NoMemory();
|
|
315
|
+
}
|
|
316
|
+
size_t outlength = 0;
|
|
317
|
+
|
|
318
|
+
// Write header
|
|
319
|
+
out[outlength++] = (unsigned char)(lstlen & 0xff);
|
|
320
|
+
|
|
321
|
+
// Write Huffman tree, reusing the now empty freq list to hold it
|
|
322
|
+
HuffNode** tree = freq;
|
|
323
|
+
int treelen = 0;
|
|
324
|
+
tree[treelen++] = lst[0];
|
|
325
|
+
for (int i = 0; i < lstlen; ++i)
|
|
326
|
+
{
|
|
327
|
+
if (lst[i]->haschildren)
|
|
328
|
+
for (int j = 0; j < 2; ++j)
|
|
329
|
+
tree[treelen++] = lst[i]->children[j];
|
|
330
|
+
}
|
|
331
|
+
for (int i = 0; i < treelen; ++i)
|
|
332
|
+
{
|
|
333
|
+
HuffNode* node = tree[i];
|
|
334
|
+
if (node->haschildren)
|
|
335
|
+
{
|
|
336
|
+
int childsum = 0;
|
|
337
|
+
for (int j = 0; j < 2; ++j)
|
|
338
|
+
if (!node->children[j]->haschildren)
|
|
339
|
+
childsum += (0x80 >> j) & 0xff;
|
|
340
|
+
node->code |= childsum & 0xff;
|
|
341
|
+
}
|
|
342
|
+
out[outlength++] = (unsigned char)(node->code & 0xff);
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
// Ensure nodes are word-aligned
|
|
346
|
+
if (lstlen % 2 == 0)
|
|
347
|
+
{
|
|
348
|
+
out[outlength++] = 0;
|
|
349
|
+
out[outlength++] = 0;
|
|
350
|
+
out[0] = (unsigned char)((lstlen + 1) & 0xff);
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
// Write bits to stream
|
|
354
|
+
unsigned int data = 0;
|
|
355
|
+
size_t setbits = 0;
|
|
356
|
+
for (size_t i = 0; i < worklength; ++i)
|
|
357
|
+
{
|
|
358
|
+
unsigned char* bits = codebits + workdata[i] * MAX_CODE_BITS;
|
|
359
|
+
int bitslength = codelengths[workdata[i]];
|
|
360
|
+
for (int j = 0; j < bitslength; ++j)
|
|
361
|
+
{
|
|
362
|
+
data = data * 2 + bits[j];
|
|
363
|
+
++setbits;
|
|
364
|
+
if (setbits % 32 == 0)
|
|
365
|
+
{
|
|
366
|
+
WRITE_32(out, outlength, data);
|
|
367
|
+
outlength += 4;
|
|
368
|
+
data = 0;
|
|
369
|
+
}
|
|
370
|
+
}
|
|
371
|
+
}
|
|
372
|
+
if (setbits % 32 != 0)
|
|
373
|
+
{
|
|
374
|
+
WRITE_32(out, outlength, data << (32 - (setbits % 32)));
|
|
375
|
+
outlength += 4;
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
PyObject *output = PyBytes_FromStringAndSize(out, outlength);
|
|
379
|
+
PyMem_Free(out);
|
|
380
|
+
PyMem_Free(nibbles);
|
|
381
|
+
PyMem_Free(nodes);
|
|
382
|
+
PyMem_Free(freq);
|
|
383
|
+
PyMem_Free(lst);
|
|
384
|
+
PyMem_Free(codebits);
|
|
385
|
+
return output;
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
static PyMethodDef Cmp_huffMethods[] = {
|
|
389
|
+
{"decompressHuffman", (PyCFunction)decompressHuffman, METH_VARARGS | METH_KEYWORDS, decompressHuffman_doc},
|
|
390
|
+
{"compressHuffman", (PyCFunction)compressHuffman, METH_VARARGS | METH_KEYWORDS, compressHuffman_doc},
|
|
391
|
+
{NULL, NULL, 0, NULL}
|
|
392
|
+
};
|
|
393
|
+
|
|
394
|
+
static struct PyModuleDef cmp_huffmodule = {
|
|
395
|
+
PyModuleDef_HEAD_INIT,
|
|
396
|
+
"cmp_huff",
|
|
397
|
+
"C implementation of the Huffman coding scheme used by the GBA and NDS BIOS.",
|
|
398
|
+
-1,
|
|
399
|
+
Cmp_huffMethods
|
|
400
|
+
};
|
|
401
|
+
|
|
402
|
+
PyMODINIT_FUNC PyInit_cmp_huff(void)
|
|
403
|
+
{
|
|
404
|
+
return moduleCreate(&cmp_huffmodule);
|
|
405
|
+
}
|