hacktools 0.40.1__tar.gz → 1.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. {hacktools-0.40.1 → hacktools-1.0.0}/LICENSE +1 -1
  2. {hacktools-0.40.1/hacktools.egg-info → hacktools-1.0.0}/PKG-INFO +4 -4
  3. {hacktools-0.40.1 → hacktools-1.0.0}/README.md +2 -0
  4. hacktools-1.0.0/hacktools/__init__.py +1 -0
  5. {hacktools-0.40.1 → hacktools-1.0.0}/hacktools/arch.py +94 -18
  6. {hacktools-0.40.1 → hacktools-1.0.0}/hacktools/c_ext/cmp_cri.c +31 -4
  7. hacktools-1.0.0/hacktools/c_ext/cmp_huff.c +405 -0
  8. {hacktools-0.40.1 → hacktools-1.0.0}/hacktools/c_ext/cmp_lzss.c +85 -27
  9. hacktools-1.0.0/hacktools/c_ext/cmp_misc.c +163 -0
  10. hacktools-1.0.0/hacktools/c_ext/cmp_prs.c +370 -0
  11. {hacktools-0.40.1 → hacktools-1.0.0}/hacktools/c_ext/cmp_racjin.c +29 -4
  12. hacktools-1.0.0/hacktools/c_ext/inc.h +26 -0
  13. {hacktools-0.40.1 → hacktools-1.0.0}/hacktools/common.py +963 -161
  14. {hacktools-0.40.1 → hacktools-1.0.0}/hacktools/cpk.py +378 -71
  15. {hacktools-0.40.1 → hacktools-1.0.0}/hacktools/gb.py +44 -3
  16. {hacktools-0.40.1 → hacktools-1.0.0}/hacktools/gui.py +403 -304
  17. {hacktools-0.40.1 → hacktools-1.0.0}/hacktools/nasm.py +20 -11
  18. {hacktools-0.40.1 → hacktools-1.0.0}/hacktools/nds.py +219 -30
  19. {hacktools-0.40.1 → hacktools-1.0.0}/hacktools/nitro.py +826 -165
  20. hacktools-1.0.0/hacktools/ps2.py +204 -0
  21. {hacktools-0.40.1 → hacktools-1.0.0}/hacktools/psp.py +602 -104
  22. {hacktools-0.40.1 → hacktools-1.0.0}/hacktools/psx.py +455 -295
  23. hacktools-1.0.0/hacktools/wii.py +756 -0
  24. {hacktools-0.40.1 → hacktools-1.0.0}/hacktools/ws.py +310 -40
  25. {hacktools-0.40.1 → hacktools-1.0.0/hacktools.egg-info}/PKG-INFO +4 -4
  26. {hacktools-0.40.1 → hacktools-1.0.0}/hacktools.egg-info/SOURCES.txt +4 -1
  27. hacktools-1.0.0/pyproject.toml +25 -0
  28. {hacktools-0.40.1 → hacktools-1.0.0}/setup.py +5 -6
  29. hacktools-1.0.0/tests/test_cmp.py +103 -0
  30. hacktools-0.40.1/hacktools/__init__.py +0 -1
  31. hacktools-0.40.1/hacktools/c_ext/cmp_misc.c +0 -60
  32. hacktools-0.40.1/hacktools/c_ext/inc.h +0 -11
  33. hacktools-0.40.1/hacktools/compression.py +0 -216
  34. hacktools-0.40.1/hacktools/wii.py +0 -285
  35. hacktools-0.40.1/tests/test_cmp.py +0 -37
  36. {hacktools-0.40.1 → hacktools-1.0.0}/MANIFEST.in +0 -0
  37. {hacktools-0.40.1 → hacktools-1.0.0}/hacktools/assets/icon.png +0 -0
  38. {hacktools-0.40.1 → hacktools-1.0.0}/hacktools.egg-info/dependency_links.txt +0 -0
  39. {hacktools-0.40.1 → hacktools-1.0.0}/hacktools.egg-info/requires.txt +0 -0
  40. {hacktools-0.40.1 → hacktools-1.0.0}/hacktools.egg-info/top_level.txt +0 -0
  41. {hacktools-0.40.1 → hacktools-1.0.0}/setup.cfg +0 -0
@@ -1,6 +1,6 @@
1
1
  MIT License
2
2
 
3
- Copyright (c) 2020-2025 Illidan
3
+ Copyright (c) 2020-2026 Illidan
4
4
 
5
5
  Permission is hereby granted, free of charge, to any person obtaining a copy
6
6
  of this software and associated documentation files (the "Software"), to deal
@@ -1,19 +1,17 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: hacktools
3
- Version: 0.40.1
3
+ Version: 1.0.0
4
4
  Summary: A set of utilities and tools for rom hacking and translations.
5
5
  Home-page: https://github.com/Illidanz/hacktools
6
6
  Author: Illidan
7
7
  License: MIT
8
- Classifier: Programming Language :: Python :: 3.8
9
- Classifier: Programming Language :: Python :: 3.9
10
8
  Classifier: Programming Language :: Python :: 3.10
11
9
  Classifier: Programming Language :: Python :: 3.11
12
10
  Classifier: Programming Language :: Python :: 3.12
13
11
  Classifier: Programming Language :: Python :: 3.13
14
12
  Classifier: Programming Language :: Python :: 3.14
15
13
  Classifier: Operating System :: OS Independent
16
- Requires-Python: >=3.7
14
+ Requires-Python: >=3.10
17
15
  Description-Content-Type: text/markdown
18
16
  License-File: LICENSE
19
17
  Provides-Extra: nds
@@ -61,6 +59,8 @@ Dynamic: summary
61
59
  # hacktools
62
60
  A set of utilities and tools for rom hacking and translations.
63
61
 
62
+ Documentation is available at [illidanz.github.io/hacktools](https://illidanz.github.io/hacktools/).
63
+
64
64
  ## Installing dependencies
65
65
  Most dependencies are optional, and can be installed with `pip install 'hacktools[name1,name2]'` or `pip install 'hacktools[all]'` to install all of them.
66
66
  ### Dependencies list
@@ -1,6 +1,8 @@
1
1
  # hacktools
2
2
  A set of utilities and tools for rom hacking and translations.
3
3
 
4
+ Documentation is available at [illidanz.github.io/hacktools](https://illidanz.github.io/hacktools/).
5
+
4
6
  ## Installing dependencies
5
7
  Most dependencies are optional, and can be installed with `pip install 'hacktools[name1,name2]'` or `pip install 'hacktools[all]'` to install all of them.
6
8
  ### Dependencies list
@@ -0,0 +1 @@
1
+ __version__ = "1.0.0"
@@ -1,29 +1,63 @@
1
+ """Support for ARCH archives, including compression and decompression.
2
+
3
+ An ARCH archive starts with an "ARCH" magic followed by a header, a name
4
+ table, a file allocation table and the file data. Files can optionally be
5
+ compressed with a byte-pair encoding scheme, based on Tinke's implementation.
6
+ """
1
7
  import os
2
8
  from collections import Counter
3
9
  from hacktools import common
4
10
 
5
11
 
6
12
  class ARCHArchive:
13
+ """Structure of an ARCH archive header.
14
+
15
+ Attributes:
16
+ filenum: Number of files in the archive.
17
+ tableoff: Offset of the name table.
18
+ fatoff: Offset of the file allocation table.
19
+ nameindexoff: Offset of the name index.
20
+ dataoff: Offset of the file data.
21
+ files: List of files in the archive.
22
+ """
7
23
  def __init__(self):
8
- self.filenum = 0
9
- self.tableoff = 0
10
- self.fatoff = 0
11
- self.nameindexoff = 0
12
- self.dataoff = 0
13
- self.files = []
24
+ self.filenum: int = 0
25
+ self.tableoff: int = 0
26
+ self.fatoff: int = 0
27
+ self.nameindexoff: int = 0
28
+ self.dataoff: int = 0
29
+ self.files: list[ARCHFile] = []
14
30
 
15
31
 
16
32
  class ARCHFile:
33
+ """Structure of a single file entry in an ARCH archive.
34
+
35
+ Attributes:
36
+ name: File name, read from the archive name table.
37
+ length: Length of the file data as stored in the archive.
38
+ declength: Decompressed length, equal to length if not encoded.
39
+ offset: Offset of the file data, relative to the archive dataoff.
40
+ nameoffset: Offset of the file name, relative to the archive tableoff.
41
+ encoded: Whether the file data is compressed.
42
+ """
17
43
  def __init__(self):
18
- self.name = ""
19
- self.length = 0
20
- self.declength = 0
21
- self.offset = 0
22
- self.nameoffset = 0
23
- self.encoded = False
44
+ self.name: str = ""
45
+ self.length: int = 0
46
+ self.declength: int = 0
47
+ self.offset: int = 0
48
+ self.nameoffset: int = 0
49
+ self.encoded: bool = False
50
+
24
51
 
52
+ def read(f: common.Stream) -> ARCHArchive:
53
+ """Read the header and file table of an ARCH archive.
25
54
 
26
- def read(f):
55
+ Args:
56
+ f: Stream opened on the archive file.
57
+
58
+ Returns:
59
+ The parsed archive structure.
60
+ """
27
61
  f.seek(4) # Magic: ARCH
28
62
  archive = ARCHArchive()
29
63
  archive.filenum = f.readUInt()
@@ -47,7 +81,19 @@ def read(f):
47
81
  return archive
48
82
 
49
83
 
50
- def repack(fin, f, archive, infolder):
84
+ def repack(fin: common.Stream, f: common.Stream, archive: ARCHArchive, infolder: str) -> None:
85
+ """Repack an ARCH archive, replacing the files found in a folder.
86
+
87
+ Files that exist in infolder are read from there, compressing the ones
88
+ marked as encoded in the archive, while the others are copied from fin.
89
+ File data is aligned to 16 bytes with zeros.
90
+
91
+ Args:
92
+ fin: Stream opened on the original archive file.
93
+ f: Stream opened on the output archive file.
94
+ archive: Archive structure returned by :func:`read`.
95
+ infolder: Path of the folder containing the replacement files.
96
+ """
51
97
  # Copy everything up to dataoff
52
98
  fin.seek(0)
53
99
  f.seek(0)
@@ -96,7 +142,16 @@ def repack(fin, f, archive, infolder):
96
142
  dataoff = f.tell() - archive.dataoff
97
143
 
98
144
 
99
- def extract(f, archive, outfolder):
145
+ def extract(f: common.Stream, archive: ARCHArchive, outfolder: str) -> None:
146
+ """Extract all the files of an ARCH archive to a folder.
147
+
148
+ Files marked as encoded are decompressed.
149
+
150
+ Args:
151
+ f: Stream opened on the archive file.
152
+ archive: Archive structure returned by :func:`read`.
153
+ outfolder: Path of the folder to extract the files to.
154
+ """
100
155
  for subfile in archive.files:
101
156
  with common.Stream(outfolder + subfile.name, "wb") as fout:
102
157
  f.seek(archive.dataoff + subfile.offset)
@@ -106,7 +161,19 @@ def extract(f, archive, outfolder):
106
161
  fout.write(decompress(f.read(subfile.length), subfile.declength))
107
162
 
108
163
 
109
- def compress(data):
164
+ def compress(data: bytes) -> bytes:
165
+ """Compress data with the byte-pair encoding used by ARCH archives.
166
+
167
+ The most common byte pairs are recursively replaced with single bytes
168
+ that don't occur in the data, and a dictionary of these replacements is
169
+ written before the encoded content.
170
+
171
+ Args:
172
+ data: Data to compress.
173
+
174
+ Returns:
175
+ The compressed data.
176
+ """
110
177
  # Find unused bytes in the data
111
178
  dictkeys = []
112
179
  for i in range(1, 0x100):
@@ -199,7 +266,16 @@ def compress(data):
199
266
  return f.read()
200
267
 
201
268
 
202
- def decompress(data, declen):
269
+ def decompress(data: bytes, declen: int) -> bytes:
270
+ """Decompress byte-pair encoded data from an ARCH archive.
271
+
272
+ Args:
273
+ data: Data to decompress.
274
+ declen: Expected length of the decompressed data, currently unused.
275
+
276
+ Returns:
277
+ The decompressed data.
278
+ """
203
279
  with common.Stream() as f:
204
280
  with common.Stream() as fout:
205
281
  f.write(data)
@@ -235,7 +311,7 @@ def decompress(data, declen):
235
311
  # Process
236
312
  numloops = (f.readByte() << 8) + f.readByte()
237
313
  common.logDebug("Decompressing with", common.toHex(numloops), "loops starting at", common.toHex(f.tell()))
238
- nextsamples = []
314
+ nextsamples: list[int] = []
239
315
  while True:
240
316
  if len(nextsamples) == 0:
241
317
  if numloops == 0:
@@ -1,5 +1,20 @@
1
1
  #include "inc.h"
2
2
 
3
+ PyDoc_STRVAR(compressCRILAYLA_doc,
4
+ "compressCRILAYLA($module, /, indata)\n"
5
+ "--\n"
6
+ "\n"
7
+ "Compress data with the CRILAYLA scheme used by CPK archives.\n"
8
+ "\n"
9
+ "The first 0x100 bytes of the input are stored uncompressed after the\n"
10
+ "compressed stream, so the input must be at least 0x100 bytes long.\n"
11
+ "\n"
12
+ "Args:\n"
13
+ " indata (bytes): Data to compress.\n"
14
+ "\n"
15
+ "Returns:\n"
16
+ " bytes: The compressed data, starting with the CRILAYLA signature.");
17
+
3
18
  // https://github.com/ConnorKrammer/cpk-tools/blob/master/LibCRIComp/LibCRIComp.cpp
4
19
  static PyObject* compressCRILAYLA(PyObject* module, PyObject* args, PyObject* kwargs)
5
20
  {
@@ -160,6 +175,18 @@ static inline uint16_t get_next_bits(unsigned char* input_buffer, long* const of
160
175
  return out_bits;
161
176
  }
162
177
 
178
+ PyDoc_STRVAR(decompressCRILAYLA_doc,
179
+ "decompressCRILAYLA($module, /, indata)\n"
180
+ "--\n"
181
+ "\n"
182
+ "Decompress CRILAYLA-compressed data.\n"
183
+ "\n"
184
+ "Args:\n"
185
+ " indata (bytes): Compressed data, starting with the CRILAYLA signature.\n"
186
+ "\n"
187
+ "Returns:\n"
188
+ " bytes: The decompressed data.");
189
+
163
190
  // https://github.com/hcs64/vgm_ripping/blob/master/multi/utf_tab/cpk_uncompress.c
164
191
  static PyObject* decompressCRILAYLA(PyObject* m, PyObject* args, PyObject* kwargs)
165
192
  {
@@ -244,20 +271,20 @@ static PyObject* decompressCRILAYLA(PyObject* m, PyObject* args, PyObject* kwarg
244
271
  }
245
272
 
246
273
  static PyMethodDef Cmp_criMethods[] = {
247
- {"compressCRILAYLA", (PyCFunction)compressCRILAYLA, METH_VARARGS | METH_KEYWORDS, "Compress CRILAYLA data."},
248
- {"decompressCRILAYLA", (PyCFunction)decompressCRILAYLA, METH_VARARGS | METH_KEYWORDS, "Decompress CRILAYLA data."},
274
+ {"compressCRILAYLA", (PyCFunction)compressCRILAYLA, METH_VARARGS | METH_KEYWORDS, compressCRILAYLA_doc},
275
+ {"decompressCRILAYLA", (PyCFunction)decompressCRILAYLA, METH_VARARGS | METH_KEYWORDS, decompressCRILAYLA_doc},
249
276
  {NULL, NULL, 0, NULL}
250
277
  };
251
278
 
252
279
  static struct PyModuleDef cmp_crimodule = {
253
280
  PyModuleDef_HEAD_INIT,
254
281
  "cmp_cri",
255
- "CRILAYLA functions.",
282
+ "C implementation of the CRILAYLA compression scheme used by CPK archives.",
256
283
  -1,
257
284
  Cmp_criMethods
258
285
  };
259
286
 
260
287
  PyMODINIT_FUNC PyInit_cmp_cri(void)
261
288
  {
262
- return PyModule_Create(&cmp_crimodule);
289
+ return moduleCreate(&cmp_crimodule);
263
290
  }
@@ -0,0 +1,405 @@
1
+ #include "inc.h"
2
+
3
+ // Free the buffer before raising, as the checks happen after allocating it
4
+ #define ERROR_CHECK_FREE(cond, error, ptr) if (cond) { PyMem_Free(ptr); PyErr_SetString(PyExc_ValueError, error); return NULL; }
5
+
6
+ // A tree can hold at most 256 leaves, one per possible value, plus the 255 branches joining them
7
+ #define MAX_NODES 512
8
+ // With at most 256 leaves the tree is at most 255 levels deep, so codes are at most 255 bits long
9
+ #define MAX_CODE_BITS 256
10
+
11
+ PyDoc_STRVAR(decompressHuffman_doc,
12
+ "decompressHuffman($module, /, rawdata, decomplength, numbits=8, little=True)\n"
13
+ "--\n"
14
+ "\n"
15
+ "Decompress Huffman-coded data, as used by the GBA and NDS BIOS.\n"
16
+ "\n"
17
+ "The data is expected to start with the tree size and the tree itself,\n"
18
+ "followed by the 32-bit codeword stream, without the 4-byte header with\n"
19
+ "compression type and length.\n"
20
+ "\n"
21
+ "Args:\n"
22
+ " rawdata (bytes): Compressed data to read.\n"
23
+ " decomplength (int): Length of the decompressed data.\n"
24
+ " numbits (int): Data size in bits, 8 or 4.\n"
25
+ " little (bool): Whether nibbles are ordered low-first when numbits is 4.\n"
26
+ "\n"
27
+ "Returns:\n"
28
+ " bytes: The decompressed data.");
29
+
30
+ static PyObject* decompressHuffman(PyObject* m, PyObject* args, PyObject* kwargs)
31
+ {
32
+ static char *kwlist[] = { "rawdata", "decomplength", "numbits", "little", NULL };
33
+
34
+ unsigned char* rawdata;
35
+ size_t datalength;
36
+ unsigned int decomplength;
37
+ int numbits = 8;
38
+ int little = 1;
39
+
40
+ if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s#I|ip", kwlist, &rawdata, &datalength, &decomplength, &numbits, &little))
41
+ return NULL;
42
+ ERROR_CHECK(numbits != 8 && numbits != 4, "Data size must be 8 or 4 bits.");
43
+ ERROR_CHECK(datalength < 2, "Not enough data.");
44
+
45
+ unsigned int treesize = rawdata[0];
46
+ unsigned int treeroot = rawdata[1];
47
+ unsigned char* treebuffer = rawdata + 2;
48
+ size_t treelength = (size_t)treesize * 2;
49
+ ERROR_CHECK(datalength < 2 + treelength, "Not enough data.");
50
+
51
+ // Every written byte holds numbits bits of the decompressed data
52
+ size_t outlength = (size_t)decomplength * (8 / numbits);
53
+ unsigned char* out = PyMem_Malloc(outlength > 0 ? outlength : 1);
54
+ MALLOC_CHECK(out);
55
+
56
+ size_t readbytes = 2 + treelength;
57
+ ERROR_CHECK_FREE(readbytes + 4 > datalength, "Not enough data.", out);
58
+ unsigned int code = READ_32(rawdata, readbytes);
59
+ readbytes += 4;
60
+
61
+ size_t writebytes = 0;
62
+ unsigned int i = 0;
63
+ unsigned int next = 0;
64
+ unsigned int pos = treeroot;
65
+ while (writebytes < outlength)
66
+ {
67
+ // All the bits of the current codeword have been read, get a new one
68
+ if (i == 32)
69
+ {
70
+ ERROR_CHECK_FREE(readbytes + 4 > datalength, "Not enough data.", out);
71
+ code = READ_32(rawdata, readbytes);
72
+ readbytes += 4;
73
+ i = 0;
74
+ }
75
+ next += (pos & 0x3f) * 2 + 2;
76
+ unsigned int direction = ((code >> (31 - i)) & 1) == 0 ? 2 : 1;
77
+ int leaf = (((pos >> 5) >> direction) & 1) != 0;
78
+ ERROR_CHECK_FREE(next - direction >= treelength, "Tree offset out of range.", out);
79
+ pos = treebuffer[next - direction];
80
+ if (leaf)
81
+ {
82
+ out[writebytes++] = (unsigned char)(pos & 0xff);
83
+ pos = treeroot;
84
+ next = 0;
85
+ }
86
+ ++i;
87
+ }
88
+
89
+ // Pack the nibbles back into bytes
90
+ if (numbits == 4)
91
+ {
92
+ for (unsigned int j = 0; j < decomplength; ++j)
93
+ {
94
+ unsigned char b1 = out[2 * j + 1];
95
+ unsigned char b2 = out[2 * j];
96
+ if (little)
97
+ out[j] = (unsigned char)((b1 * 16 + b2) & 0xff);
98
+ else
99
+ out[j] = (unsigned char)((b2 * 16 + b1) & 0xff);
100
+ }
101
+ }
102
+
103
+ PyObject *output = PyBytes_FromStringAndSize(out, decomplength);
104
+ PyMem_Free(out);
105
+ return output;
106
+ }
107
+
108
+ // Node of the binary tree built by compressHuffman
109
+ typedef struct HuffNode
110
+ {
111
+ unsigned int freqcount;
112
+ int code;
113
+ int score;
114
+ int haschildren;
115
+ struct HuffNode* children[2];
116
+ } HuffNode;
117
+
118
+ // Sort the nodes by frequency, keeping the order of equal elements.
119
+ // The list is always sorted except for the node appended by the last iteration,
120
+ // so the insertion sort only has to move that one back into place.
121
+ static void sortByFreq(HuffNode** freq, int freqlen)
122
+ {
123
+ for (int i = 1; i < freqlen; ++i)
124
+ {
125
+ HuffNode* node = freq[i];
126
+ int j = i - 1;
127
+ while (j >= 0 && freq[j]->freqcount > node->freqcount)
128
+ {
129
+ freq[j + 1] = freq[j];
130
+ --j;
131
+ }
132
+ freq[j + 1] = node;
133
+ }
134
+ }
135
+
136
+ // Store the code of every leaf node under this node as a string of 0 and 1 bits
137
+ static void getHuffCodes(HuffNode* node, unsigned char* seed, int seedlen, unsigned char* codebits, int* codelengths)
138
+ {
139
+ if (!node->haschildren)
140
+ {
141
+ for (int i = 0; i < seedlen; ++i)
142
+ codebits[node->code * MAX_CODE_BITS + i] = seed[i];
143
+ codelengths[node->code] = seedlen;
144
+ return;
145
+ }
146
+ for (int i = 0; i < 2; ++i)
147
+ {
148
+ seed[seedlen] = (unsigned char)i;
149
+ getHuffCodes(node->children[i], seed, seedlen + 1, codebits, codelengths);
150
+ }
151
+ }
152
+
153
+ PyDoc_STRVAR(compressHuffman_doc,
154
+ "compressHuffman($module, /, indata, numbits=8, little=True)\n"
155
+ "--\n"
156
+ "\n"
157
+ "Compress data with Huffman coding, as used by the GBA and NDS BIOS.\n"
158
+ "\n"
159
+ "The output starts with the tree size and the tree itself, followed by\n"
160
+ "the 32-bit codeword stream. The 4-byte header with compression type and\n"
161
+ "length is not included and should be written separately.\n"
162
+ "\n"
163
+ "Args:\n"
164
+ " indata (bytes): Data to compress.\n"
165
+ " numbits (int): Data size in bits, 8 or 4.\n"
166
+ " little (bool): Whether nibbles are ordered low-first when numbits is 4.\n"
167
+ "\n"
168
+ "Returns:\n"
169
+ " bytes: The compressed data.");
170
+
171
+ static PyObject* compressHuffman(PyObject* m, PyObject* args, PyObject* kwargs)
172
+ {
173
+ static char *kwlist[] = { "indata", "numbits", "little", NULL };
174
+
175
+ unsigned char* indata;
176
+ size_t inlength;
177
+ int numbits = 8;
178
+ int little = 1;
179
+
180
+ if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s#|ip", kwlist, &indata, &inlength, &numbits, &little))
181
+ return NULL;
182
+ ERROR_CHECK(numbits != 8 && numbits != 4, "Data size must be 8 or 4 bits.");
183
+ ERROR_CHECK(inlength == 0, "No data to compress.");
184
+
185
+ // Read indata as nibbles if numbits is 4
186
+ unsigned char* workdata = indata;
187
+ size_t worklength = inlength;
188
+ unsigned char* nibbles = NULL;
189
+ if (numbits == 4)
190
+ {
191
+ nibbles = PyMem_Malloc(inlength * 2);
192
+ MALLOC_CHECK(nibbles);
193
+ for (size_t i = 0; i < inlength; ++i)
194
+ {
195
+ unsigned char b1 = indata[i] % 16;
196
+ unsigned char b2 = indata[i] / 16;
197
+ nibbles[2 * i] = little ? b1 : b2;
198
+ nibbles[2 * i + 1] = little ? b2 : b1;
199
+ }
200
+ workdata = nibbles;
201
+ worklength = inlength * 2;
202
+ }
203
+
204
+ // Get frequencies
205
+ unsigned int freqcount[256] = { 0 };
206
+ for (size_t i = 0; i < worklength; ++i)
207
+ ++freqcount[workdata[i]];
208
+
209
+ HuffNode* nodes = PyMem_Malloc(MAX_NODES * sizeof(HuffNode));
210
+ HuffNode** freq = PyMem_Malloc(MAX_NODES * sizeof(HuffNode*));
211
+ HuffNode** lst = PyMem_Malloc(MAX_NODES * sizeof(HuffNode*));
212
+ unsigned char* codebits = PyMem_Malloc(256 * MAX_CODE_BITS);
213
+ if (nodes == NULL || freq == NULL || lst == NULL || codebits == NULL)
214
+ {
215
+ PyMem_Free(nibbles);
216
+ PyMem_Free(nodes);
217
+ PyMem_Free(freq);
218
+ PyMem_Free(lst);
219
+ PyMem_Free(codebits);
220
+ return PyErr_NoMemory();
221
+ }
222
+ int nodecount = 0;
223
+ int freqlen = 0;
224
+ for (int i = 0; i < 256; ++i)
225
+ {
226
+ if (freqcount[i] > 0)
227
+ {
228
+ HuffNode* node = &nodes[nodecount++];
229
+ node->freqcount = freqcount[i];
230
+ node->code = i;
231
+ node->score = 0;
232
+ node->haschildren = 0;
233
+ freq[freqlen++] = node;
234
+ }
235
+ }
236
+
237
+ // Add a stub entry in the special case that there's only one item
238
+ if (freqlen == 1)
239
+ {
240
+ HuffNode* node = &nodes[nodecount++];
241
+ node->freqcount = 0;
242
+ node->code = (workdata[0] + 1) & 0xff;
243
+ node->score = 0;
244
+ node->haschildren = 0;
245
+ freq[freqlen++] = node;
246
+ }
247
+
248
+ // Sort and create the tree
249
+ while (freqlen > 1)
250
+ {
251
+ sortByFreq(freq, freqlen);
252
+ HuffNode* node = &nodes[nodecount++];
253
+ node->freqcount = freq[0]->freqcount + freq[1]->freqcount;
254
+ node->code = 0;
255
+ node->score = 0;
256
+ node->haschildren = 1;
257
+ node->children[0] = freq[0];
258
+ node->children[1] = freq[1];
259
+ for (int i = 2; i < freqlen; ++i)
260
+ freq[i - 2] = freq[i];
261
+ freqlen -= 2;
262
+ freq[freqlen++] = node;
263
+ }
264
+
265
+ // Label nodes to keep bandwidth small
266
+ int lstlen = 0;
267
+ while (freqlen > 0)
268
+ {
269
+ // Take the node with the lowest score, the first one if there are ties
270
+ int best = 0;
271
+ for (int i = 0; i < freqlen; ++i)
272
+ {
273
+ freq[i]->score = freq[i]->code - i;
274
+ if (freq[i]->score < freq[best]->score)
275
+ best = i;
276
+ }
277
+ HuffNode* node = freq[best];
278
+ for (int i = best; i < freqlen - 1; ++i)
279
+ freq[i] = freq[i + 1];
280
+ --freqlen;
281
+ node->code = (lstlen - node->code) & 0xff;
282
+ lst[lstlen++] = node;
283
+ if (node->haschildren)
284
+ {
285
+ for (int i = 1; i >= 0; --i)
286
+ {
287
+ if (node->children[i]->haschildren)
288
+ {
289
+ node->children[i]->code = lstlen & 0xff;
290
+ freq[freqlen++] = node->children[i];
291
+ }
292
+ }
293
+ }
294
+ }
295
+
296
+ // Convert our list of nodes to a table of bytes -> huffman codes
297
+ int codelengths[256] = { 0 };
298
+ unsigned char seed[MAX_CODE_BITS];
299
+ getHuffCodes(lst[0], seed, 0, codebits, codelengths);
300
+
301
+ // The output holds the tree, then the codes padded to a multiple of 32 bits
302
+ size_t totalbits = 0;
303
+ for (int i = 0; i < 256; ++i)
304
+ totalbits += (size_t)freqcount[i] * codelengths[i];
305
+ size_t outcap = 2 + 2 * (size_t)lstlen + 2 + (totalbits + 31) / 32 * 4 + 4;
306
+ unsigned char* out = PyMem_Malloc(outcap);
307
+ if (out == NULL)
308
+ {
309
+ PyMem_Free(nibbles);
310
+ PyMem_Free(nodes);
311
+ PyMem_Free(freq);
312
+ PyMem_Free(lst);
313
+ PyMem_Free(codebits);
314
+ return PyErr_NoMemory();
315
+ }
316
+ size_t outlength = 0;
317
+
318
+ // Write header
319
+ out[outlength++] = (unsigned char)(lstlen & 0xff);
320
+
321
+ // Write Huffman tree, reusing the now empty freq list to hold it
322
+ HuffNode** tree = freq;
323
+ int treelen = 0;
324
+ tree[treelen++] = lst[0];
325
+ for (int i = 0; i < lstlen; ++i)
326
+ {
327
+ if (lst[i]->haschildren)
328
+ for (int j = 0; j < 2; ++j)
329
+ tree[treelen++] = lst[i]->children[j];
330
+ }
331
+ for (int i = 0; i < treelen; ++i)
332
+ {
333
+ HuffNode* node = tree[i];
334
+ if (node->haschildren)
335
+ {
336
+ int childsum = 0;
337
+ for (int j = 0; j < 2; ++j)
338
+ if (!node->children[j]->haschildren)
339
+ childsum += (0x80 >> j) & 0xff;
340
+ node->code |= childsum & 0xff;
341
+ }
342
+ out[outlength++] = (unsigned char)(node->code & 0xff);
343
+ }
344
+
345
+ // Ensure nodes are word-aligned
346
+ if (lstlen % 2 == 0)
347
+ {
348
+ out[outlength++] = 0;
349
+ out[outlength++] = 0;
350
+ out[0] = (unsigned char)((lstlen + 1) & 0xff);
351
+ }
352
+
353
+ // Write bits to stream
354
+ unsigned int data = 0;
355
+ size_t setbits = 0;
356
+ for (size_t i = 0; i < worklength; ++i)
357
+ {
358
+ unsigned char* bits = codebits + workdata[i] * MAX_CODE_BITS;
359
+ int bitslength = codelengths[workdata[i]];
360
+ for (int j = 0; j < bitslength; ++j)
361
+ {
362
+ data = data * 2 + bits[j];
363
+ ++setbits;
364
+ if (setbits % 32 == 0)
365
+ {
366
+ WRITE_32(out, outlength, data);
367
+ outlength += 4;
368
+ data = 0;
369
+ }
370
+ }
371
+ }
372
+ if (setbits % 32 != 0)
373
+ {
374
+ WRITE_32(out, outlength, data << (32 - (setbits % 32)));
375
+ outlength += 4;
376
+ }
377
+
378
+ PyObject *output = PyBytes_FromStringAndSize(out, outlength);
379
+ PyMem_Free(out);
380
+ PyMem_Free(nibbles);
381
+ PyMem_Free(nodes);
382
+ PyMem_Free(freq);
383
+ PyMem_Free(lst);
384
+ PyMem_Free(codebits);
385
+ return output;
386
+ }
387
+
388
+ static PyMethodDef Cmp_huffMethods[] = {
389
+ {"decompressHuffman", (PyCFunction)decompressHuffman, METH_VARARGS | METH_KEYWORDS, decompressHuffman_doc},
390
+ {"compressHuffman", (PyCFunction)compressHuffman, METH_VARARGS | METH_KEYWORDS, compressHuffman_doc},
391
+ {NULL, NULL, 0, NULL}
392
+ };
393
+
394
+ static struct PyModuleDef cmp_huffmodule = {
395
+ PyModuleDef_HEAD_INIT,
396
+ "cmp_huff",
397
+ "C implementation of the Huffman coding scheme used by the GBA and NDS BIOS.",
398
+ -1,
399
+ Cmp_huffMethods
400
+ };
401
+
402
+ PyMODINIT_FUNC PyInit_cmp_huff(void)
403
+ {
404
+ return moduleCreate(&cmp_huffmodule);
405
+ }