hacktools 0.33.1__tar.gz → 0.35.0__tar.gz

{hacktools-0.33.1 → hacktools-0.35.0}/LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2020-2023 Illidan
+Copyright (c) 2020-2024 Illidan
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

{hacktools-0.33.1/hacktools.egg-info → hacktools-0.35.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hacktools
-Version: 0.33.1
+Version: 0.35.0
 Summary: A set of utilities and tools for rom hacking and translations.
 Home-page: https://github.com/Illidanz/hacktools
 Author: Illidan
@@ -96,7 +96,7 @@ External dependencies not included are marked as `(through *dependency*)`
 ### Other / Generic
 - CPK archives
 - ARCH archives
-- LZ10, LZ11, Huffman and CRILAYLA compression/decompression
+- LZ10, LZ11, Huffman, CRILAYLA and RACJIN compression/decompression
 - ARM/MIPS binary patching
 - xdelta patch creation
 - IPS patch creation

{hacktools-0.33.1 → hacktools-0.35.0}/README.md

@@ -47,7 +47,7 @@ External dependencies not included are marked as `(through *dependency*)`
 ### Other / Generic
 - CPK archives
 - ARCH archives
-- LZ10, LZ11, Huffman and CRILAYLA compression/decompression
+- LZ10, LZ11, Huffman, CRILAYLA and RACJIN compression/decompression
 - ARM/MIPS binary patching
 - xdelta patch creation
 - IPS patch creation

hacktools-0.35.0/hacktools/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.35.0"

hacktools-0.35.0/hacktools/c_ext/cmp_racjin.c

@@ -0,0 +1,227 @@
+#include "inc.h"
+
+// https://github.com/Raw-man/Racjin-de-compression/blob/master/src/encode.cpp
+static PyObject* compressRACJIN(PyObject* module, PyObject* args, PyObject* kwargs)
+{
+    static char *kwlist[] = { "indata", NULL };
+
+    unsigned char* src;
+    size_t srclen;
+
+    if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s#", kwlist, &src, &srclen))
+        return NULL;
+
+    unsigned char* compressed_buffer = PyMem_Malloc(srclen * 2);
+    MALLOC_CHECK(compressed_buffer);
+    unsigned int destlen = 0;
+
+    unsigned int index = 0; //position of an element from the input buffer
+    unsigned char last_enc_byte = 0;//last encoded byte
+    unsigned char bit_shift = 0; //shift by bitShift (used to fold codes)
+    unsigned char* frequencies = PyMem_Calloc(256, sizeof(unsigned char));
+    MALLOC_CHECK(frequencies);
+    for (int i = 0; i < 256; ++i)
+        frequencies[i] = 0;
+    unsigned int* seq_indices = PyMem_Calloc(8192, sizeof(unsigned int));
+    MALLOC_CHECK(seq_indices);
+    for (int i = 0; i < 8192; ++i)
+        seq_indices[i] = 0;
+    unsigned short* codes = PyMem_Calloc(srclen, sizeof(unsigned short));
+    MALLOC_CHECK(codes);
+    for (int i = 0; i < srclen; ++i)
+        codes[i] = 0;
+    unsigned int codeslen = 0;
+
+    while (index < srclen)
+    {
+        unsigned char best_freq = 0;
+        unsigned char best_match = 0;
+
+        // To get the exact same compression for CDDATA.DIG files uncomment the following:
+        if (frequencies[last_enc_byte] == 256)
+            frequencies[last_enc_byte] = 0x00;
+
+        unsigned char positions_to_check = frequencies[last_enc_byte] < 32 ? (frequencies[last_enc_byte] & 0x1F) : 32;
+        unsigned int seq_index = index;
+
+        for (unsigned char freq = 0; freq < positions_to_check; freq++)
+        {
+            unsigned short key = freq + last_enc_byte * 32; //0x1F + 0xFF*32 = 8191
+            unsigned int src_index = seq_indices[key];
+            unsigned char matched = 0;
+            unsigned char max_length = index + 8 < srclen ? 8 : srclen - index;
+
+            for (unsigned char offset = 0; offset < max_length; ++offset)
+            {
+                if (src[src_index + offset] == src[index + offset])
+                    ++matched;
+                else
+                    break;
+            }
+
+            if (matched > best_match)
+            {
+                best_freq = freq;
+                best_match = matched;
+            }
+        }
+
+        unsigned short code = 0x00;
+        if (best_match > 0) //found a better match?
+        {
+            code = code | (best_freq << 3); //f|ooooolll //f=0 (flag), o - occurrences/frequency, l -length
+            code = code | (best_match - 1); //encode a reference
+            index += best_match;
+        }
+        else //encode byte literal
+        {
+            code = 0x100 | src[index]; //f|bbbbbbbb //f=1
+            ++index;
+        }
+
+        code = code << bit_shift; //prepare for folding
+        codes[codeslen] = code;
+        codeslen++;
+
+        ++bit_shift;
+        if (bit_shift == 8)
+            bit_shift = 0;
+
+        unsigned int key = (frequencies[last_enc_byte] & 0x1F) + last_enc_byte * 32; //0x1F + 0xFF*32 = 8191
+        seq_indices[key] = seq_index;
+        frequencies[last_enc_byte] = frequencies[last_enc_byte] + 1; //increase by 1 (up to 31)
+        last_enc_byte = src[index - 1];
+    }
+    //Fold codes (8 codes, 16 bytes -> 8 codes, 9 bytes)
+    for (unsigned int i = 0; i < codeslen; i = i + 8)
+    {
+        unsigned char group_size = i + 8 < codeslen ? 8 : codeslen - i;
+        for (unsigned char s = 0; s <= group_size; s += 2)
+        {
+            unsigned short first = s > 0 ? codes[s + i - 1] : 0x00;
+            unsigned short middle = s < group_size ? codes[s + i] : 0x00;
+            unsigned short last = s < group_size - 1 ? codes[s + i + 1] : 0x00;
+            unsigned short result = middle | (first >> 8) | (last << 8);
+            compressed_buffer[destlen] = result & 0xFF;
+            ++destlen;
+            if (s < group_size)
+            {
+                compressed_buffer[destlen] = result >> 8;
+                ++destlen;
+            }
+        }
+    }
+
+    PyObject *output = PyBytes_FromStringAndSize(compressed_buffer, destlen);
+    PyMem_Free(compressed_buffer);
+    PyMem_Free(frequencies);
+    PyMem_Free(seq_indices);
+    PyMem_Free(codes);
+    return output;
+}
+
+// https://github.com/Raw-man/Racjin-de-compression/blob/master/src/decode.cpp
+static PyObject* decompressRACJIN(PyObject* m, PyObject* args, PyObject* kwargs)
+{
+    static char *kwlist[] = { "indata", "decomplength", NULL };
+
+    unsigned char* input_buffer;
+    size_t input_size;
+    unsigned int decomplength;
+
+    if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s#I", kwlist, &input_buffer, &input_size, &decomplength))
+        return NULL;
+
+    unsigned int index = 0; //position of a byte from the input buffer
+    unsigned int dest_index = 0; //position (destination) of a decoded byte in the output buffer
+    unsigned char last_dec_byte = 0; //last decoded byte of the previus decoding iteration
+    unsigned char bit_shift = 0; //shift right by bitShift
+
+    // Allocate buffers
+    unsigned char* frequencies = PyMem_Malloc(256);
+    MALLOC_CHECK(frequencies);
+    for (int i = 0; i < 256; ++i)
+        frequencies[i] = 0;
+    unsigned int* seq_indices = PyMem_Calloc(8192, sizeof(unsigned int));
+    MALLOC_CHECK(seq_indices);
+    for (int i = 0; i < 8192; ++i)
+        seq_indices[i] = 0;
+    unsigned char* decompressed_buffer = PyMem_Malloc(decomplength);
+    MALLOC_CHECK(decompressed_buffer);
+
+    while (index < input_size)
+    {
+        unsigned short next_code = input_buffer[index + 1];  //next pair of bytes to decode from the input buffer
+        next_code = next_code << 8;
+        next_code = next_code | input_buffer[index];
+        next_code = next_code >> bit_shift; //unfold 9 bit token
+
+        //The result can be interpreted as follows:
+        // iiiiiiif|ooooolll //f=0
+        // iiiiiiif|bbbbbbbb //f=1
+        //i - ignore
+        //f - flag  (is literal or offset/length pair)
+        //l - length (add 1 to get the real length)
+        //o - occurrences/frequency
+        //b - byte literal
+
+        ++bit_shift;
+        ++index;
+
+        if (bit_shift == 8)
+        {
+            bit_shift = 0;
+            ++index;
+        }
+
+        unsigned int seq_index = dest_index; //start of a byte sequence
+        if ((next_code & 0x100) != 0) // bit flag: is nextToken a literal or a reference?
+        {
+            decompressed_buffer[dest_index] = (unsigned char)(next_code & 0xFF); //store the literal
+            ++dest_index;
+        }
+        else
+        {
+            unsigned int key = ((next_code >> 3) & 0x1F) + last_dec_byte * 32; //0x1F + 0xFF*32 = 8191
+            unsigned int src_index = seq_indices[key]; //get a reference to a previously decoded sequence
+
+            for (unsigned char length = 0; length < (next_code & 0x07) + 1; ++length, ++dest_index, ++src_index)
+            {
+                decompressed_buffer[dest_index] = decompressed_buffer[src_index]; //copy a previously decoded byte sequence (up to 8)
+            }
+        }
+
+        if (dest_index >= decomplength)
+            break;
+
+        unsigned int key = frequencies[last_dec_byte] + last_dec_byte * 32; //0x1F + 0xFF*32 = 8191
+        seq_indices[key] = seq_index;
+        frequencies[last_dec_byte] = (unsigned char)((frequencies[last_dec_byte] + 1) & 0x1F); //increase by 1 (up to 31)
+        last_dec_byte = decompressed_buffer[dest_index - 1];
+    }
+
+    PyObject *output = PyBytes_FromStringAndSize(decompressed_buffer, decomplength);
+    PyMem_Free(frequencies);
+    PyMem_Free(seq_indices);
+    PyMem_Free(decompressed_buffer);
+    return output;
+}
+
+static PyMethodDef Cmp_racjinMethods[] = {
+    {"compressRACJIN", (PyCFunction)compressRACJIN, METH_VARARGS | METH_KEYWORDS, "Compress RACJIN data."},
+    {"decompressRACJIN", (PyCFunction)decompressRACJIN, METH_VARARGS | METH_KEYWORDS, "Decompress RACJIN data."},
+    {NULL, NULL, 0, NULL}
+};
+
+static struct PyModuleDef cmp_racjinmodule = {
+    PyModuleDef_HEAD_INIT,
+    "cmp_racjin",
+    "RACJIN functions.",
+    -1,
+    Cmp_racjinMethods
+};
+
+PyMODINIT_FUNC PyInit_cmp_racjin(void)
+{
+    return PyModule_Create(&cmp_racjinmodule);
+}

{hacktools-0.33.1 → hacktools-0.35.0}/hacktools/common.py

@@ -756,7 +756,7 @@ class TranslationFile:
             note = ET.SubElement(unit, "note")
             note.text = comment
 
-    def preloadLookup(self):
+    def preloadLookup(self, comments="#"):
         self.lookup = {}
         self.chartot = 0
         self.transtot = 0
@@ -764,6 +764,8 @@ class TranslationFile:
             for unit in self.files[file][0]:
                 self.chartot += len(unit[0].text)
                 if unit[1].text is not None and unit[1].text != "":
+                    if comments in unit[1].text:
+                        unit[1].text = unit[1].text.split(comments)[0]
                     self.lookup[unit[0].text] = unit[1].text
                     self.transtot += len(unit[0].text)
 
@@ -1030,7 +1032,7 @@ class BinaryPointer:
         self.str = str
 
 
-def repackBinaryStrings(section, infile, outfile, binranges, freeranges=None, readfunc=detectEncodedString, writefunc=writeEncodedString, encoding="shift_jis", pointerstart=0, injectstart=0, fallbackf=None, injectfallback=0, sectionname="bin"):
+def repackBinaryStrings(section, infile, outfile, binranges, freeranges=None, readfunc=detectEncodedString, writefunc=writeEncodedString, encoding="shift_jis", pointerstart=0, injectstart=0, fallbackf=None, injectfallback=0, sectionname="bin", preformat=None, postformat=None):
     insize = os.path.getsize(infile)
     notfound = []
     with Stream(infile, "rb") as fi:
@@ -1045,6 +1047,9 @@ def repackBinaryStrings(section, infile, outfile, binranges, freeranges=None, re
                     pos = fi.tell()
                     check = readfunc(fi, encoding)
                     if check != "":
+                        pre = post = ""
+                        if preformat != None:
+                            check, pre, post = preformat(check)
                         if isinstance(section, TranslationFile):
                             newsjis = section.getEntry(check, sectionname, pos)
                         else:
@@ -1055,6 +1060,8 @@ def repackBinaryStrings(section, infile, outfile, binranges, freeranges=None, re
                         if newsjis != "":
                             if newsjis == "!":
                                 newsjis = ""
+                            if postformat != None:
+                                newsjis = postformat(newsjis, pre, post)
                             newsjislog = newsjis.encode("ascii", "ignore")
                             logDebug("Replacing string at", toHex(pos), "with", newsjislog)
                             fo.seek(pos)
@@ -1104,7 +1111,10 @@ def repackBinaryStrings(section, infile, outfile, binranges, freeranges=None, re
                                             newpointer = range[0]
                                             # For the injected range, add injectstart, otherwise add pointerstart
                                             if (len(range) == 3):
-                                                newpointer += injectstart
+                                                if isinstance(range[2], bool):
+                                                    newpointer += injectstart
+                                                else:
+                                                    newpointer += int(range[2])
                                             else:
                                                 newpointer += pointerstart
                                             range[0] = fo.tell()

{hacktools-0.33.1 → hacktools-0.35.0}/hacktools/nds.py

@@ -134,7 +134,7 @@ def extractBIN(binrange, readfunc=common.detectEncodedString, encoding="shift_ji
 
 
 def repackBIN(binrange, freeranges=[], readfunc=common.detectEncodedString, writefunc=common.writeEncodedString, encoding="shift_jis", comments="#",
-              binin="data/extract/arm9.bin", binout="data/repack/arm9.bin", binfile="data/bin_input.txt", fixchars=[], pointerstart=0x02000000, injectstart=0x02000000, fallbackf=None, injectfallback=0, nocopy=False, sectionname="bin"):
+              binin="data/extract/arm9.bin", binout="data/repack/arm9.bin", binfile="data/bin_input.txt", fixchars=[], pointerstart=0x02000000, injectstart=0x02000000, fallbackf=None, injectfallback=0, nocopy=False, sectionname="bin", preformat=None, postformat=None):
     if not os.path.isfile(binfile):
         common.logError("Input file", binfile, "not found")
         return False
@@ -152,7 +152,7 @@ def repackBIN(binrange, freeranges=[], readfunc=common.detectEncodedString, writ
         section.preloadLookup()
     if type(binrange) == tuple:
         binrange = [binrange]
-    notfound = common.repackBinaryStrings(section, binin, binout, binrange, freeranges, readfunc, writefunc, encoding, pointerstart, injectstart, fallbackf, injectfallback, sectionname)
+    notfound = common.repackBinaryStrings(section, binin, binout, binrange, freeranges, readfunc, writefunc, encoding, pointerstart, injectstart, fallbackf, injectfallback, sectionname, preformat, postformat)
     for pointer in notfound:
         common.logError("Pointer", common.toHex(pointer.old), "->", common.toHex(pointer.new), "not found for string", pointer.str)
     if binfile.endswith(".txt"):
@@ -176,13 +176,17 @@ class BINSection:
             self.data = bytearray(ramlen)
 
 
-def expandBIN(binin, binout, headerin, headerout, newlength, injectpos):
+def expandBIN(binin, binout, headerin, headerout, newlengths, injectpos):
     if not os.path.isfile(binin):
         common.logError("Input file", binin, "not found")
         return False
     if not os.path.isfile(headerin):
         common.logError("Header file", headerin, "not found")
         return False
+    if not isinstance(newlengths, list):
+        newlengths = [newlengths]
+    if not isinstance(injectpos, list):
+        injectpos = [injectpos]
     codesettings = -1
     with common.Stream(headerin, "rb") as fin:
         fin.seek(0x20)
@@ -225,13 +229,15 @@ def expandBIN(binin, binout, headerin, headerout, newlength, injectpos):
             sections.append(BINSection(fin, start, size, datastart, bsssize))
             datastart += size
     # Write the new extended arm9.bin
-    sections.append(BINSection(None, injectpos, newlength, 0, 0))
+    for i in range(len(newlengths)):
+        sections.append(BINSection(None, injectpos[i], newlengths[i], 0, 0, True if i < len(newlengths) - 1 else False))
     with common.Stream(binout, "wb") as f:
         # Write the section data first
         f.write(sections[0].data)
         datastart = f.tell()
         for i in range(1, len(sections)):
             sections[i].offset = f.tell()
+            common.logDebug("Section", i, "offset:", common.toHex(f.tell()))
             f.write(sections[i].data)
         # Write the new copytable
         copytablestart = f.tell()

{hacktools-0.33.1 → hacktools-0.35.0}/hacktools/psp.py

@@ -31,7 +31,7 @@ def extractIso(isofile, extractfolder, workfolder="", fixfilename=False):
     common.logMessage("Done!")
 
 
-def repackIso(isofile, isopatch, workfolder, patchfile=""):
+def repackIso(isofile, isopatch, workfolder, patchfile="", fixfilename=False):
     try:
         import pycdlib
     except ImportError:
@@ -45,7 +45,7 @@ def repackIso(isofile, isopatch, workfolder, patchfile=""):
     for file in common.showProgress(files):
         filelen = os.path.getsize(workfolder + file)
         with open(workfolder + file, "rb") as f:
-            iso.modify_file_in_place(f, filelen, "/" + file)
+            iso.modify_file_in_place(f, filelen, "/" + file + (";1" if fixfilename and not file.endswith(".") else ""))
     iso.close()
     common.logMessage("Done!")
     # Create xdelta patch

{hacktools-0.33.1 → hacktools-0.35.0/hacktools.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hacktools
-Version: 0.33.1
+Version: 0.35.0
 Summary: A set of utilities and tools for rom hacking and translations.
 Home-page: https://github.com/Illidanz/hacktools
 Author: Illidan
@@ -96,7 +96,7 @@ External dependencies not included are marked as `(through *dependency*)`
 ### Other / Generic
 - CPK archives
 - ARCH archives
-- LZ10, LZ11, Huffman and CRILAYLA compression/decompression
+- LZ10, LZ11, Huffman, CRILAYLA and RACJIN compression/decompression
 - ARM/MIPS binary patching
 - xdelta patch creation
 - IPS patch creation

{hacktools-0.33.1 → hacktools-0.35.0}/hacktools.egg-info/SOURCES.txt

@@ -25,5 +25,6 @@ hacktools/assets/icon.png
 hacktools/c_ext/cmp_cri.c
 hacktools/c_ext/cmp_lzss.c
 hacktools/c_ext/cmp_misc.c
+hacktools/c_ext/cmp_racjin.c
 hacktools/c_ext/inc.h
 tests/test_cmp.py

{hacktools-0.33.1 → hacktools-0.35.0}/setup.py

@@ -41,6 +41,7 @@ setup(
         Extension("hacktools.cmp_lzss", sources=["hacktools/c_ext/cmp_lzss.c"]),
         Extension("hacktools.cmp_cri",  sources=["hacktools/c_ext/cmp_cri.c"]),
         Extension("hacktools.cmp_misc", sources=["hacktools/c_ext/cmp_misc.c"]),
+        Extension("hacktools.cmp_racjin",  sources=["hacktools/c_ext/cmp_racjin.c"]),
     ],
     classifiers=[
         "Programming Language :: Python :: 3.7",

{hacktools-0.33.1 → hacktools-0.35.0}/tests/test_cmp.py

@@ -1,6 +1,6 @@
 import pytest
 import os.path
-from hacktools import cmp_lzss, cmp_cri
+from hacktools import cmp_lzss, cmp_cri, cmp_racjin
 
 @pytest.fixture
 def data():
@@ -28,3 +28,10 @@ def test_cmp_cri(data):
     decmp = cmp_cri.decompressCRILAYLA(cmp)
     assert len(data) == len(decmp)
     assert data == decmp
+
+
+def test_cmp_racjin(data):
+    cmp = cmp_racjin.compressRACJIN(data)
+    decmp = cmp_racjin.decompressRACJIN(cmp, len(data))
+    assert len(data) == len(decmp)
+    assert data == decmp

hacktools-0.33.1/hacktools/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.33.1"