hacktools 0.33.0__tar.gz → 0.34.0__tar.gz

{hacktools-0.33.0 → hacktools-0.34.0}/LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2020-2023 Illidan
+Copyright (c) 2020-2024 Illidan
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

{hacktools-0.33.0/hacktools.egg-info → hacktools-0.34.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hacktools
-Version: 0.33.0
+Version: 0.34.0
 Summary: A set of utilities and tools for rom hacking and translations.
 Home-page: https://github.com/Illidanz/hacktools
 Author: Illidan
@@ -96,7 +96,7 @@ External dependencies not included are marked as `(through *dependency*)`
 ### Other / Generic
 - CPK archives
 - ARCH archives
-- LZ10, LZ11, Huffman and CRILAYLA compression/decompression
+- LZ10, LZ11, Huffman, CRILAYLA and RACJIN compression/decompression
 - ARM/MIPS binary patching
 - xdelta patch creation
 - IPS patch creation

{hacktools-0.33.0 → hacktools-0.34.0}/README.md

@@ -47,7 +47,7 @@ External dependencies not included are marked as `(through *dependency*)`
 ### Other / Generic
 - CPK archives
 - ARCH archives
-- LZ10, LZ11, Huffman and CRILAYLA compression/decompression
+- LZ10, LZ11, Huffman, CRILAYLA and RACJIN compression/decompression
 - ARM/MIPS binary patching
 - xdelta patch creation
 - IPS patch creation

hacktools-0.34.0/hacktools/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.34.0"

hacktools-0.34.0/hacktools/c_ext/cmp_racjin.c

@@ -0,0 +1,227 @@
+#include "inc.h"
+
+// https://github.com/Raw-man/Racjin-de-compression/blob/master/src/encode.cpp
+static PyObject* compressRACJIN(PyObject* module, PyObject* args, PyObject* kwargs)
+{
+    static char *kwlist[] = { "indata", NULL };
+
+    unsigned char* src;
+    size_t srclen;
+
+    if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s#", kwlist, &src, &srclen))
+        return NULL;
+
+    unsigned char* compressed_buffer = PyMem_Malloc(srclen);
+    MALLOC_CHECK(compressed_buffer);
+    unsigned int destlen = 0;
+
+    unsigned int index = 0; //position of an element from the input buffer
+    unsigned char last_enc_byte = 0;//last encoded byte
+    unsigned char bit_shift = 0; //shift by bitShift (used to fold codes)
+    unsigned char* frequencies = PyMem_Calloc(256, sizeof(unsigned char));
+    MALLOC_CHECK(frequencies);
+    for (int i = 0; i < 256; ++i)
+        frequencies[i] = 0;
+    unsigned int* seq_indices = PyMem_Calloc(8192, sizeof(unsigned int));
+    MALLOC_CHECK(seq_indices);
+    for (int i = 0; i < 8192; ++i)
+        seq_indices[i] = 0;
+    unsigned short* codes = PyMem_Calloc(srclen, sizeof(unsigned short));
+    MALLOC_CHECK(codes);
+    for (int i = 0; i < srclen; ++i)
+        codes[i] = 0;
+    unsigned int codeslen = 0;
+
+    while (index < srclen)
+    {
+        unsigned char best_freq = 0;
+        unsigned char best_match = 0;
+
+        // To get the exact same compression for CDDATA.DIG files uncomment the following:
+        if (frequencies[last_enc_byte] == 256)
+            frequencies[last_enc_byte] = 0x00;
+
+        unsigned char positions_to_check = frequencies[last_enc_byte] < 32 ? (frequencies[last_enc_byte] & 0x1F) : 32;
+        unsigned int seq_index = index;
+
+        for (unsigned char freq = 0; freq < positions_to_check; freq++)
+        {
+            unsigned short key = freq + last_enc_byte * 32; //0x1F + 0xFF*32 = 8191
+            unsigned int src_index = seq_indices[key];
+            unsigned char matched = 0;
+            unsigned char max_length = index + 8 < srclen ? 8 : srclen - index;
+
+            for (unsigned char offset = 0; offset < max_length; ++offset)
+            {
+                if (src[src_index + offset] == src[index + offset])
+                    ++matched;
+                else
+                    break;
+            }
+
+            if (matched > best_match)
+            {
+                best_freq = freq;
+                best_match = matched;
+            }
+        }
+
+        unsigned short code = 0x00;
+        if (best_match > 0) //found a better match?
+        {
+            code = code | (best_freq << 3); //f|ooooolll //f=0 (flag), o - occurrences/frequency, l -length
+            code = code | (best_match - 1); //encode a reference
+            index += best_match;
+        }
+        else //encode byte literal
+        {
+            code = 0x100 | src[index]; //f|bbbbbbbb //f=1
+            ++index;
+        }
+
+        code = code << bit_shift; //prepare for folding
+        codes[codeslen] = code;
+        codeslen++;
+
+        ++bit_shift;
+        if (bit_shift == 8)
+            bit_shift = 0;
+
+        unsigned int key = (frequencies[last_enc_byte] & 0x1F) + last_enc_byte * 32; //0x1F + 0xFF*32 = 8191
+        seq_indices[key] = seq_index;
+        frequencies[last_enc_byte] = frequencies[last_enc_byte] + 1; //increase by 1 (up to 31)
+        last_enc_byte = src[index - 1];
+    }
+    //Fold codes (8 codes, 16 bytes -> 8 codes, 9 bytes)
+    for (unsigned int i = 0; i < codeslen; i = i + 8)
+    {
+        unsigned char group_size = i + 8 < codeslen ? 8 : codeslen - i;
+        for (unsigned char s = 0; s <= group_size; s += 2)
+        {
+            unsigned short first = s > 0 ? codes[s + i - 1] : 0x00;
+            unsigned short middle = s < group_size ? codes[s + i] : 0x00;
+            unsigned short last = s < group_size - 1 ? codes[s + i + 1] : 0x00;
+            unsigned short result = middle | (first >> 8) | (last << 8);
+            compressed_buffer[destlen] = result & 0xFF;
+            ++destlen;
+            if (s < group_size)
+            {
+                compressed_buffer[destlen] = result >> 8;
+                ++destlen;
+            }
+        }
+    }
+
+    PyObject *output = PyBytes_FromStringAndSize(compressed_buffer, destlen);
+    PyMem_Free(compressed_buffer);
+    PyMem_Free(frequencies);
+    PyMem_Free(seq_indices);
+    PyMem_Free(codes);
+    return output;
+}
+
+// https://github.com/Raw-man/Racjin-de-compression/blob/master/src/decode.cpp
+static PyObject* decompressRACJIN(PyObject* m, PyObject* args, PyObject* kwargs)
+{
+    static char *kwlist[] = { "indata", "decomplength", NULL };
+
+    unsigned char* input_buffer;
+    size_t input_size;
+    unsigned int decomplength;
+
+    if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s#I", kwlist, &input_buffer, &input_size, &decomplength))
+        return NULL;
+
+    unsigned int index = 0; //position of a byte from the input buffer
+    unsigned int dest_index = 0; //position (destination) of a decoded byte in the output buffer
+    unsigned char last_dec_byte = 0; //last decoded byte of the previus decoding iteration
+    unsigned char bit_shift = 0; //shift right by bitShift
+
+    // Allocate buffers
+    unsigned char* frequencies = PyMem_Malloc(256);
+    MALLOC_CHECK(frequencies);
+    for (int i = 0; i < 256; ++i)
+        frequencies[i] = 0;
+    unsigned int* seq_indices = PyMem_Calloc(8192, sizeof(unsigned int));
+    MALLOC_CHECK(seq_indices);
+    for (int i = 0; i < 8192; ++i)
+        seq_indices[i] = 0;
+    unsigned char* decompressed_buffer = PyMem_Malloc(decomplength);
+    MALLOC_CHECK(decompressed_buffer);
+
+    while (index < input_size)
+    {
+        unsigned short next_code = input_buffer[index + 1];  //next pair of bytes to decode from the input buffer
+        next_code = next_code << 8;
+        next_code = next_code | input_buffer[index];
+        next_code = next_code >> bit_shift; //unfold 9 bit token
+
+        //The result can be interpreted as follows:
+        // iiiiiiif|ooooolll //f=0
+        // iiiiiiif|bbbbbbbb //f=1
+        //i - ignore
+        //f - flag  (is literal or offset/length pair)
+        //l - length (add 1 to get the real length)
+        //o - occurrences/frequency
+        //b - byte literal
+
+        ++bit_shift;
+        ++index;
+
+        if (bit_shift == 8)
+        {
+            bit_shift = 0;
+            ++index;
+        }
+
+        unsigned int seq_index = dest_index; //start of a byte sequence
+        if ((next_code & 0x100) != 0) // bit flag: is nextToken a literal or a reference?
+        {
+            decompressed_buffer[dest_index] = (unsigned char)(next_code & 0xFF); //store the literal
+            ++dest_index;
+        }
+        else
+        {
+            unsigned int key = ((next_code >> 3) & 0x1F) + last_dec_byte * 32; //0x1F + 0xFF*32 = 8191
+            unsigned int src_index = seq_indices[key]; //get a reference to a previously decoded sequence
+
+            for (unsigned char length = 0; length < (next_code & 0x07) + 1; ++length, ++dest_index, ++src_index)
+            {
+                decompressed_buffer[dest_index] = decompressed_buffer[src_index]; //copy a previously decoded byte sequence (up to 8)
+            }
+        }
+
+        if (dest_index >= decomplength)
+            break;
+
+        unsigned int key = frequencies[last_dec_byte] + last_dec_byte * 32; //0x1F + 0xFF*32 = 8191
+        seq_indices[key] = seq_index;
+        frequencies[last_dec_byte] = (unsigned char)((frequencies[last_dec_byte] + 1) & 0x1F); //increase by 1 (up to 31)
+        last_dec_byte = decompressed_buffer[dest_index - 1];
+    }
+
+    PyObject *output = PyBytes_FromStringAndSize(decompressed_buffer, decomplength);
+    PyMem_Free(frequencies);
+    PyMem_Free(seq_indices);
+    PyMem_Free(decompressed_buffer);
+    return output;
+}
+
+static PyMethodDef Cmp_racjinMethods[] = {
+    {"compressRACJIN", (PyCFunction)compressRACJIN, METH_VARARGS | METH_KEYWORDS, "Compress RACJIN data."},
+    {"decompressRACJIN", (PyCFunction)decompressRACJIN, METH_VARARGS | METH_KEYWORDS, "Decompress RACJIN data."},
+    {NULL, NULL, 0, NULL}
+};
+
+static struct PyModuleDef cmp_racjinmodule = {
+    PyModuleDef_HEAD_INIT,
+    "cmp_racjin",
+    "RACJIN functions.",
+    -1,
+    Cmp_racjinMethods
+};
+
+PyMODINIT_FUNC PyInit_cmp_racjin(void)
+{
+    return PyModule_Create(&cmp_racjinmodule);
+}

{hacktools-0.33.0 → hacktools-0.34.0}/hacktools/common.py

@@ -193,12 +193,14 @@ class Stream(object):
         self.seek(current)
         return ret
 
-    def readHalf(self):
+    def readHalf(self, big=False):
         if self.half is None:
             self.half = self.readByte()
+            if big:
+                return self.half >> 4
             return self.half & 0x0f
         else:
-            ret = self.half >> 4
+            ret = self.half >> 4 if not big else self.half & 0x0f
             self.half = None
             return ret
 

{hacktools-0.33.0 → hacktools-0.34.0}/hacktools/psp.py

@@ -7,7 +7,7 @@ import struct
 from hacktools import common
 
 
-def extractIso(isofile, extractfolder, workfolder=""):
+def extractIso(isofile, extractfolder, workfolder="", fixfilename=False):
     try:
         import pycdlib
     except ImportError:
@@ -20,7 +20,10 @@ def extractIso(isofile, extractfolder, workfolder=""):
     for dirname, dirlist, filelist in iso.walk(iso_path="/"):
         common.makeFolders(extractfolder + dirname[1:])
         for file in filelist:
-            with open(extractfolder + dirname[1:] + "/" + file, "wb") as f:
+            extractname = file
+            if fixfilename:
+                extractname = extractname.replace(";1", "")
+            with open(extractfolder + dirname[1:] + "/" + extractname, "wb") as f:
                 iso.get_file_from_iso_fp(f, iso_path=dirname + "/" + file)
     iso.close()
     if workfolder != "":

{hacktools-0.33.0 → hacktools-0.34.0/hacktools.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hacktools
-Version: 0.33.0
+Version: 0.34.0
 Summary: A set of utilities and tools for rom hacking and translations.
 Home-page: https://github.com/Illidanz/hacktools
 Author: Illidan
@@ -96,7 +96,7 @@ External dependencies not included are marked as `(through *dependency*)`
 ### Other / Generic
 - CPK archives
 - ARCH archives
-- LZ10, LZ11, Huffman and CRILAYLA compression/decompression
+- LZ10, LZ11, Huffman, CRILAYLA and RACJIN compression/decompression
 - ARM/MIPS binary patching
 - xdelta patch creation
 - IPS patch creation

{hacktools-0.33.0 → hacktools-0.34.0}/hacktools.egg-info/SOURCES.txt

@@ -25,5 +25,6 @@ hacktools/assets/icon.png
 hacktools/c_ext/cmp_cri.c
 hacktools/c_ext/cmp_lzss.c
 hacktools/c_ext/cmp_misc.c
+hacktools/c_ext/cmp_racjin.c
 hacktools/c_ext/inc.h
 tests/test_cmp.py

{hacktools-0.33.0 → hacktools-0.34.0}/setup.py

@@ -41,6 +41,7 @@ setup(
         Extension("hacktools.cmp_lzss", sources=["hacktools/c_ext/cmp_lzss.c"]),
         Extension("hacktools.cmp_cri",  sources=["hacktools/c_ext/cmp_cri.c"]),
         Extension("hacktools.cmp_misc", sources=["hacktools/c_ext/cmp_misc.c"]),
+        Extension("hacktools.cmp_racjin",  sources=["hacktools/c_ext/cmp_racjin.c"]),
     ],
     classifiers=[
         "Programming Language :: Python :: 3.7",

{hacktools-0.33.0 → hacktools-0.34.0}/tests/test_cmp.py

@@ -1,6 +1,6 @@
 import pytest
 import os.path
-from hacktools import cmp_lzss, cmp_cri
+from hacktools import cmp_lzss, cmp_cri, cmp_racjin
 
 @pytest.fixture
 def data():
@@ -28,3 +28,10 @@ def test_cmp_cri(data):
     decmp = cmp_cri.decompressCRILAYLA(cmp)
     assert len(data) == len(decmp)
     assert data == decmp
+
+
+def test_cmp_racjin(data):
+    cmp = cmp_racjin.compressRACJIN(data)
+    decmp = cmp_racjin.decompressRACJIN(cmp, len(data))
+    assert len(data) == len(decmp)
+    assert data == decmp

hacktools-0.33.0/hacktools/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.33.0"