PyCriCodecsEx 0.0.2__tar.gz → 0.0.5__tar.gz

{pycricodecsex-0.0.2 → pycricodecsex-0.0.5}/CriCodecsEx/adx.cpp

@@ -542,7 +542,7 @@ static PyObject* AdxEncode(PyObject* self, PyObject* args){
         PyAdxSetError(AdxErrorCode);
         return NULL;
     }
-    unsigned int len = adx.size;
+    Py_ssize_t len = adx.size;
     return Py_BuildValue("y#", adxdata, len);
 }
 
@@ -557,6 +557,6 @@ static PyObject* AdxDecode(PyObject* self, PyObject* args){
         return NULL;
     }
     unsigned char *out = wav.WAVEBuffer;
-    unsigned int len = wav.wav.size+8;
+    Py_ssize_t len = wav.wav.size+8;
     return Py_BuildValue("y#", out, len);
 }

{pycricodecsex-0.0.2 → pycricodecsex-0.0.5}/CriCodecsEx/crilayla.cpp

@@ -1,13 +1,19 @@
-/* 
-    CRI layla decompression.
-	written by tpu. (https://forum.xentax.com/viewtopic.php?f=21&t=5137&p=44220&hilit=CRILAYLA#p44220)
-	Python wrapper by me (and modification).
+/* CRILAYLA Encoder/Decoder
 
-	CRIcompress method by KenTse
-    Taken from wmltogether's fork of CriPakTools.
-    Python wrapper by me.
 
-	Note: I have no idea how this compression technique works. I just made the wrapper.
+CRI layla decompression.
+written by tpu. (https://forum.xentax.com/viewtopic.php?f=21&t=5137&p=44220&hilit=CRILAYLA#p44220)
+Python wrapper by https://github.com/Youjose/PyCriCodecs (and modification).
+
+CRIcompress method by KenTse
+Taken from wmltogether's fork of CriPakTools.
+Python wrapper by https://github.com/Youjose/PyCriCodecs.    
+TODO: This implementation may produce larger output - which shouldn't be
+possible with LZ-based compression. Investigate and fix.
+For now, if compression fails, the original data is returned.
+See also:
+    - https://github.com/FanTranslatorsInternational/Kuriimu2/blob/imgui/src/lib/Kompression/Encoder/CrilaylaEncoder.cs
+    - https://glinscott.github.io/lz/index.html
 */
 #define PY_SSIZE_T_CLEAN
 #pragma once
@@ -16,38 +22,43 @@
 #include <stdlib.h>
 #include <string.h>
 
+unsigned fourCC(const char a, const char b, const char c, const char d) {
+	return (a << 0) | (b << 8) | (c << 16) | (d << 24);
+};
+const unsigned CRILAYLA_LO = fourCC('C', 'R', 'I', 'L');
+const unsigned CRILAYLA_HI = fourCC('A', 'Y', 'L', 'A');
+const unsigned long long CRILAYLA_MAGIC =  CRILAYLA_LO | (unsigned long long)CRILAYLA_HI << 32uLL;
+
 struct crilayla_header{
     unsigned long long crilayla;
     unsigned int decompress_size;
     unsigned int compressed_size;
 };
 
-
-unsigned char *sbuf;
-unsigned int bitcnt;
-unsigned int bitdat;
-unsigned int get_bits(unsigned int n){
-	unsigned int data, mask;
-
-    if (bitcnt<n){
-	  data = ((24-bitcnt)>>3)+1;
-	  bitcnt += data*8;
-      while(data) {
-		bitdat = (bitdat<<8) | (*sbuf--);
-		data--;
-      }
-    }
-
-	data = bitdat>>(bitcnt-n);
-	bitcnt -= n;
-	mask = (1<<n)-1;
-	data &= mask;
-	return data;
-}
-
 unsigned int llcp_dec(unsigned char *src, unsigned int src_len, unsigned char *dst, unsigned int dst_len){
-	unsigned char *dbuf, *pbuf;
+    unsigned char *dbuf, *pbuf;
 	unsigned int plen, poffset, byte;
+    unsigned char *sbuf;
+    unsigned int bitcnt;
+    unsigned int bitdat;
+    auto get_bits = [&](unsigned int n){
+        unsigned int data, mask;
+    
+        if (bitcnt<n){
+          data = ((24-bitcnt)>>3)+1;
+          bitcnt += data*8;
+          while(data) {
+            bitdat = (bitdat<<8) | (*sbuf--);
+            data--;
+          }
+        }
+    
+        data = bitdat>>(bitcnt-n);
+        bitcnt -= n;
+        mask = (1<<n)-1;
+        data &= mask;
+        return data;
+    };
 
 	sbuf = src+src_len-1;
 	dbuf = dst+dst_len-1;
@@ -108,40 +119,41 @@ unsigned char* layla_decomp(unsigned char* data, crilayla_header header){
 	return dst;
 }
 
-unsigned int layla_comp(unsigned char* dest, unsigned int* destLen, unsigned char* src, unsigned int srcLen){
-    unsigned int n = srcLen - 1, m = *destLen - 0x1, T = 0, d = 0, p, q, i, j, k;
-    unsigned char* odest = dest;
+unsigned int layla_comp(unsigned char *dest, int *destLen, unsigned char *src, int srcLen)
+{
+    int n = srcLen - 1, m = *destLen - 0x1, T = 0, d = 0, p, q, i, j, k;
+    unsigned char *odest = dest;
     for (; n >= 0x100;)
     {
         j = n + 3 + 0x2000;
-        if (j > srcLen) j = srcLen;
-        for (i = n + 3, p = 0; i < j; i++)
+        if (j>srcLen) j = srcLen;
+        for (i = n + 3, p = 0; i<j; i++)
         {
             for (k = 0; k <= n - 0x100; k++)
             {
                 if (*(src + n - k) != *(src + i - k)) break;
             }
-            if (k > p)
+            if (k>p)
             {
                 q = i - n - 3; p = k;
             }
         }
-        if (p < 3)
+        if (p<3)
         {
             d = (d << 9) | (*(src + n--)); T += 9;
         }
         else
         {
             d = (((d << 1) | 1) << 13) | q; T += 14; n -= p;
-            if (p < 6)
+            if (p<6)
             {
                 d = (d << 2) | (p - 3); T += 2;
             }
-            else if (p < 13)
+            else if (p<13)
             {
                 d = (((d << 2) | 3) << 3) | (p - 6); T += 5;
             }
-            else if (p < 44)
+            else if (p<44)
             {
                 d = (((d << 5) | 0x1f) << 5) | (p - 13); T += 10;
             }
@@ -150,45 +162,50 @@ unsigned int layla_comp(unsigned char* dest, unsigned int* destLen, unsigned cha
                 d = ((d << 10) | 0x3ff); T += 10; p -= 44;
                 for (;;)
                 {
-                    for (; T >= 8;)
+                    for (; m > 0 && T >= 8;)
                     {
-                        *(dest + m--) = (d >> (T - 8)) & 0xff; T -= 8; d = d & ((1 << T) - 1);
+                        *(dest + m--) = (d >> (T - 8)) & 0xff; T -= 8; d = d&((1 << T) - 1);
                     }
-                    if (p < 255) break;
+                    if (p<255) break;
                     d = (d << 8) | 0xff; T += 8; p = p - 0xff;
                 }
                 d = (d << 8) | p; T += 8;
             }
         }
-        for (; T >= 8;)
+        for (; m > 0 && T >= 8;)
         {
-            *(dest + m--) = (d >> (T - 8)) & 0xff; T -= 8; d = d & ((1 << T) - 1);
+            *(dest + m--) = (d >> (T - 8)) & 0xff; T -= 8; d = d&((1 << T) - 1);
         }
     }
-    if (T != 0)
+    if (m > 0 && T != 0)
     {
         *(dest + m--) = d << (8 - T);
     }
-    *(dest + m--) = 0; *(dest + m) = 0;
-    for (;;)
-    {
-        if (((*destLen - m) & 3) == 0) break;
-        *(dest + m--) = 0;
+    if (m > 0) {
+        *(dest + m--) = 0; *(dest + m) = 0;
+        for (;;)
+        {
+            if (((*destLen - m) & 3) == 0) break;
+            *(dest + m--) = 0;
+        }
     }
+    if (m <= 0) 
+        return 0; // Underflow    
     *destLen = *destLen - m; dest += m;
-    unsigned int l[] = { 0x4c495243,0x414c5941,srcLen - 0x100,*destLen };
-    for (j = 0; j < 4; j++)
+    // CRIL AYLA srcLen-0x100 destLen
+    int l[] = { (int)CRILAYLA_LO,(int)CRILAYLA_HI,srcLen - 0x100,*destLen };
+    for (j = 0; j<4; j++)
     {
-        for (i = 0; i < 4; i++)
+        for (i = 0; i<4; i++)
         {
             *(odest + i + j * 4) = l[j] & 0xff; l[j] >>= 8;
         }
     }
-    for (j = 0, odest += 0x10; j < *destLen; j++)
+    for (j = 0, odest += 0x10; j<*destLen; j++)
     {
         *(odest++) = *(dest + j);
     }
-    for (j = 0; j < 0x100; j++)
+    for (j = 0; j<0x100; j++)
     {
         *(odest++) = *(src + j);
     }
@@ -199,7 +216,17 @@ unsigned int layla_comp(unsigned char* dest, unsigned int* destLen, unsigned cha
 PyObject* CriLaylaDecompress(PyObject* self, PyObject* d){
 	unsigned char *data = (unsigned char *)PyBytes_AsString(d);
 	crilayla_header header = *(crilayla_header*)data;
-	unsigned char *out = layla_decomp((data+16), header);
+    
+    if (header.crilayla != CRILAYLA_MAGIC) {
+        PyErr_SetString(PyExc_ValueError, "Invalid CRILAYLA header.");
+        return NULL;
+    }
+    
+    unsigned char *out;
+    Py_BEGIN_ALLOW_THREADS
+	out = layla_decomp((data+16), header);
+    Py_END_ALLOW_THREADS
+    
     PyObject *outObj = Py_BuildValue("y#", out, header.decompress_size+256);
     delete[] out;
 	return outObj;
@@ -213,14 +240,26 @@ unsigned char* CriLaylaDecompress(unsigned char* d){
 
 PyObject* CriLaylaCompress(PyObject* self, PyObject* args){
 	unsigned char *data;
-	unsigned int data_size;
+	Py_ssize_t data_size;
     if(!PyArg_ParseTuple(args, "y#", &data, &data_size)){
         return NULL;
     }
-    unsigned char *buf = new unsigned char[data_size];
-    memset(buf, 0, data_size);
-    layla_comp(buf, &data_size, data, data_size);
-	PyObject* bufObj = Py_BuildValue("y#", buf, data_size);
+
+    unsigned char *buf = new unsigned char[data_size + 0x200];
+
+    int compressed_size = data_size;    
+    int res = 0;
+    Py_BEGIN_ALLOW_THREADS
+    res = layla_comp(buf, &compressed_size, data, compressed_size);
+    Py_END_ALLOW_THREADS
+    PyObject* bufObj;
+    if (res && res < data_size) {
+        bufObj = Py_BuildValue("y#", buf, compressed_size);
+    } else {        
+        // FIXME
+        PyErr_SetString(PyExc_RuntimeError, "Compression failure.");
+        return NULL;
+    }
     delete[] buf;
     return bufObj;
 }

{pycricodecsex-0.0.2 → pycricodecsex-0.0.5}/CriCodecsEx/hca.cpp

@@ -3296,7 +3296,7 @@ static PyObject* HcaCrypt(PyObject* self, PyObject* args){
     }
 
     unsigned char *buffer = (unsigned char *)view.buf;
-    unsigned int length = view.len;
+    Py_ssize_t length = view.len;
     PyBuffer_Release(&view);
 
     clHCA* hca = (clHCA*)malloc(sizeof(clHCA));
@@ -3345,7 +3345,7 @@ static PyObject* HcaDecode(PyObject* self, PyObject* args){
     data *wavData = &wav.wav.chunks.WAVEdata;
 
     unsigned char *data;
-    unsigned int data_size;
+    Py_ssize_t data_size;
     unsigned int header_size;
     unsigned long long keycode;
     unsigned short subkey;

pycricodecsex-0.0.5/PKG-INFO

@@ -0,0 +1,35 @@
+Metadata-Version: 2.4
+Name: PyCriCodecsEx
+Version: 0.0.5
+Summary: Criware formats library for Python
+Home-page: https://mos9527.github.io/PyCriCodecsEx/
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Provides-Extra: usm
+Requires-Dist: ffmpeg-python; extra == "usm"
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license-file
+Dynamic: provides-extra
+Dynamic: requires-python
+Dynamic: summary
+
+PyCriCodecsEx
+---
+A continuation of @Youjose's work on Criware formats. Feautres are still in flux and subject to change. When in doubt, Refer to the [original repo](https://github.com/Youjose/PyCriCodecs) for more information.
+
+Detailed documentation, installation instructions are available at https://mos9527.com/PyCriCodecsEx
+
+
+# Credits
+- https://github.com/Youjose/PyCriCodecs
+- https://github.com/Mikewando/PyCriCodecs ([PR#1 on USM](https://github.com/mos9527/PyCriCodecsEx/pull/1))
+- https://github.com/donmai-me/WannaCRI
+- https://github.com/vgmstream/vgmstream
+- https://github.com/K0lb3/UnityPy (For CI script)

pycricodecsex-0.0.5/PyCriCodecsEx/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.0.5"

{pycricodecsex-0.0.2 → pycricodecsex-0.0.5}/PyCriCodecsEx/acb.py

@@ -3,22 +3,27 @@
 # - Original work by https://github.com/Youjose/PyCriCodecs
 # See Research/ACBSchema.py for more details.
 
-from typing import Generator, List, Tuple
+from typing import Generator, List, Tuple, BinaryIO
 from PyCriCodecsEx.chunk import *
 from PyCriCodecsEx.utf import UTF, UTFBuilder, UTFViewer
-from PyCriCodecsEx.usm import HCACodec, ADXCodec
+from PyCriCodecsEx.hca import HCACodec
+from PyCriCodecsEx.adx import ADXCodec
 from PyCriCodecsEx.awb import AWB, AWBBuilder
 from dataclasses import dataclass
 from copy import deepcopy
 
 class CueNameTable(UTFViewer):
     CueIndex: int
+    '''Index into CueTable'''
     CueName: str
+    '''Name of the cue'''
 
 
 class CueTable(UTFViewer):
     CueId: int
+    '''Corresponds to the cue index found in CueNameTable'''
     Length: int
+    '''Duration of the cue in milliseconds'''
     ReferenceIndex: int
     ReferenceType: int
 
@@ -51,14 +56,21 @@ class WaveformTable(UTFViewer):
 
 
 class ACBTable(UTFViewer):
+    '''ACB Table View'''
+
     AcbGuid: bytes
+    '''GUID of the ACB. This SHOULD be different for each ACB file.'''
     Name: str
-    Version: int
+    '''Name of the ACB. This is usually the name of the sound bank.'''
+    Version: int    
     VersionString: str
 
     AwbFile: bytes
     CueNameTable: List[CueNameTable]
+    '''A list of cue names with their corresponding indices into CueTable'''
     CueTable: List[CueTable]
+    '''A list of cues with their corresponding references'''
+
     SequenceTable: List[SequenceTable]
     SynthTable: List[SynthTable]
     TrackEventTable: List[TrackEventTable]
@@ -66,7 +78,7 @@ class ACBTable(UTFViewer):
     WaveformTable: List[WaveformTable]
 
     @staticmethod
-    def decode_tlv(data : bytes):
+    def _decode_tlv(data : bytes):
         pos = 0
         while pos < len(data):
             tag = data[pos : pos + 2]
@@ -75,16 +87,16 @@ class ACBTable(UTFViewer):
             pos += 3 + length
             yield (tag, value)
 
-    def waveform_of_track(self, index: int):
-        tlv = self.decode_tlv(self.TrackEventTable[index])
+    def _waveform_of_track(self, index: int):
+        tlv = self._decode_tlv(self.TrackEventTable[index])
         def noteOn(data: bytes):
             # Handle note on event
             tlv_type, tlv_index = AcbTrackCommandNoteOnStruct.unpack(data[:AcbTrackCommandNoteOnStruct.size])
             match tlv_type:
                 case 0x02: # Synth
-                    yield from self.waveform_of_synth(tlv_index)
+                    yield from self._waveform_of_synth(tlv_index)
                 case 0x03: # Sequence
-                    yield from self.waveform_of_sequence(tlv_index)
+                    yield from self._waveform_of_sequence(tlv_index)
                 # Ignore others silently                
         for code, data in tlv:
             match code:
@@ -93,13 +105,13 @@ class ACBTable(UTFViewer):
                 case 2003:
                     yield from noteOn(data)            
 
-    def waveform_of_sequence(self, index : int):
+    def _waveform_of_sequence(self, index : int):
         seq = self.SequenceTable[index]
         for i in range(seq.NumTracks):
             track_index = int.from_bytes(seq.TrackIndex[i*2:i*2+2], 'big')
             yield self.WaveformTable[track_index]
 
-    def waveform_of_synth(self, index: int):        
+    def _waveform_of_synth(self, index: int):        
         item_type, item_index = AcbSynthReferenceStruct.unpack(self.SynthTable[index].ReferenceItems)
         match item_type:
             case 0x00: # No audio
@@ -107,38 +119,52 @@ class ACBTable(UTFViewer):
             case 0x01: # Waveform
                 yield self.WaveformTable[item_index]
             case 0x02: # Yet another synth...
-                yield from self.waveform_of_synth(item_index)
+                yield from self._waveform_of_synth(item_index)
             case 0x03: # Sequence
-                yield from self.waveform_of_sequence(item_index)
+                yield from self._waveform_of_sequence(item_index)
             case _:
                 raise NotImplementedError(f"Unknown synth reference type: {item_type} at index {index}")
 
     def waveform_of(self, index : int) -> List["WaveformTable"]:
+        """Retrieves the waveform(s) associated with a cue.
+
+        Cues may reference multiple waveforms, which could also be reused."""
         cue = next(filter(lambda c: c.CueId == index, self.CueTable), None)
         assert cue, "cue of index %d not found" % index
         match cue.ReferenceType:
             case 0x01:
                 return [self.WaveformTable[index]]
             case 0x02:
-                return list(self.waveform_of_synth(index))
+                return list(self._waveform_of_synth(index))
             case 0x03:
-                return list(self.waveform_of_sequence(index))
+                return list(self._waveform_of_sequence(index))
             case 0x08:
                 raise NotImplementedError("BlockSequence type not implemented yet")
             case _:
                 raise NotImplementedError(f"Unknown cue reference type: {cue.ReferenceType}")
 
 @dataclass(frozen=True)
-class CueItem:
+class PackedCueItem:
+    '''Helper class for read-only cue information'''
+
     CueId: int
+    '''Cue ID'''
     CueName: str
+    '''Cue name'''
     Length: float
-    Waveforms: list[int] # List of waveform IDs
+    '''Duration in seconds'''
+    Waveforms: list[int]
+    '''List of waveform IDs, corresponds to ACB.get_waveforms()'''
 
 class ACB(UTF):
-    """An ACB is basically a giant @UTF table. Use this class to extract any ACB, and potentially modifiy it in place."""
-    def __init__(self, filename) -> None:
-        super().__init__(filename,recursive=True)
+    """Use this class to read, and modify ACB files in memory."""
+    def __init__(self, stream : str | BinaryIO) -> None:
+        """Loads an ACB file from the given stream.
+
+        Args:
+            stream (str | BinaryIO): The path to the ACB file or a BinaryIO stream containing the ACB data.
+        """
+        super().__init__(stream, recursive=True)
 
     @property
     def payload(self) -> dict:
@@ -162,10 +188,17 @@ class ACB(UTF):
         """Returns the AWB object associated with the ACB."""
         return AWB(self.view.AwbFile)
 
-    def get_waveforms(self) -> List[HCACodec | ADXCodec | Tuple[AcbEncodeTypes, int, int, int,  bytes]]:
+    def get_waveforms(self, **kwargs) -> List[HCACodec | ADXCodec | Tuple[AcbEncodeTypes, int, int, int,  bytes]]:
         """Returns a list of decoded waveforms.
 
         Item may be a codec (if known), or a tuple of (Codec ID, Channel Count, Sample Count, Sample Rate, Raw data).
+
+        Additional keyword arguments are passed to the codec constructors. e.g. for encrypted HCA payloads,
+        you may do the following:
+        ```python
+        get_waveforms(key=..., subkey=...)
+        ```
+        See also the respective docs (ADXCodec, HCACodec) for more details.
         """
         CODEC_TABLE = {
             AcbEncodeTypes.ADX: ADXCodec,
@@ -178,7 +211,7 @@ class ACB(UTF):
             encode = AcbEncodeTypes(wav.EncodeType)
             codec = (CODEC_TABLE.get(encode, None))
             if codec:
-                wavs.append(codec(awb.get_file_at(wav.MemoryAwbId)))
+                wavs.append(codec(awb.get_file_at(wav.MemoryAwbId), **kwargs))
             else:
                 wavs.append((encode, wav.NumChannels, wav.NumSamples, wav.SamplingRate, awb.get_file_at(wav.MemoryAwbId)))
         return wavs
@@ -188,7 +221,7 @@ class ACB(UTF):
 
         Input item may be a codec (if known), or a tuple of (Codec ID, Channel Count, Sample Count, Sample Rate, Raw data).
 
-        NOTE: Cue duration is not set. You need to change that manually.
+        NOTE: Cue duration is not set. You need to change that manually - this is usually unecessary as the player will just play until the end of the waveform.
         """
         WAVEFORM = self.view.WaveformTable[0]._payload.copy()
         encoded = []
@@ -228,7 +261,7 @@ class ACB(UTF):
         pass
 
     @property
-    def cues(self) -> Generator[CueItem, None, None]:
+    def cues(self) -> Generator[PackedCueItem, None, None]:
         """Returns a generator of **read-only** Cues.
 
         Cues reference waveform bytes by their AWB IDs, which can be accessed via `waveforms`.
@@ -236,20 +269,25 @@ class ACB(UTF):
         """
         for name, cue in zip(self.view.CueNameTable, self.view.CueTable):
             waveforms = self.view.waveform_of(cue.CueId)
-            yield CueItem(cue.CueId, name.CueName, cue.Length / 1000.0, [waveform.MemoryAwbId for waveform in waveforms])
+            yield PackedCueItem(cue.CueId, name.CueName, cue.Length / 1000.0, [waveform.MemoryAwbId for waveform in waveforms])
 
 class ACBBuilder:
+    """Use this class to build ACB files from an existing ACB object."""
     acb: ACB
 
     def __init__(self, acb: ACB) -> None:
         """Initializes the ACBBuilder with an existing ACB object.
         
+        Args:
+            acb (ACB): The ACB object to build from.
+
         Building ACB from scratch isn't planned for now since:
-        * We don't know how SeqCommandTable TLVs work. This is the biggest issue.
-        * Many fields are unknown or not well understood
-          - Games may expect AcfReferenceTable, Asiac stuff etc to be present for their own assets in conjunction
-            with their own ACF table. Missing these is not a fun debugging experience.
-        * ACB tables differ a LOT from game to game (e.g. Lipsync info), contary to USM formats.
+                
+            * We don't know how SeqCommandTable TLVs work. This is the biggest issue.
+            * Many fields are unknown or not well understood
+            - Games may expect AcfReferenceTable, Asiac stuff etc to be present for their own assets in conjunction
+                with their own ACF table. Missing these is not a fun debugging experience.
+            * ACB tables differ a LOT from game to game (e.g. Lipsync info), contary to USM formats.
 
         Maybe one day I'll get around to this. But otherwise starting from nothing is a WONTFIX for now.
         """