turbine-lib 0.1.0__py3-none-any.whl

turbine_lib/datfile.py

@@ -0,0 +1,338 @@
+import logging
+
+from binarydata import BinaryData
+from database import Database
+from datexportapi import DatExportApi
+from ddssubfile import DDSSubfile
+from fontsubfile import SubfileFONT
+from subfile import FILE_TYPE, file_type_from_file_contents, string_from_file_type
+from subfiledata import SubfileData
+from textsubfile import TextSubfile
+
+
+class SubfileInfo:
+
+    def __init__(self):
+        self.file_id = -1
+        self.size = -1
+        self.iteration = -1
+
+    def __lt__(self, other):
+        return self.file_id < other.file_id
+
+
+class DatFile:
+    # Class variable - shared by all instances
+    api_ = DatExportApi()
+
+    def __init__(self, file_handle):
+        self.file_handle_ = file_handle
+        self.initialized_ = False
+        self.files_info_ = {}  # Dictionary mapping file_id to SubfileInfo
+        self.filename_ = ""
+        self.export_data_buf_ = BinaryData(bytearray(64 * 1024 * 1024))  # 64 MB - max file size
+
+    def __del__(self):
+        self.deinit()
+
+    def init(self, filename):
+        if self.initialized_:
+            self.deinit()
+
+        if self.api_.open_dat_file(self.file_handle_, filename, 130) == self.file_handle_:
+            self.initialized_ = True
+            self.filename_ = filename
+            self.load_all_files_info()
+            return True
+
+        return False
+
+    def load_all_files_info(self):
+        subfiles_num = self.api_.get_num_subfiles(self.file_handle_)
+        file_ids = [0] * subfiles_num
+        sizes = [0] * subfiles_num
+        iterations = [0] * subfiles_num
+
+        # Initialize lists with dummy values
+        for i in range(subfiles_num):
+            file_ids[i] = 0
+            sizes[i] = 0
+            iterations[i] = 0
+
+        if subfiles_num > 0:
+            self.api_.get_subfile_sizes(self.file_handle_, file_ids, sizes, iterations, 0, subfiles_num)
+
+        for i in range(subfiles_num):
+            file_info = SubfileInfo()
+            file_info.file_id = file_ids[i]
+            file_info.size = sizes[i]
+            file_info.iteration = iterations[i]
+            self.files_info_[file_info.file_id] = file_info
+
+    def deinit(self):
+        if self.initialized_:
+            self.api_.close_dat_file(self.file_handle_)
+            self.files_info_.clear()
+            self.initialized_ = False
+
+    def initialized(self):
+        return self.initialized_
+
+    def get_filename(self):
+        return self.filename_
+
+    def get_subfile_info(self, file_id):
+        if file_id in self.files_info_:
+            return self.files_info_[file_id]
+        else:
+            return SubfileInfo()
+
+    def get_files_num_in_dat_file(self):
+        return self.api_.get_num_subfiles(self.file_handle_)
+
+    def patch_all_files_from_database(self, db):
+        patched_files_num = 0
+
+        file = db.get_next_file()
+        i = 0
+        total_files = db.count_rows()
+        logging.info("Patching all files from database...")
+
+        while not file.empty():
+            if i > 0 and total_files > 0 and i * 100 // total_files != (i - 1) * 100 // total_files:
+                logging.info(f"Completed {i * 100 // total_files}%")
+                print(f"Completed {i * 100 // total_files}%")
+                # gl.progress_wnd.UpdateStageText(f"正在安装第{i}条翻译到dat文件...")
+                # gl.progress_wnd.UpdatePercent(int(i * 100 / total_files))
+            i += 1
+
+            if not file.options or "fid" not in file.options:
+                logging.info(f"Incorrect db entry - no file_id specified")
+                file = db.get_next_file()
+                continue
+
+            self.patch_file(file)
+            patched_files_num += 1
+            file = db.get_next_file()
+
+        return patched_files_num
+
+    def patch_file(self, file_data=None, file_id=None, file_type=None, path_to_file=None, version=-1, iteration=-1):
+        if file_data is not None:
+            # Patch with SubfileData
+            if not file_data.options or "fid" not in file_data.options:
+                print("Trying to patch file, but file id is not specified, skipping!")
+                return
+
+            file_id_inner = int(file_data.options["fid"])
+
+            if file_id_inner not in self.files_info_:
+                print(f"Trying to patch file, not existing in files_info. File id = {file_id_inner}")
+                return
+
+            file_info = self.files_info_[file_id_inner]
+
+            existing_file_version = 0  # will be evaluated with api_.GetSubfileData
+            size, existing_file_version = self.api_.get_subfile_data(
+                self.file_handle_, file_id_inner, self.export_data_buf_.data(), existing_file_version)
+
+            if size <= 0:
+                print(f"Trying to patch file, not existing in .dat file. File id = {file_id_inner}")
+                return
+
+            old_data = self.export_data_buf_.cut_data(0, size)
+
+            # Check file size is reasonable
+            if size > 64 * 1024 * 1024:  # Exceeds buffer size
+                print(f"File size too large for buffer. File id = {file_id_inner}, Size = {size}")
+                return
+
+            # Build file for import
+            try:
+                file_binary = self.build_for_import(old_data, file_data)
+            except Exception as e:
+                print(f"Exception in build_for_import: {e} file_id {file_id_inner}")
+                raise e
+                return
+
+            if version == -1:
+                version = existing_file_version
+
+            if iteration == -1:
+                iteration = file_info.iteration
+
+            # Convert BinaryData to bytearray for ctypes
+            file_bytes = file_binary.data()
+            self.api_.put_subfile_data(
+                self.file_handle_, file_id_inner, file_bytes, 0, len(file_bytes), version, iteration, False)
+
+        elif file_id is not None and file_type is not None and path_to_file is not None:
+            # Patch with file path
+            new_data = BinaryData(bytearray(64 * 1024 * 1024))
+
+            try:
+                with open(path_to_file, 'rb') as f:
+                    file_content = f.read()
+                    data_size = len(file_content)
+                    new_data.data_[:data_size] = file_content
+            except Exception as e:
+                print(f"Error reading file {path_to_file}: {e}")
+                return
+
+            imported_subfile = SubfileData()
+            imported_subfile.binary_data = new_data.cut_data(0, data_size)
+            imported_subfile.options = {"ext": string_from_file_type(file_type), "fid": file_id}
+
+            self.patch_file(imported_subfile, version=version, iteration=iteration)
+
+    def get_existing_file_type(self, file_id):
+        version = 0
+        size, version = self.api_.get_subfile_data(
+            self.file_handle_, file_id, self.export_data_buf_.data(), version)
+        return file_type_from_file_contents(file_id, self.export_data_buf_)
+
+    def perform_operation_on_all_subfiles(self, operation):
+        if not self.files_info_:
+            self.load_all_files_info()
+
+        print("Performing operation on all files...")
+        i = 0
+        for file_id, info in self.files_info_.items():
+            if i > 0 and len(self.files_info_) > 0 and i * 100 // len(self.files_info_) != (i - 1) * 100 // len(
+                    self.files_info_):
+                print(f"Completed {i * 100 // len(self.files_info_)}%")
+            operation(info)
+            i += 1
+
+    def export_files_by_type(self, file_type, db_or_path):
+        num_files = 0
+
+        if isinstance(db_or_path, Database):
+            # Export to database
+            def operation(info):
+                nonlocal num_files
+                file_type_existing = self.get_existing_file_type(info.file_id)
+                if file_type_existing == file_type:
+                    self.export_file_by_id(info.file_id, db_or_path)
+                    num_files += 1
+
+            self.perform_operation_on_all_subfiles(operation)
+            return num_files
+        else:
+            # Export to directory
+            path_to_directory = db_or_path
+
+            def operation(info):
+                nonlocal num_files
+                file_type_existing = self.get_existing_file_type(info.file_id)
+                if file_type_existing == file_type:
+                    target_path = f"{path_to_directory}/{info.file_id}"
+                    self.export_file_by_id(info.file_id, target_path)
+                    num_files += 1
+
+            self.perform_operation_on_all_subfiles(operation)
+            return num_files
+
+    def export_file_by_id(self, file_id, target):
+        if file_id <= 0:
+            print("Invalid file ID!")
+            return
+        version = 0
+        size, version = self.api_.get_subfile_data(
+            self.file_handle_, file_id, self.export_data_buf_.data(), version)
+
+        data = self.export_data_buf_.cut_data(0, size)
+        file_result = self.build_for_export(file_id, data)
+
+        if isinstance(target, Database):
+            # Export to database
+            target.push_file(file_result)
+        else:
+            # Export to file path
+            ext = string_from_file_type(self.get_existing_file_type(file_id))
+            target_file_path = f'{target}{ext}'
+            # target_file_path = f'{target}.dds'
+            try:
+                with open(target_file_path, 'wb') as f:
+                    if ext == ".txt":
+                        f.write(file_result.text_data.encode("utf-8"))
+                    else:
+                        f.write(file_result.binary_data.data())
+            except Exception as e:
+                print(f"Error writing file {target_file_path}: {e}")
+
+    def get_file_version(self, file_id):
+        return self.api_.get_subfile_version(self.file_handle_, file_id)
+
+    def get_file(self, file_id):
+        version = 0
+        size, version = self.api_.get_subfile_data(
+            self.file_handle_, file_id, self.export_data_buf_.data(), version)
+        data = self.export_data_buf_.cut_data(0, size)
+        return self.build_for_export(file_id, data)
+
+    def build_for_import(self, old_data, outer_data):
+        if not outer_data.options or "ext" not in outer_data.options:
+            print(f"No extension established for file with id {outer_data.options.get('fid', 'unknown')}")
+            return BinaryData()
+
+        # In Python, we'll directly call the appropriate subfile class
+        ext = outer_data.options["ext"]
+        if ext == ".txt":
+            return TextSubfile.build_for_import(old_data, outer_data)
+        elif ext == ".dds":
+            for_import = DDSSubfile.build_for_import(old_data, outer_data)
+            return for_import
+        elif ext == ".fontbin":
+            for_import = SubfileFONT.build_for_import(old_data, outer_data)
+            return for_import
+        # Add other file types as needed
+        else:
+            # Default implementation for unknown types
+            return old_data  # Just return the original data
+
+    def compress(self, buffer):
+        # Handle compression (reverse of decompression)
+        decompressed_length = len(buffer)
+
+        # Create buffer for compressed data (with 4 bytes extra for length header)
+        compressed_buffer = bytearray(decompressed_length + 1000)  # Extra space for compression
+        compressed_length = decompressed_length + 1000  # Using list to pass by reference
+
+        # Compress the data
+        compressed_length = self.api_.compress2(compressed_buffer, compressed_length, buffer.data(),
+                                                decompressed_length, 9)
+
+        # Create final buffer with length header
+        final_buffer = bytearray(4 + compressed_length)
+
+        # Write decompressed length as header (little endian)
+        final_buffer[0] = decompressed_length & 0xFF
+        final_buffer[1] = (decompressed_length >> 8) & 0xFF
+        final_buffer[2] = (decompressed_length >> 16) & 0xFF
+        final_buffer[3] = (decompressed_length >> 24) & 0xFF
+
+        # Copy compressed data
+        for i in range(compressed_length):
+            final_buffer[4 + i] = compressed_buffer[i]
+
+        # Update buffer
+        return BinaryData(final_buffer)
+
+    def build_for_export(self, file_id, inner_data):
+        file_type = file_type_from_file_contents(file_id, inner_data)
+        result = SubfileData()
+
+        if file_type == FILE_TYPE.TEXT:
+            result = TextSubfile.build_for_export(inner_data)
+        elif file_type == FILE_TYPE.DDS:
+            result = DDSSubfile.build_for_export(inner_data)
+        elif file_type == FILE_TYPE.FONT:
+            result = SubfileFONT.build_for_export(inner_data)
+        # Add other file types as needed
+        else:
+            # Default implementation for unknown types
+            result.binary_data = inner_data
+
+        result.options["fid"] = file_id
+        return result

turbine_lib/ddssubfile.py

@@ -0,0 +1,305 @@
+from subfile import Subfile, FILE_TYPE
+from binarydata import BinaryData
+from subfiledata import SubfileData
+
+
+class DDSSubfile(Subfile):
+    """Implementation of Subfile for DDS type files"""
+
+    @staticmethod
+    def build_for_export(file_data):
+        if file_data.empty() or len(file_data) < 256:
+            return SubfileData()
+
+        dds_data = BinaryData(bytearray(len(file_data) - 24 + 128))
+        for i in range(128):
+            dds_data[i] = 0
+
+        # Copy data from offset 24 in file_data to offset 128 in dds_data
+        for i in range(len(file_data) - 24):
+            dds_data[128 + i] = file_data[24 + i]
+
+        if len(file_data) >= 20 and file_data[16] == 0x44 and file_data[17] == 0x58 and file_data[18] == 0x54:  # dxt
+
+            if 0x31 == file_data[19]:  # dxt1
+                dds_data[0] = 0x44  # D
+                dds_data[1] = 0x44  # D
+                dds_data[2] = 0x53  # S
+                dds_data[3] = 0x20  # ''
+
+                dds_data[4] = 0x7C
+
+                dds_data[8] = 0x7
+                dds_data[9] = 0x10
+
+                # width, height
+                dds_data[12] = file_data[12]
+                dds_data[13] = file_data[13]
+                dds_data[14] = file_data[14]
+                dds_data[15] = file_data[15]
+
+                dds_data[16] = file_data[8]
+                dds_data[17] = file_data[9]
+                dds_data[18] = file_data[10]
+                dds_data[19] = file_data[11]
+
+                dds_data[76] = 0x20
+                dds_data[80] = 0x4
+
+                dds_data[84] = 0x44  # 'D'
+                dds_data[85] = 0x58  # 'X'
+                dds_data[86] = 0x54  # 'T'
+                dds_data[87] = 0x31  # '1'
+            elif file_data[19] == 0x33:  ## dxt3
+                dds_data[0] = 0x44  # D
+                dds_data[1] = 0x44  # D
+                dds_data[2] = 0x53  # S
+                dds_data[3] = 0x20  # ''
+
+                dds_data[4] = 0x7C
+
+                dds_data[8] = 0x7
+                dds_data[9] = 0x10
+
+                # width, height
+                dds_data[12] = file_data[12]
+                dds_data[13] = file_data[13]
+                dds_data[14] = file_data[14]
+                dds_data[15] = file_data[15]
+
+                dds_data[16] = file_data[8]
+                dds_data[17] = file_data[9]
+                dds_data[18] = file_data[10]
+                dds_data[19] = file_data[11]
+
+                dds_data[22] = 0x1
+
+                dds_data[76] = 0x20
+                dds_data[80] = 0x4
+
+                dds_data[84] = 0x44  # 'D'
+                dds_data[85] = 0x58  # 'X'
+                dds_data[86] = 0x54  # 'T'
+                dds_data[87] = 0x33  # '3'
+
+                dds_data[108] = 0x8
+                dds_data[109] = 0x10
+                dds_data[110] = 0x40
+
+            elif file_data[19] == 0x35:  # dxt5
+                dds_data[0] = 0x44  # ‘D’
+                dds_data[1] = 0x44  # ’D‘
+                dds_data[2] = 0x53  # ‘S’
+                dds_data[3] = 0x20  # ‘’
+
+                dds_data[4] = 0x7C
+
+                dds_data[8] = 0x7
+                dds_data[9] = 0x10
+                dds_data[10] = 0x8
+
+                # width, height
+                dds_data[12] = file_data[12]
+                dds_data[13] = file_data[13]
+                dds_data[14] = file_data[14]
+                dds_data[15] = file_data[15]
+
+                dds_data[16] = file_data[8]
+                dds_data[17] = file_data[9]
+                dds_data[18] = file_data[10]
+                dds_data[19] = file_data[11]
+
+                dds_data[22] = 0x1
+                dds_data[28] = 0x1
+
+                dds_data[76] = 0x20
+                dds_data[80] = 0x4
+
+                dds_data[84] = 0x44  # 'D'
+                dds_data[85] = 0x58  # ‘X’
+                dds_data[86] = 0x54  # ‘T’
+                dds_data[87] = 0x35  # ‘5’
+
+                # dds_data[88] = 0x20
+                dds_data[88] = 0x20
+                dds_data[94] = 0xFF
+                dds_data[97] = 0xFF
+                dds_data[100] = 0xFF
+                dds_data[107] = 0xFF
+                dds_data[113] = 0x10
+        else:
+            dds_data[0] = 0x44  # D
+            dds_data[1] = 0x44  # D
+            dds_data[2] = 0x53  # S
+            dds_data[3] = 0x20  # ''
+
+            dds_data[4] = 0x7C
+
+            dds_data[8] = 0x7
+            dds_data[9] = 0x10
+
+            # width, height
+            dds_data[12] = file_data[12]
+            dds_data[13] = file_data[13]
+            dds_data[14] = file_data[14]
+            dds_data[15] = file_data[15]
+            dds_data[16] = file_data[8]
+            dds_data[17] = file_data[9]
+            dds_data[18] = file_data[10]
+            dds_data[19] = file_data[11]
+
+            dds_data[76] = 0x20
+            dds_data[80] = 0x40
+            # dds_data[88] = 0x18
+            dds_data[88] = 0x08
+            dds_data[94] = 0xFF
+            dds_data[97] = 0xFF
+            dds_data[100] = 0xFF
+
+        # compression = file_data.to_number(4, 0x10)
+        #
+        # if compression == 20:  # 14 00 00 00 - 888 (R8G8B8)
+        #     dds_data[0x4C] = 0x20  # ?
+        #     dds_data[0x50] = 0x40  # compressed or not
+        #
+        #     dds_data[0x58] = 0x18  # bytes per pixel
+        #     dds_data[0x5E] = 0xFF
+        #     dds_data[0x61] = 0xFF
+        #     dds_data[0x64] = 0xFF
+        # elif compression == 21:  # 15 00 00 00 - 8888 (R8G8B8A8)
+        #     dds_data[0x4C] = 0x20  # ?
+        #     dds_data[0x50] = 0x40  # compressed or not
+        #
+        #     dds_data[0x58] = 0x20  # bytes per pixel
+        #     dds_data[0x5E] = 0xFF
+        #     dds_data[0x61] = 0xFF
+        #     dds_data[0x64] = 0xFF
+        #     dds_data[0x6B] = 0xFF
+        # elif compression == 28:  # 1C 00 00 00 - 332 (?)
+        #     dds_data[0x4C] = 0x20  # ?
+        #     dds_data[0x50] = 0x40  # compressed or not
+        #
+        #     dds_data[0x58] = 0x08  # bytes per pixel
+        #     dds_data[0x5E] = 0xFF
+        #     dds_data[0x61] = 0xFF
+        #     dds_data[0x64] = 0xFF
+        # elif compression == 827611204:  # 44 58 54 31 - DXT1
+        #     dds_data[76] = 32
+        #     dds_data[80] = 4
+        #
+        #     dds_data[84] = 68
+        #     dds_data[85] = 88
+        #     dds_data[86] = 84
+        #     dds_data[87] = 49
+        # elif compression == 861165636:  # 44 58 54 33 - DXT3
+        #     dds_data[22] = 1
+        #     dds_data[76] = 32
+        #     dds_data[80] = 4
+        #
+        #     dds_data[84] = 68
+        #     dds_data[85] = 88
+        #     dds_data[86] = 84
+        #     dds_data[87] = 51
+        #
+        #     dds_data[108] = 8
+        #     dds_data[109] = 16
+        #     dds_data[110] = 64
+        # elif compression == 894720068:  # 44 58 54 35 - DXT5
+        #     dds_data[10] = 8
+        #     dds_data[22] = 1
+        #     dds_data[28] = 1
+        #     dds_data[76] = 32
+        #     dds_data[80] = 4
+        #
+        #     dds_data[84] = 68
+        #     dds_data[85] = 88
+        #     dds_data[86] = 84
+        #     dds_data[87] = 53
+        #
+        #     dds_data[88] = 32
+        #     dds_data[94] = 255
+        #     dds_data[97] = 255
+        #     dds_data[100] = 255
+        #     dds_data[107] = 255
+        #     dds_data[109] = 16
+        # else:
+        #     print("Unknown header format.")
+        #     return SubfileData()
+
+        result = SubfileData()
+        result.binary_data = dds_data
+        result.options = {"ext": ".dds"}
+        return result
+
+    @staticmethod
+    def build_for_import(old_data, data):
+        file_size = BinaryData.from_number(4, len(data.binary_data) - 128)
+        import_header = BinaryData(bytearray(20))
+        for i in range(20):
+            import_header[i] = 0
+        #     file_id
+        # import_header[0]= 0x0C
+        # import_header[1]= 0x3B
+        # import_header[2]= 0x00
+        # import_header[3]= 0x41
+
+        old_data_arr = old_data.data()
+        import_header[0] = old_data_arr[0]
+        import_header[1] = old_data_arr[1]
+        import_header[2] = old_data_arr[2]
+        import_header[3] = old_data_arr[3]
+
+        # import_header[4]= 0x0F
+        import_header[4] = old_data_arr[4]
+        import_header[5] = old_data_arr[5]
+        import_header[6] = old_data_arr[6]
+        import_header[7] = old_data_arr[7]
+
+        # width height
+        # import_header[8]= 0x00
+        # import_header[9]= 0x01
+        # import_header[10]= 0x00
+        # import_header[11]= 0x00
+        # import_header[12]= 0x44
+        # import_header[13]= 0x01
+        # import_header[14]= 0x00
+        # import_header[15]= 0x00
+
+        import_header[8] = data.binary_data[16]
+        import_header[9] = data.binary_data[17]
+        import_header[10] = data.binary_data[18]
+        import_header[11] = data.binary_data[19]
+        import_header[12] = data.binary_data[12]
+        import_header[13] = data.binary_data[13]
+        import_header[14] = data.binary_data[14]
+        import_header[15] = data.binary_data[15]
+
+        # import_header[16]= 0x1C
+        import_header[16] = old_data_arr[16]
+        import_header[17] = old_data_arr[17]
+        import_header[18] = old_data_arr[18]
+        import_header[19] = old_data_arr[19]
+
+        # print("new_header:")
+        # DDSSubfile.print_hex_bytes(import_header.data())
+
+        # print("old_header:")
+        # DDSSubfile.print_hex_bytes(old_data.data())
+
+        # return old_data.cut_data(0, 20) + file_size + data.binary_data.cut_data(128)
+        return import_header.cut_data(0, 20) + file_size + data.binary_data.cut_data(128)
+
+    @staticmethod
+    def print_hex_bytes(data, count=20):
+        """
+        打印bytearray前count字节的十六进制表示
+
+        Args:
+            data: bytearray 或 bytes 对象
+            count: 要打印的字节数，默认为20
+        """
+        # 获取前count个字节
+        bytes_to_print = data[:count]
+
+        # 方法3：使用f-string (Python 3.6+)
+        print("十六进制(f-string):", ' '.join(f'{b:02X}' for b in bytes_to_print))