PyCriCodecsEx 0.0.5__cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
PyCriCodecsEx/usm.py ADDED
@@ -0,0 +1,1001 @@
1
+ import os
2
+ import itertools, shutil
3
+ from typing import BinaryIO, List
4
+ from io import FileIO, BytesIO
5
+ from functools import cached_property
6
+
7
+ from PyCriCodecsEx.chunk import *
8
+ from PyCriCodecsEx.utf import UTF, UTFBuilder
9
+ try:
10
+ import ffmpeg
11
+ except ImportError:
12
+ raise ImportError("ffmpeg-python is required for USM support. Install via PyCriCodecsEx[usm] extra.")
13
+ import tempfile
14
+
15
+ # Big thanks and credit for k0lb3 and 9th helping me write this specific code.
16
+ # Also credit for the original C++ code from Nyagamon/bnnm and https://github.com/donmai-me/WannaCRI/
17
+
18
+ class USMCrypt:
19
+ """USM related crypto functions"""
20
+ videomask1: bytearray
21
+ videomask2: bytearray
22
+ audiomask: bytearray
23
+
24
+ def init_key(self, key: str):
25
+ if type(key) == str:
26
+ if len(key) <= 16:
27
+ key = key.rjust(16, "0")
28
+ key1 = bytes.fromhex(key[8:])
29
+ key2 = bytes.fromhex(key[:8])
30
+ else:
31
+ raise ValueError("Invalid input key.")
32
+ elif type(key) == int:
33
+ key1 = int.to_bytes(key & 0xFFFFFFFF, 4, "big")
34
+ key2 = int.to_bytes(key >> 32, 4, "big")
35
+ else:
36
+ raise ValueError(
37
+ "Invalid key format, must be either a string or an integer."
38
+ )
39
+ t = bytearray(0x20)
40
+ t[0x00:0x09] = [
41
+ key1[3],
42
+ key1[2],
43
+ key1[1],
44
+ (key1[0] - 0x34) % 0x100,
45
+ (key2[3] + 0xF9) % 0x100,
46
+ (key2[2] ^ 0x13) % 0x100,
47
+ (key2[1] + 0x61) % 0x100,
48
+ (key1[3] ^ 0xFF) % 0x100,
49
+ (key1[1] + key1[2]) % 0x100,
50
+ ]
51
+ t[0x09:0x0C] = [
52
+ (t[0x01] - t[0x07]) % 0x100,
53
+ (t[0x02] ^ 0xFF) % 0x100,
54
+ (t[0x01] ^ 0xFF) % 0x100,
55
+ ]
56
+ t[0x0C:0x0E] = [
57
+ (t[0x0B] + t[0x09]) % 0x100,
58
+ (t[0x08] - t[0x03]) % 0x100,
59
+ ]
60
+ t[0x0E:0x10] = [
61
+ (t[0x0D] ^ 0xFF) % 0x100,
62
+ (t[0x0A] - t[0x0B]) % 0x100,
63
+ ]
64
+ t[0x10] = (t[0x08] - t[0x0F]) % 0x100
65
+ t[0x11:0x17] = [
66
+ (t[0x10] ^ t[0x07]) % 0x100,
67
+ (t[0x0F] ^ 0xFF) % 0x100,
68
+ (t[0x03] ^ 0x10) % 0x100,
69
+ (t[0x04] - 0x32) % 0x100,
70
+ (t[0x05] + 0xED) % 0x100,
71
+ (t[0x06] ^ 0xF3) % 0x100,
72
+ ]
73
+ t[0x17:0x1A] = [
74
+ (t[0x13] - t[0x0F]) % 0x100,
75
+ (t[0x15] + t[0x07]) % 0x100,
76
+ (0x21 - t[0x13]) % 0x100,
77
+ ]
78
+ t[0x1A:0x1C] = [
79
+ (t[0x14] ^ t[0x17]) % 0x100,
80
+ (t[0x16] + t[0x16]) % 0x100,
81
+ ]
82
+ t[0x1C:0x1F] = [
83
+ (t[0x17] + 0x44) % 0x100,
84
+ (t[0x03] + t[0x04]) % 0x100,
85
+ (t[0x05] - t[0x16]) % 0x100,
86
+ ]
87
+ t[0x1F] = (t[0x1D] ^ t[0x13]) % 0x100
88
+ t2 = [b"U", b"R", b"U", b"C"]
89
+ self.videomask1 = t
90
+ self.videomask2 = bytearray(map(lambda x: x ^ 0xFF, t))
91
+ self.audiomask = bytearray(0x20)
92
+ for x in range(0x20):
93
+ if (x & 1) == 1:
94
+ self.audiomask[x] = ord(t2[(x >> 1) & 3])
95
+ else:
96
+ self.audiomask[x] = self.videomask2[x]
97
+
98
+ # Decrypt SFV chunks or ALP chunks, should only be used if the video data is encrypted.
99
+ def VideoMask(self, memObj: bytearray) -> bytearray:
100
+ head = memObj[:0x40]
101
+ memObj = memObj[0x40:]
102
+ size = len(memObj)
103
+ # memObj len is a cached property, very fast to lookup
104
+ if size <= 0x200:
105
+ return head + memObj
106
+ data_view = memoryview(memObj).cast("Q")
107
+
108
+ # mask 2
109
+ mask = bytearray(self.videomask2)
110
+ mask_view = memoryview(mask).cast("Q")
111
+ vmask = self.videomask2
112
+ vmask_view = memoryview(vmask).cast("Q")
113
+
114
+ mask_index = 0
115
+
116
+ for i in range(32, size // 8):
117
+ data_view[i] ^= mask_view[mask_index]
118
+ mask_view[mask_index] = data_view[i] ^ vmask_view[mask_index]
119
+ mask_index = (mask_index + 1) % 4
120
+
121
+ # mask 1
122
+ mask = bytearray(self.videomask1)
123
+ mask_view = memoryview(mask).cast("Q")
124
+ mask_index = 0
125
+ for i in range(32):
126
+ mask_view[mask_index] ^= data_view[i + 32]
127
+ data_view[i] ^= mask_view[mask_index]
128
+ mask_index = (mask_index + 1) % 4
129
+
130
+ return head + memObj
131
+
132
+ # Decrypts SFA chunks, should just be used with ADX files.
133
+ def AudioMask(self, memObj: bytearray) -> bytearray:
134
+ head = memObj[:0x140]
135
+ memObj = memObj[0x140:]
136
+ size = len(memObj)
137
+ data_view = memoryview(memObj).cast("Q")
138
+ mask = bytearray(self.audiomask)
139
+ mask_view = memoryview(mask).cast("Q")
140
+ for i in range(size // 8):
141
+ data_view[i] ^= mask_view[i % 4]
142
+ return head + memObj
143
+
144
+
145
+ # There are a lot of unknowns, minbuf(minimum buffer of what?) and avbps(average bitrate per second)
146
+ # are still unknown how to derive them, at least video wise it is possible, no idea how it's calculated audio wise nor anything else
147
+ # seems like it could be random values and the USM would still work.
148
+ class FFmpegCodec:
149
+ """Base codec for FFMpeg-based Video streams"""
150
+ filename: str
151
+ filesize: int
152
+
153
+ info: dict
154
+ file: FileIO
155
+
156
+ minchk: int
157
+ minbuf: int
158
+ avbps: int
159
+
160
+ def __init__(self, stream: str | bytes):
161
+ """Initialize FFmpegCodec with a media stream, gathering metadata and frame info.
162
+
163
+ Args:
164
+ stream (str | bytes): The media stream to process.
165
+ NOTE:
166
+ A temp file maybe created for probing only. Which will be deleted after use.
167
+ """
168
+ if type(stream) == str:
169
+ self.filename = stream
170
+ else:
171
+ self.tempfile = tempfile.NamedTemporaryFile(delete=False)
172
+ self.tempfile.write(stream)
173
+ self.tempfile.close()
174
+ self.filename = self.tempfile.name
175
+ self.info = ffmpeg.probe(
176
+ self.filename, show_entries="packet=dts,pts_time,pos,flags,duration_time"
177
+ )
178
+ if type(stream) == str:
179
+ self.file = open(self.filename, "rb")
180
+ self.filesize = os.path.getsize(self.filename)
181
+ else:
182
+ os.unlink(self.tempfile.name)
183
+ self.file = BytesIO(stream)
184
+ self.filesize = len(stream)
185
+
186
+ @property
187
+ def format(self):
188
+ return self.info["format"]["format_name"]
189
+
190
+ @property
191
+ def stream(self) -> dict:
192
+ return self.info["streams"][0]
193
+
194
+ @property
195
+ def codec(self):
196
+ return self.stream["codec_name"]
197
+
198
+ @cached_property
199
+ def framerate(self):
200
+ """Running framerate (max frame rate)"""
201
+ # Lesson learned. Do NOT trust the metadata.
202
+ # num, denom = self.stream["r_frame_rate"].split("/")
203
+ # return int(int(num) / int(denom))
204
+ return 1 / min((dt for _, _, _, dt in self.frames()))
205
+
206
+ @cached_property
207
+ def avg_framerate(self):
208
+ """Average framerate"""
209
+ # avg_frame_rate = self.stream.get("avg_frame_rate", None)
210
+ # if avg_frame_rate:
211
+ # num, denom = avg_frame_rate.split("/")
212
+ # return int(int(num) / int(denom))
213
+ return self.frame_count / sum((dt for _, _, _, dt in self.frames()))
214
+
215
+ @property
216
+ def packets(self):
217
+ return self.info["packets"]
218
+
219
+ @property
220
+ def width(self):
221
+ return self.stream["width"]
222
+
223
+ @property
224
+ def height(self):
225
+ return self.stream["height"]
226
+
227
+ @property
228
+ def frame_count(self):
229
+ return len(self.packets)
230
+
231
+ def frames(self):
232
+ """Generator of [frame data, frame dict, is keyframe, duration]"""
233
+ offsets = [int(packet["pos"]) for packet in self.packets] + [self.filesize]
234
+ for i, frame in enumerate(self.packets):
235
+ frame_size = offsets[i + 1] - offsets[i]
236
+ self.file.seek(offsets[i])
237
+ raw_frame = self.file.read(frame_size)
238
+ yield raw_frame, frame, frame["flags"][0] == "K", float(frame["duration_time"])
239
+
240
+ def generate_SFV(self, builder: "USMBuilder"):
241
+ v_framerate = int(self.framerate)
242
+ current_interval = 0
243
+ SFV_list = []
244
+ SFV_chunk = b""
245
+ count = 0
246
+ self.minchk = 0
247
+ self.minbuf = 0
248
+ bitrate = 0
249
+ for data, _, is_keyframe, dt in self.frames():
250
+ # SFV has priority in chunks, it comes first.
251
+ datalen = len(data)
252
+ padlen = 0x20 - (datalen % 0x20) if datalen % 0x20 != 0 else 0
253
+ SFV_chunk = USMChunkHeader.pack(
254
+ USMChunckHeaderType.SFV.value,
255
+ datalen + 0x18 + padlen,
256
+ 0,
257
+ 0x18,
258
+ padlen,
259
+ 0,
260
+ 0,
261
+ 0,
262
+ 0,
263
+ int(current_interval),
264
+ v_framerate,
265
+ 0,
266
+ 0,
267
+ )
268
+ if builder.encrypt:
269
+ data = builder.VideoMask(data)
270
+ SFV_chunk += data
271
+ SFV_chunk = SFV_chunk.ljust(datalen + 0x18 + padlen + 0x8, b"\x00")
272
+ SFV_list.append(SFV_chunk)
273
+ count += 1
274
+ current_interval += 2997 * dt # 29.97 as base
275
+ if is_keyframe:
276
+ self.minchk += 1
277
+ if self.minbuf < datalen:
278
+ self.minbuf = datalen
279
+ bitrate += datalen * 8 * v_framerate
280
+ else:
281
+ self.avbps = int(bitrate / count)
282
+ SFV_chunk = USMChunkHeader.pack(
283
+ USMChunckHeaderType.SFV.value, 0x38, 0, 0x18, 0, 0, 0, 0, 2, 0, 30, 0, 0
284
+ )
285
+ SFV_chunk += b"#CONTENTS END ===============\x00"
286
+ SFV_list.append(SFV_chunk)
287
+ return SFV_list
288
+
289
+ def save(self, filepath: str):
290
+ '''Saves the underlying video stream to a file.'''
291
+ tell = self.file.tell()
292
+ self.file.seek(0)
293
+ shutil.copyfileobj(self.file, open(filepath, 'wb'))
294
+ self.file.seek(tell)
295
+
296
+ class VP9Codec(FFmpegCodec):
297
+ """VP9 Video stream codec.
298
+
299
+ Only streams with `.ivf` containers are supported."""
300
+ MPEG_CODEC = 9
301
+ MPEG_DCPREC = 0
302
+ VERSION = 16777984
303
+
304
+ def __init__(self, filename: str | bytes):
305
+ super().__init__(filename)
306
+ assert self.format == "ivf", "must be ivf format."
307
+ class H264Codec(FFmpegCodec):
308
+ """H264 Video stream codec.
309
+
310
+ Only streams with `.h264` containers are supported."""
311
+ MPEG_CODEC = 5
312
+ MPEG_DCPREC = 11
313
+ VERSION = 0
314
+
315
+ def __init__(self, filename : str | bytes):
316
+ super().__init__(filename)
317
+ assert (
318
+ self.format == "h264"
319
+ ), "must be raw h264 data. transcode with '.h264' suffix as output"
320
+ class MPEG1Codec(FFmpegCodec):
321
+ """MPEG1 Video stream codec.
322
+
323
+ Only streams with `.mpeg1` containers are supported."""
324
+ MPEG_CODEC = 1
325
+ MPEG_DCPREC = 11
326
+ VERSION = 0
327
+
328
+ def __init__(self, stream : str | bytes):
329
+ super().__init__(stream)
330
+ assert self.format == "mpegvideo", "must be m1v format (mpegvideo)."
331
+
332
+ from PyCriCodecsEx.hca import HCACodec
333
+ from PyCriCodecsEx.adx import ADXCodec
334
+
335
+ class USM(USMCrypt):
336
+ """Use this class to extract infromation and data from a USM file."""
337
+
338
+ filename: str
339
+ decrypt: bool
340
+ stream: BinaryIO
341
+ CRIDObj: UTF
342
+ output: dict[str, bytes]
343
+ size: int
344
+ demuxed: bool
345
+
346
+ audio_codec: int
347
+ video_codec: int
348
+
349
+ metadata: list
350
+
351
+ def __init__(self, filename : str | BinaryIO, key: str | int = None):
352
+ """Loads a USM file into memory and prepares it for processing.
353
+
354
+ Args:
355
+ filename (str): The path to the USM file.
356
+ key (str, optional): The decryption key. Either int64 or a hex string. Defaults to None.
357
+ """
358
+ self.filename = filename
359
+ self.decrypt = False
360
+
361
+ if key:
362
+ self.decrypt = True
363
+ self.init_key(key)
364
+ self._load_file()
365
+
366
+ def _load_file(self):
367
+ if type(self.filename) == str:
368
+ self.stream = open(self.filename, "rb")
369
+ else:
370
+ self.stream = self.filename
371
+ self.stream.seek(0, 2)
372
+ self.size = self.stream.tell()
373
+ self.stream.seek(0)
374
+ header = self.stream.read(4)
375
+ if header != USMChunckHeaderType.CRID.value:
376
+ raise NotImplementedError(f"Unsupported file type: {header}")
377
+ self.stream.seek(0)
378
+ self._demux()
379
+
380
+ def _demux(self) -> None:
381
+ """Gets data from USM chunks and assignes them to output."""
382
+ self.stream.seek(0)
383
+ self.metadata = list()
384
+ (
385
+ header,
386
+ chuncksize,
387
+ unk08,
388
+ offset,
389
+ padding,
390
+ chno,
391
+ unk0D,
392
+ unk0E,
393
+ type,
394
+ frametime,
395
+ framerate,
396
+ unk18,
397
+ unk1C,
398
+ ) = USMChunkHeader.unpack(self.stream.read(USMChunkHeader.size))
399
+ chuncksize -= 0x18
400
+ offset -= 0x18
401
+ self.CRIDObj = UTF(self.stream.read(chuncksize))
402
+ CRID_payload = self.CRIDObj.dictarray
403
+ headers = [
404
+ (int.to_bytes(x["stmid"][1], 4, "big")).decode() for x in CRID_payload[1:]
405
+ ]
406
+ chnos = [x["chno"][1] for x in CRID_payload[1:]]
407
+ output = dict()
408
+ for i in range(len(headers)):
409
+ output[headers[i] + "_" + str(chnos[i])] = bytearray()
410
+ while self.stream.tell() < self.size:
411
+ header: bytes
412
+ (
413
+ header,
414
+ chuncksize,
415
+ unk08,
416
+ offset,
417
+ padding,
418
+ chno,
419
+ unk0D,
420
+ unk0E,
421
+ type,
422
+ frametime,
423
+ framerate,
424
+ unk18,
425
+ unk1C,
426
+ ) = USMChunkHeader.unpack(self.stream.read(USMChunkHeader.size))
427
+ chuncksize -= 0x18
428
+ offset -= 0x18
429
+ if header.decode() in headers:
430
+ if type == 0:
431
+ data = self._reader(chuncksize, offset, padding, header)
432
+ output[header.decode() + "_" + str(chno)].extend(data)
433
+ elif type == 1 or type == 3:
434
+ ChunkObj = UTF(self.stream.read(chuncksize))
435
+ self.metadata.append(ChunkObj)
436
+ if type == 1:
437
+ if header == USMChunckHeaderType.SFA.value:
438
+ codec = ChunkObj.dictarray[0]
439
+ self.audio_codec = codec["audio_codec"][1]
440
+ # So far, audio_codec of 2, means ADX, while audio_codec 4 means HCA.
441
+ if header == USMChunckHeaderType.SFV.value:
442
+ self.video_codec = ChunkObj.dictarray[0]['mpeg_codec'][1]
443
+ else:
444
+ self.stream.seek(chuncksize, 1)
445
+ else:
446
+ # It is likely impossible for the code to reach here, since the code right now is suitable
447
+ # for any chunk type specified in the CRID header.
448
+ # But just incase somehow there's an extra chunk, this code might handle it.
449
+ if header in [chunk.value for chunk in USMChunckHeaderType]:
450
+ if type == 0:
451
+ output[header.decode() + "_0"] = bytearray()
452
+ data = self._reader(chuncksize, offset, padding, header)
453
+ output[header.decode() + "_0"].extend(
454
+ data
455
+ ) # No channel number info, code here assumes it's a one channel data type.
456
+ elif type == 1 or type == 3:
457
+ ChunkObj = UTF(self.stream.read(chuncksize))
458
+ self.metadata.append(ChunkObj)
459
+ if type == 1 and header == USMChunckHeaderType.SFA.value:
460
+ codec = ChunkObj.dictarray[0]
461
+ self.audio_codec = codec["audio_codec"][1]
462
+ else:
463
+ self.stream.seek(chuncksize, 1)
464
+ else:
465
+ raise NotImplementedError(f"Unsupported chunk type: {header}")
466
+ self.output = output
467
+ self.demuxed = True
468
+
469
+ def _reader(self, chuncksize, offset, padding, header) -> bytearray:
470
+ """Chunks reader function, reads all data in a chunk and returns a bytearray."""
471
+ data = bytearray(self.stream.read(chuncksize)[offset:])
472
+ if (
473
+ header == USMChunckHeaderType.SFV.value
474
+ or header == USMChunckHeaderType.ALP.value
475
+ ):
476
+ data = self.VideoMask(data) if self.decrypt else data
477
+ elif header == USMChunckHeaderType.SFA.value:
478
+ data = self.AudioMask(data) if (self.audio_codec == 2 and self.decrypt) else data
479
+ if padding:
480
+ data = data[:-padding]
481
+ return data
482
+
483
+ @property
484
+ def streams(self):
485
+ """Generator of Tuple[Stream Type ("@SFV", "@SFA"), File name, Raw stream data]"""
486
+ for stream in self.CRIDObj.dictarray[1:]:
487
+ filename, stmid, chno = stream["filename"][1], stream["stmid"][1], stream["chno"][1]
488
+ stmid = int.to_bytes(stmid, 4, 'big', signed='False')
489
+ yield stmid, str(filename), self.output.get(f'{stmid.decode()}_{chno}', None)
490
+
491
+ def get_video(self) -> VP9Codec | H264Codec | MPEG1Codec:
492
+ """Create a video codec from the available streams.
493
+
494
+ NOTE: A temporary file may be created with this process to determine the stream information."""
495
+ stype, sfname, sraw = next(filter(lambda x: x[0] == USMChunckHeaderType.SFV.value, self.streams), (None, None, None))
496
+ stream = None
497
+ match self.video_codec:
498
+ case MPEG1Codec.MPEG_CODEC:
499
+ stream = MPEG1Codec(sraw)
500
+ case H264Codec.MPEG_CODEC:
501
+ stream = H264Codec(sraw)
502
+ case VP9Codec.MPEG_CODEC:
503
+ stream = VP9Codec(sraw)
504
+ case _:
505
+ raise NotImplementedError(f"Unsupported video codec: {self.video_codec}")
506
+ stream.filename = sfname
507
+ return stream
508
+
509
+ def get_audios(self) -> List[ADXCodec | HCACodec]:
510
+ """Create a list of audio codecs from the available streams."""
511
+ match self.audio_codec:
512
+ case ADXCodec.AUDIO_CODEC:
513
+ return [ADXCodec(s[2], s[1]) for s in self.streams if s[0] == USMChunckHeaderType.SFA.value]
514
+ case HCACodec.AUDIO_CODEC:
515
+ return [HCACodec(s[2], s[1]) for s in self.streams if s[0] == USMChunckHeaderType.SFA.value] # HCAs are never encrypted in USM
516
+ case _:
517
+ return []
518
+
519
+ class USMBuilder(USMCrypt):
520
+ """Use this class to build USM files."""
521
+ video_stream: VP9Codec | H264Codec | MPEG1Codec
522
+ audio_streams: List[HCACodec | ADXCodec]
523
+
524
+ key: int = None
525
+ encrypt: bool = False
526
+ encrypt_audio: bool = False
527
+
528
+ def __init__(
529
+ self,
530
+ key = None,
531
+ encrypt_audio = False
532
+ ) -> None:
533
+ """Initialize the USMBuilder from set source files.
534
+
535
+ Args:
536
+ key (str | int, optional): The encryption key. Either int64 or a hex string. Defaults to None.
537
+ encrypt_audio (bool, optional): Whether to also encrypt the audio. Defaults to False.
538
+ """
539
+ if key:
540
+ self.init_key(key)
541
+ self.encrypt = True
542
+ self.encrypt_audio = encrypt_audio
543
+ self.audio_streams = []
544
+
545
+ def add_video(self, video : str | H264Codec | VP9Codec | MPEG1Codec):
546
+ """Sets the video stream from the specified video file.
547
+
548
+ USMs only support one video stream. Consecutive calls to this method will replace the existing video stream.
549
+
550
+ When `video` is str - it will be treated as a file path. The video source format will be used to map accordingly to the ones Sofdec use.
551
+ - MPEG1 (with M1V container): MPEG1 Codec (Sofdec Prime)
552
+ - H264 (with H264 container): H264 Codec
553
+ - VP9 (with IVF container): VP9 Codec
554
+
555
+ Args:
556
+ video (str | FFmpegCodec): The path to the video file or an FFmpegCodec instance.
557
+ """
558
+ if isinstance(video, str):
559
+ temp_stream = FFmpegCodec(video)
560
+ self.video_stream = None
561
+ match temp_stream.stream["codec_name"]:
562
+ case "h264":
563
+ self.video_stream = H264Codec(video)
564
+ case "vp9":
565
+ self.video_stream = VP9Codec(video)
566
+ case "mpeg1video":
567
+ self.video_stream = MPEG1Codec(video)
568
+ assert self.video_stream, (
569
+ "fail to match suitable video codec. Codec=%s"
570
+ % temp_stream.stream["codec_name"]
571
+ )
572
+ else:
573
+ self.video_stream = video
574
+
575
+ def add_audio(self, audio : ADXCodec | HCACodec):
576
+ """Append the audio stream(s) from the specified audio file(s).
577
+
578
+ Args:
579
+ audio (ADXCodec | HCACodec): The path(s) to the audio file(s).
580
+ """
581
+ self.audio_streams.append(audio)
582
+
583
+ def build(self) -> bytes:
584
+ """Build the USM payload"""
585
+ SFV_list = self.video_stream.generate_SFV(self)
586
+ if self.audio_streams:
587
+ SFA_chunks = [s.generate_SFA(i, self) for i, s in enumerate(self.audio_streams) ]
588
+ else:
589
+ SFA_chunks = []
590
+ SBT_chunks = [] # TODO: Subtitles
591
+ header = self._build_header(SFV_list, SFA_chunks, SBT_chunks)
592
+ chunks = list(itertools.chain(SFV_list, *SFA_chunks))
593
+
594
+ def chunk_key_sort(chunk):
595
+ (
596
+ header,
597
+ chuncksize,
598
+ unk08,
599
+ offset,
600
+ padding,
601
+ chno,
602
+ unk0D,
603
+ unk0E,
604
+ type,
605
+ frametime,
606
+ framerate,
607
+ unk18,
608
+ unk1C,
609
+ ) = USMChunkHeader.unpack(chunk[: USMChunkHeader.size])
610
+ prio = 0 if header == USMChunckHeaderType.SFV else 1
611
+ # all stream chunks before section_end chunks, then sort by frametime, with SFV chunks before SFA chunks
612
+ return (type, frametime, prio)
613
+
614
+ chunks.sort(key=chunk_key_sort)
615
+ self.usm = header
616
+ chunks = b''.join(chunks)
617
+ self.usm += chunks
618
+ return self.usm
619
+
620
+ def _build_header(
621
+ self, SFV_list: list, SFA_chunks: list, SBT_chunks: list # TODO: Not used
622
+ ) -> bytes:
623
+ # Main USM file
624
+ CRIUSF_DIR_STREAM = [
625
+ dict(
626
+ fmtver=(UTFTypeValues.uint, self.video_stream.VERSION),
627
+ filename=(
628
+ UTFTypeValues.string,
629
+ os.path.splitext(os.path.basename(self.video_stream.filename))[0]
630
+ + ".usm",
631
+ ),
632
+ filesize=(UTFTypeValues.uint, -1), # Will be updated later.
633
+ datasize=(UTFTypeValues.uint, 0),
634
+ stmid=(UTFTypeValues.uint, 0),
635
+ chno=(UTFTypeValues.ushort, 0xFFFF),
636
+ minchk=(UTFTypeValues.ushort, 1),
637
+ minbuf=(UTFTypeValues.uint, -1), # Will be updated later.
638
+ avbps=(UTFTypeValues.uint, -1), # Will be updated later.
639
+ )
640
+ ]
641
+
642
+ total_avbps = self.video_stream.avbps
643
+ minbuf = 4 + self.video_stream.minbuf
644
+
645
+ v_filesize = self.video_stream.filesize
646
+
647
+ video_dict = dict(
648
+ fmtver=(UTFTypeValues.uint, self.video_stream.VERSION),
649
+ filename=(
650
+ UTFTypeValues.string,
651
+ os.path.basename(self.video_stream.filename),
652
+ ),
653
+ filesize=(UTFTypeValues.uint, v_filesize),
654
+ datasize=(UTFTypeValues.uint, 0),
655
+ stmid=(
656
+ UTFTypeValues.uint,
657
+ int.from_bytes(USMChunckHeaderType.SFV.value, "big"),
658
+ ),
659
+ chno=(UTFTypeValues.ushort, 0),
660
+ minchk=(UTFTypeValues.ushort, self.video_stream.minchk),
661
+ minbuf=(UTFTypeValues.uint, self.video_stream.minbuf),
662
+ avbps=(UTFTypeValues.uint, self.video_stream.avbps),
663
+ )
664
+ CRIUSF_DIR_STREAM.append(video_dict)
665
+
666
+ if self.audio_streams:
667
+ chno = 0
668
+ for stream in self.audio_streams:
669
+ avbps = stream.avbps
670
+ total_avbps += avbps
671
+ minbuf += 27860
672
+ audio_dict = dict(
673
+ fmtver=(UTFTypeValues.uint, 0),
674
+ filename=(UTFTypeValues.string, stream.filename),
675
+ filesize=(UTFTypeValues.uint, stream.filesize),
676
+ datasize=(UTFTypeValues.uint, 0),
677
+ stmid=(
678
+ UTFTypeValues.uint,
679
+ int.from_bytes(USMChunckHeaderType.SFA.value, "big"),
680
+ ),
681
+ chno=(UTFTypeValues.ushort, chno),
682
+ minchk=(UTFTypeValues.ushort, 1),
683
+ minbuf=(
684
+ UTFTypeValues.uint,
685
+ 27860,
686
+ ), # minbuf is fixed at that for audio.
687
+ avbps=(UTFTypeValues.uint, avbps),
688
+ )
689
+ CRIUSF_DIR_STREAM.append(audio_dict)
690
+ chno += 1
691
+
692
+ CRIUSF_DIR_STREAM[0]["avbps"] = (UTFTypeValues.uint, total_avbps)
693
+ CRIUSF_DIR_STREAM[0]["minbuf"] = (
694
+ UTFTypeValues.uint,
695
+ minbuf,
696
+ ) # Wrong. TODO Despite being fixed per SFA stream, seems to change internally before summation.
697
+
698
+ def gen_video_hdr_info(metadata_size: int):
699
+ hdr = [
700
+ {
701
+ "width": (UTFTypeValues.uint, self.video_stream.width),
702
+ "height": (UTFTypeValues.uint, self.video_stream.height),
703
+ "mat_width": (UTFTypeValues.uint, self.video_stream.width),
704
+ "mat_height": (UTFTypeValues.uint, self.video_stream.height),
705
+ "disp_width": (UTFTypeValues.uint, self.video_stream.width),
706
+ "disp_height": (UTFTypeValues.uint, self.video_stream.height),
707
+ "scrn_width": (UTFTypeValues.uint, 0),
708
+ "mpeg_dcprec": (UTFTypeValues.uchar, self.video_stream.MPEG_DCPREC),
709
+ "mpeg_codec": (UTFTypeValues.uchar, self.video_stream.MPEG_CODEC),
710
+ "alpha_type": (UTFTypeValues.uint, 0),
711
+ "total_frames": (UTFTypeValues.uint, self.video_stream.frame_count),
712
+ "framerate_n": (
713
+ UTFTypeValues.uint,
714
+ int(self.video_stream.framerate * 1000),
715
+ ),
716
+ "framerate_d": (UTFTypeValues.uint, 1000), # Denominator
717
+ "metadata_count": (
718
+ UTFTypeValues.uint,
719
+ 1,
720
+ ), # Could be 0 and ignore metadata?
721
+ "metadata_size": (
722
+ UTFTypeValues.uint,
723
+ metadata_size,
724
+ ),
725
+ "ixsize": (UTFTypeValues.uint, self.video_stream.minbuf),
726
+ "pre_padding": (UTFTypeValues.uint, 0),
727
+ "max_picture_size": (UTFTypeValues.uint, 0),
728
+ "color_space": (UTFTypeValues.uint, 0),
729
+ "picture_type": (UTFTypeValues.uint, 0),
730
+ }
731
+ ]
732
+ v = UTFBuilder(hdr, table_name="VIDEO_HDRINFO")
733
+ v.strings = b"<NULL>\x00" + v.strings
734
+ hdr = v.bytes()
735
+ padding = 0x20 - (len(hdr) % 0x20) if (len(hdr) % 0x20) != 0 else 0
736
+ chk = USMChunkHeader.pack(
737
+ USMChunckHeaderType.SFV.value,
738
+ len(hdr) + 0x18 + padding,
739
+ 0,
740
+ 0x18,
741
+ padding,
742
+ 0,
743
+ 0,
744
+ 0,
745
+ 1,
746
+ 0,
747
+ 30,
748
+ 0,
749
+ 0,
750
+ )
751
+ chk += hdr.ljust(len(hdr) + padding, b"\x00")
752
+ return chk
753
+
754
+ audio_metadata = []
755
+ audio_headers = []
756
+ if self.audio_streams:
757
+ chno = 0
758
+ for stream in self.audio_streams:
759
+ metadata = stream.get_metadata()
760
+ if not metadata:
761
+ break
762
+ else:
763
+ padding = (
764
+ 0x20 - (len(metadata) % 0x20)
765
+ if len(metadata) % 0x20 != 0
766
+ else 0
767
+ )
768
+ chk = USMChunkHeader.pack(
769
+ USMChunckHeaderType.SFA.value,
770
+ len(metadata) + 0x18 + padding,
771
+ 0,
772
+ 0x18,
773
+ padding,
774
+ chno,
775
+ 0,
776
+ 0,
777
+ 3,
778
+ 0,
779
+ 30,
780
+ 0,
781
+ 0,
782
+ )
783
+ chk += metadata.ljust(len(metadata) + padding, b"\x00")
784
+ audio_metadata.append(chk)
785
+ chno += 1
786
+
787
+ chno = 0
788
+ for stream in self.audio_streams:
789
+ AUDIO_HDRINFO = [
790
+ {
791
+ "audio_codec": (UTFTypeValues.uchar, stream.AUDIO_CODEC),
792
+ "sampling_rate": (UTFTypeValues.uint, stream.sampling_rate),
793
+ "total_samples": (UTFTypeValues.uint, stream.total_samples),
794
+ "num_channels": (UTFTypeValues.uchar, stream.chnls),
795
+ "metadata_count": (UTFTypeValues.uint, stream.METADATA_COUNT),
796
+ "metadat_size": (UTFTypeValues.uint, len(audio_metadata[chno]) if audio_metadata else 0),
797
+ "ixsize": (UTFTypeValues.uint, 27860),
798
+ "ambisonics": (UTFTypeValues.uint, 0)
799
+ }
800
+ ]
801
+ p = UTFBuilder(AUDIO_HDRINFO, table_name="AUDIO_HDRINFO")
802
+ p.strings = b"<NULL>\x00" + p.strings
803
+ header = p.bytes()
804
+ padding = (
805
+ 0x20 - (len(header) % 0x20) if (len(header) % 0x20) != 0 else 0
806
+ )
807
+ chk = USMChunkHeader.pack(
808
+ USMChunckHeaderType.SFA.value,
809
+ len(header) + 0x18 + padding,
810
+ 0,
811
+ 0x18,
812
+ padding,
813
+ chno,
814
+ 0,
815
+ 0,
816
+ 1,
817
+ 0,
818
+ 30,
819
+ 0,
820
+ 0,
821
+ )
822
+ chk += header.ljust(len(header) + padding, b"\x00")
823
+ audio_headers.append(chk)
824
+ chno += 1
825
+
826
+ keyframes = [
827
+ (data["pos"], i)
828
+ for i, (frame, data, is_keyframe, duration) in enumerate(self.video_stream.frames())
829
+ if is_keyframe
830
+ ]
831
+
832
+ def comp_seek_info(first_chk_ofs):
833
+ seek = [
834
+ {
835
+ "ofs_byte": (UTFTypeValues.ullong, first_chk_ofs + int(pos)),
836
+ "ofs_frmid": (UTFTypeValues.int, i),
837
+ "num_skip": (UTFTypeValues.short, 0),
838
+ "resv": (UTFTypeValues.short, 0),
839
+ }
840
+ for pos, i in keyframes
841
+ ]
842
+ seek = UTFBuilder(seek, table_name="VIDEO_SEEKINFO")
843
+ seek.strings = b"<NULL>\x00" + seek.strings
844
+ seek = seek.bytes()
845
+ padding = 0x20 - len(seek) % 0x20 if len(seek) % 0x20 != 0 else 0
846
+ seekinf = USMChunkHeader.pack(
847
+ USMChunckHeaderType.SFV.value,
848
+ len(seek) + 0x18 + padding,
849
+ 0,
850
+ 0x18,
851
+ padding,
852
+ 0,
853
+ 0,
854
+ 0,
855
+ 3,
856
+ 0,
857
+ 30,
858
+ 0,
859
+ 0,
860
+ )
861
+ seekinf += seek.ljust(len(seek) + padding, b"\x00")
862
+ return seekinf
863
+
864
+ len_seek = len(comp_seek_info(0))
865
+ len_audio_headers = sum([len(x) + 0x40 for x in audio_headers])
866
+ len_audio_metadata = sum([len(x) + 0x40 for x in audio_metadata])
867
+ first_chk_ofs = (
868
+ 0x800 # CRID
869
+ + 512 # VIDEO_HDRINFO
870
+ + len_seek
871
+ + 128 # SFV_END * 2
872
+ + len_audio_headers
873
+ + len_audio_metadata
874
+ )
875
+ VIDEO_SEEKINFO = comp_seek_info(first_chk_ofs)
876
+ VIDEO_HDRINFO = gen_video_hdr_info(len(VIDEO_SEEKINFO))
877
+
878
+ total_len = sum([len(x) for x in SFV_list]) + first_chk_ofs
879
+ if self.audio_streams:
880
+ sum_len = 0
881
+ for stream in SFA_chunks:
882
+ for x in stream:
883
+ sum_len += len(x)
884
+ total_len += sum_len
885
+
886
+ CRIUSF_DIR_STREAM[0]["filesize"] = (UTFTypeValues.uint, total_len)
887
+ CRIUSF_DIR_STREAM = UTFBuilder(
888
+ CRIUSF_DIR_STREAM, table_name="CRIUSF_DIR_STREAM"
889
+ )
890
+ CRIUSF_DIR_STREAM.strings = b"<NULL>\x00" + CRIUSF_DIR_STREAM.strings
891
+ CRIUSF_DIR_STREAM = CRIUSF_DIR_STREAM.bytes()
892
+
893
+ ##############################################
894
+ # Parsing everything.
895
+ ##############################################
896
+ header = bytes()
897
+ # CRID
898
+ padding = 0x800 - len(CRIUSF_DIR_STREAM)
899
+ CRID = USMChunkHeader.pack(
900
+ USMChunckHeaderType.CRID.value,
901
+ 0x800 - 0x8,
902
+ 0,
903
+ 0x18,
904
+ padding - 0x20,
905
+ 0,
906
+ 0,
907
+ 0,
908
+ 1,
909
+ 0,
910
+ 30,
911
+ 0,
912
+ 0,
913
+ )
914
+ CRID += CRIUSF_DIR_STREAM.ljust(0x800 - 0x20, b"\x00")
915
+ header += CRID
916
+
917
+ # Header chunks
918
+ header += VIDEO_HDRINFO
919
+ if self.audio_streams:
920
+ header += b''.join(audio_headers)
921
+ SFV_END = USMChunkHeader.pack(
922
+ USMChunckHeaderType.SFV.value,
923
+ 0x38,
924
+ 0,
925
+ 0x18,
926
+ 0x0,
927
+ 0x0,
928
+ 0x0,
929
+ 0x0,
930
+ 2,
931
+ 0,
932
+ 30,
933
+ 0,
934
+ 0,
935
+ )
936
+ SFV_END += b"#HEADER END ===============\x00"
937
+ header += SFV_END
938
+
939
+ SFA_chk_END = b'' # Maybe reused
940
+ if self.audio_streams:
941
+ SFA_chk_END = b''.join([
942
+ USMChunkHeader.pack(
943
+ USMChunckHeaderType.SFA.value,
944
+ 0x38,
945
+ 0,
946
+ 0x18,
947
+ 0x0,
948
+ i,
949
+ 0x0,
950
+ 0x0,
951
+ 2,
952
+ 0,
953
+ 30,
954
+ 0,
955
+ 0,
956
+ ) + b"#HEADER END ===============\x00" for i in range(len(audio_headers))
957
+ ])
958
+ header += SFA_chk_END # Ends audio_headers
959
+ header += VIDEO_SEEKINFO
960
+
961
+ if self.audio_streams:
962
+ header += b''.join(audio_metadata)
963
+ SFV_END = USMChunkHeader.pack(
964
+ USMChunckHeaderType.SFV.value,
965
+ 0x38,
966
+ 0,
967
+ 0x18,
968
+ 0x0,
969
+ 0x0,
970
+ 0x0,
971
+ 0x0,
972
+ 2,
973
+ 0,
974
+ 30,
975
+ 0,
976
+ 0,
977
+ )
978
+ SFV_END += b"#METADATA END ===============\x00"
979
+ header += SFV_END
980
+
981
+ if audio_metadata:
982
+ SFA_chk_END = b''.join([
983
+ USMChunkHeader.pack(
984
+ USMChunckHeaderType.SFA.value,
985
+ 0x38,
986
+ 0,
987
+ 0x18,
988
+ 0x0,
989
+ i,
990
+ 0x0,
991
+ 0x0,
992
+ 2,
993
+ 0,
994
+ 30,
995
+ 0,
996
+ 0,
997
+ ) + b"#METADATA END ===============\x00" for i in range(len(audio_headers))
998
+ ])
999
+ header += SFA_chk_END # Ends audio_headers
1000
+
1001
+ return header