sssekai 0.0.1__tar.gz

sssekai-0.0.1/PKG-INFO

@@ -0,0 +1,64 @@
+Metadata-Version: 2.1
+Name: sssekai
+Version: 0.0.1
+Home-page: https://github.com/mos9527/sssekai
+Author: greats3an
+Author-email: greats3an@gmail.com
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+Requires-Dist: msgpack
+Requires-Dist: pycryptodome
+Requires-Dist: unitypy
+Requires-Dist: wannacri
+
+# sssekai
+Command-line tool for downloading/deobfuscating the game's assets, along with some other tools.
+
+    usage: sssekai [-h]
+                  {apidecrypt,abdecrypt,usmdemux,abcache,live2dextract,mitm} ...
+
+    SSSekai Proejct SEKAI feat. Hatsune Miku (Android) Modding Tools
+    Installation:
+        pip install git+https://github.com/mos9527/sssekai                                    
+
+    options:
+      -h, --help            show this help message and exit
+
+    subcommands:
+      valid subcommands
+
+      {apidecrypt,abdecrypt,usmdemux,abcache,live2dextract,mitm}
+                            additional help
+        apidecrypt          API crypto dumper
+                            This crypto applies to:
+                                - API request/response body dumped by packet sniffer (mitmproxy, wireshark, etc.)
+                                - AssetBundleInfo (can be found at /sdcard/Android/data/com.hermes.mk.asia/files/data/AssetBundleInfo,or see sssekai.abcache)
+        abdecrypt           Decrypt Sekai AssetBundle
+                            These can be found at /sdcard/Android/data/com.hermes.mk.asia/files/data/                                             
+        usmdemux            Demux Sekai USM Video in a AssetBundle
+        abcache             Sekai AssetBundle local cache
+                            Downloads/Updates *ALL* PJSK TW assets to local devices.
+                            NOTE: The assets can take quite a lot of space (est. 27GB) so be prepared
+                            NOTE: The AssetBundles cached are NOT OBFUSCATED. They can be used as is by various Unity ripping tools (and sssekai by extension)
+                                  that supports stripped Unity version (should be 2020.3.21f1).
+        live2dextract       Extract Sekai Live2D Models in a AssetBundle
+        mitm                Run Sekai API MITM proxy (WIP)
+
+# sssekai_blender_io
+Blender I/O for the game's assets. (actually it's imports only)
+
+Refer to https://github.com/mos9527/sssekai for downloads!
+
+Currently supports:  
+  * Textures
+  * Character NPR Material (via [SekaiShaderStandalone](https://github.com/mos9527/sssekai/blob/main/sssekai_blender_io/assets/SekaiShaderStandalone.blend))
+  * Static Mesh
+  * Skinned Mesh
+  * Armatures (built through GameObject hierarchy)
+  * Animations (Skeletal, BlendShape)
+
+## Installation:
+*WIP*

sssekai-0.0.1/README.md

@@ -0,0 +1,48 @@
+# sssekai
+Command-line tool for downloading/deobfuscating the game's assets, along with some other tools.
+
+    usage: sssekai [-h]
+                  {apidecrypt,abdecrypt,usmdemux,abcache,live2dextract,mitm} ...
+
+    SSSekai Proejct SEKAI feat. Hatsune Miku (Android) Modding Tools
+    Installation:
+        pip install git+https://github.com/mos9527/sssekai                                    
+
+    options:
+      -h, --help            show this help message and exit
+
+    subcommands:
+      valid subcommands
+
+      {apidecrypt,abdecrypt,usmdemux,abcache,live2dextract,mitm}
+                            additional help
+        apidecrypt          API crypto dumper
+                            This crypto applies to:
+                                - API request/response body dumped by packet sniffer (mitmproxy, wireshark, etc.)
+                                - AssetBundleInfo (can be found at /sdcard/Android/data/com.hermes.mk.asia/files/data/AssetBundleInfo,or see sssekai.abcache)
+        abdecrypt           Decrypt Sekai AssetBundle
+                            These can be found at /sdcard/Android/data/com.hermes.mk.asia/files/data/                                             
+        usmdemux            Demux Sekai USM Video in a AssetBundle
+        abcache             Sekai AssetBundle local cache
+                            Downloads/Updates *ALL* PJSK TW assets to local devices.
+                            NOTE: The assets can take quite a lot of space (est. 27GB) so be prepared
+                            NOTE: The AssetBundles cached are NOT OBFUSCATED. They can be used as is by various Unity ripping tools (and sssekai by extension)
+                                  that supports stripped Unity version (should be 2020.3.21f1).
+        live2dextract       Extract Sekai Live2D Models in a AssetBundle
+        mitm                Run Sekai API MITM proxy (WIP)
+
+# sssekai_blender_io
+Blender I/O for the game's assets. (actually it's imports only)
+
+Refer to https://github.com/mos9527/sssekai for downloads!
+
+Currently supports:  
+  * Textures
+  * Character NPR Material (via [SekaiShaderStandalone](https://github.com/mos9527/sssekai/blob/main/sssekai_blender_io/assets/SekaiShaderStandalone.blend))
+  * Static Mesh
+  * Skinned Mesh
+  * Armatures (built through GameObject hierarchy)
+  * Animations (Skeletal, BlendShape)
+
+## Installation:
+*WIP*

sssekai-0.0.1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

sssekai-0.0.1/setup.py

@@ -0,0 +1,27 @@
+import sys,os
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+import setuptools, sssekai
+
+with open("README.md", "r", encoding='utf-8') as fh:
+    long_description = fh.read()
+
+setuptools.setup(
+    name="sssekai",
+    version=sssekai.__version__,
+    author="greats3an",
+    author_email="greats3an@gmail.com",
+    description="",
+    long_description=long_description,
+    long_description_content_type="text/markdown",
+    url="https://github.com/mos9527/sssekai",
+    packages=setuptools.find_packages(),
+    classifiers=[
+        "Programming Language :: Python :: 3",
+        "License :: OSI Approved :: Apache Software License",
+        "Operating System :: OS Independent",
+    ],
+    install_requires=["msgpack","pycryptodome","unitypy","wannacri"],
+    entry_points={"console_scripts": ["sssekai = sssekai.__main__:__main__"]},
+    python_requires=">=3.8",
+)

sssekai-0.0.1/sssekai/__init__.py

@@ -0,0 +1,6 @@
+__VERSION_MAJOR__ = 0
+__VERSION_MINOR__ = 0
+__VERSION_PATCH__ = 1
+
+__version__ = '%s.%s.%s' % (__VERSION_MAJOR__,__VERSION_MINOR__,__VERSION_PATCH__)
+    

sssekai-0.0.1/sssekai/__main__.py

@@ -0,0 +1,77 @@
+import codecs
+import sys, os
+import argparse
+from sssekai.entrypoint.apidecrypt import main_apidecrypt
+from sssekai.entrypoint.abdecrypt import main_abdecrypt
+from sssekai.entrypoint.mitm import main_mitm
+from sssekai.entrypoint.usmdemux import main_usmdemux
+from sssekai.entrypoint.abcache import main_abcache, DEFAULT_CACHE_DIR
+from sssekai.entrypoint.live2dextract import main_live2dextract
+from sssekai.unity import SEKAI_UNITY_VERSION
+def __main__():
+    import coloredlogs
+    from logging import basicConfig
+    coloredlogs.install(
+            level='INFO',
+            fmt="%(asctime)s %(name)s [%(levelname).4s] %(message)s",
+            isatty=True,
+        )
+    basicConfig(
+        level='INFO', format="[%(levelname).4s] %(name)s %(message)s"
+    )
+    
+    parser = argparse.ArgumentParser(description='''SSSekai Proejct SEKAI feat. Hatsune Miku (Android) Modding Tools
+Installation:
+    pip install git+https://github.com/mos9527/sssekai                                    
+''', formatter_class=argparse.RawTextHelpFormatter)
+    subparsers = parser.add_subparsers(title='subcommands', description='valid subcommands', help='additional help')
+    # apidecrypt
+    apidecrypt_parser = subparsers.add_parser('apidecrypt', help='''API crypto dumper
+This crypto applies to:
+    - API request/response body dumped by packet sniffer (mitmproxy, wireshark, etc.)
+    - AssetBundleInfo (can be found at /sdcard/Android/data/com.hermes.mk.asia/files/data/AssetBundleInfo,or see sssekai.abcache)''')
+    apidecrypt_parser.add_argument('infile', type=str, help='input dump file')
+    apidecrypt_parser.add_argument('outfile', type=str, help='output json file')
+    apidecrypt_parser.set_defaults(func=main_apidecrypt)
+    # abdecrypt
+    abdecrypt_parser = subparsers.add_parser('abdecrypt', help='''Decrypt Sekai AssetBundle
+These can be found at /sdcard/Android/data/com.hermes.mk.asia/files/data/                                             
+''')
+    abdecrypt_parser.add_argument('indir', type=str, help='input directory')
+    abdecrypt_parser.add_argument('outdir', type=str, help='output directory')
+    abdecrypt_parser.set_defaults(func=main_abdecrypt)
+    # usmdemux
+    usmdemux_parser = subparsers.add_parser('usmdemux', help='''Demux Sekai USM Video in a AssetBundle''')
+    usmdemux_parser.add_argument('infile', type=str, help='input file')
+    usmdemux_parser.add_argument('outdir', type=str, help='output directory')
+    usmdemux_parser.set_defaults(func=main_usmdemux)
+    # abcache
+    abcache_parser = subparsers.add_parser('abcache', help='''Sekai AssetBundle local cache
+Downloads/Updates *ALL* PJSK TW assets to local devices.
+NOTE: The assets can take quite a lot of space (est. 27GB) so be prepared
+NOTE: The AssetBundles cached are NOT OBFUSCATED. They can be used as is by various Unity ripping tools (and sssekai by extension)
+      that supports stripped Unity version (should be %s).''' % SEKAI_UNITY_VERSION)
+    abcache_parser.add_argument('--cache-dir', type=str, help='cache directory',default=DEFAULT_CACHE_DIR)
+    abcache_parser.add_argument('--skip-update',action='store_true',help='skip all updates and use cached assets as is.')
+    abcache_parser.add_argument('--open', action='store_true',help='open cache directory. this will skip all updates.')
+    abcache_parser.set_defaults(func=main_abcache)
+    # live2dextract
+    live2dextract_parser = subparsers.add_parser('live2dextract', help='''Extract Sekai Live2D Models in a AssetBundle''')
+    live2dextract_parser.add_argument('infile', type=str, help='input file')
+    live2dextract_parser.add_argument('outdir', type=str, help='output directory')
+    live2dextract_parser.add_argument('--no-anim',action='store_true',help='don\'t extract animation clips')
+    live2dextract_parser.set_defaults(func=main_live2dextract)
+    # mitm
+    mitm_parser = subparsers.add_parser('mitm', help='Run Sekai API MITM proxy (WIP)')
+    mitm_parser.set_defaults(func=main_mitm)
+    # parse args
+    args = parser.parse_args()
+    if 'func' in args:
+        args.func(args)
+    else:
+        parser.print_help()
+
+
+if __name__ == "__main__":
+    __main__()
+    sys.exit(0)

sssekai-0.0.1/sssekai/abcache/__init__.py

@@ -0,0 +1,188 @@
+from os import path, remove, makedirs, stat
+from typing import List, Mapping
+from dataclasses import dataclass, asdict
+from requests import get as http_get, Session
+from logging import getLogger
+from json import dump, load
+from pathlib import Path
+from concurrent.futures import ThreadPoolExecutor
+logger = getLogger('sssekai.abcache')
+
+from sssekai.crypto.APIManager import decrypt
+from sssekai.crypto.AssetBundle import decrypt_headaer_inplace, SEKAI_AB_MAGIC
+from msgpack import unpackb
+from tqdm import tqdm
+DEFAULT_CACHE_DIR = '~/.sssekai/abcache'
+# 2.6.1
+SEKAI_CDN = 'https://lf16-mkovscdn-sg.bytedgame.com/obj/sf-game-alisg/gdl_app_5245/'
+SEKAI_AB_BASE_PATH = 'AssetBundle/2.6.0/Release/online/'
+SEKAI_AB_INDEX_PATH = SEKAI_AB_BASE_PATH + 'android21/AssetBundleInfo.json'
+
+class ThreadpoolDownloader(ThreadPoolExecutor):
+    session : Session
+    progress : tqdm
+
+    def download(self, url, fname, length):
+        RETRIES = 1
+        for _ in range(0,RETRIES):
+            try:
+                resp = self.session.get(url,stream=True)
+                resp.raise_for_status()
+                makedirs(path.dirname(fname),exist_ok=True)
+                with open(fname, 'wb') as f:
+                    magic = next(resp.iter_content(4))
+                    if magic == SEKAI_AB_MAGIC:
+                        header = next(resp.iter_content(128))            
+                        self.progress.update(128)
+                        f.write(decrypt_headaer_inplace(bytearray(header)))
+                    else:
+                        f.write(magic)                
+                    for chunk in resp.iter_content(65536):
+                        self.progress.update(len(chunk))
+                        f.write(chunk)
+                    return
+            except Exception as e:
+                logger.error('While downloading %s : %s. Retrying' % (url,e))
+        if _ == RETRIES - 1:
+            logger.critical('Did not download %s' % url)
+    def add_link(self, url, fname, length):
+        self.progress.total += length
+        return self.submit(self.download, url, fname, length)
+    def __init__(self) -> None:
+        self.session = Session()
+        self.progress = tqdm(bar_format="{desc}: {percentage:.1f}%|{bar}| {n_fmt}/{total_fmt} {rate_fmt} {elapsed}<{remaining}", total=0,unit='B', unit_scale=True, unit_divisor=1024,desc="Downloading")
+        super().__init__() # workers = processor count * 5
+
+class AbCacheConfig:
+    cache_dir : str # absolute path to cache directory
+    downloader : ThreadpoolDownloader
+    def __init__(self, downloader : ThreadpoolDownloader, cache_dir: str = DEFAULT_CACHE_DIR) -> None:
+        self.cache_dir = path.expanduser(cache_dir)
+        self.cache_dir = path.abspath(self.cache_dir)
+        self.downloader = downloader
+
+@dataclass
+class AbCacheEntry(dict):
+    bundleName: str
+    cacheFileName: str
+    cacheDirectoryName: str
+    hash: str
+    category: str
+    crc: int
+    fileSize:int
+    dependencies : List[str]
+    paths : List[str]
+    isBuiltin : bool
+    md5Hash : str
+    downloadPath : str
+
+    def up_to_date(self, config: AbCacheConfig, other) -> bool:
+        fsize = self.get_file_size(config)     
+        return self.hash == other.hash and self.fileSize <= fsize # fsize may be fileSize + 4
+    
+    def get_file_size(self, config: AbCacheConfig):
+        if self.get_file_exists(config):
+            return stat(self.get_file_path(config)).st_size
+        else:
+            return 0
+
+    def get_file_exists(self, config: AbCacheConfig) -> bool:
+        return path.exists(self.get_file_path(config))
+    
+    def get_file_path(self, config: AbCacheConfig) -> str:
+        return path.join(config.cache_dir, self.bundleName)
+@dataclass
+class AbCacheIndex(dict):
+    version : str
+    bundles : Mapping[str, AbCacheEntry]
+    def get_bundle_by_abcache_path(self, config: AbCacheConfig, fpath) -> AbCacheEntry | None:
+        # "C:\Users\mos9527\.sssekai\abcache\title_screen\anniversary_2nd_bg"
+        fpath = Path(fpath)
+        cpath = Path(config.cache_dir)
+        assert fpath.is_relative_to(cpath), "File provided is not inside the cache"
+        relpath = fpath.relative_to(cpath).as_posix()
+        return self.bundles.get(relpath,None)
+    
+class AbCache:
+    config : AbCacheConfig
+    index : AbCacheIndex
+    
+    @staticmethod
+    def download_cache_index() -> AbCacheIndex:
+        resp = http_get(
+            url=SEKAI_CDN + SEKAI_AB_INDEX_PATH,
+            headers={
+                'Accept-Encoding': 'deflate, gzip'
+            }
+        )
+        resp.raise_for_status()
+        data = decrypt(resp.content)
+        data = unpackb(data)
+        cache = AbCacheIndex(**data)
+        for k,v in cache.bundles.items():
+            cache.bundles[k] = AbCacheEntry(**v)
+        return cache
+
+    def update_cahce_entry(self, entry : AbCacheEntry, new_entry : AbCacheEntry = None):
+        if new_entry:
+            entry = new_entry
+        return self.config.downloader.add_link(
+            SEKAI_CDN + SEKAI_AB_BASE_PATH + entry.downloadPath,
+            entry.get_file_path(self.config),
+            entry.fileSize
+        )
+                
+    def update_cahce_index(self):
+        logger.info('Downloading AssetBundleInfo.json')
+        dl = AbCache.download_cache_index()
+        all_keys = sorted(list({k for k in dl.bundles.keys()}.union({k for k in self.index.bundles.keys()})))
+        update_count = 0
+        for k in all_keys:
+            if k in dl.bundles and k in self.index.bundles:
+                # update
+                if not dl.bundles[k].up_to_date(self.config, self.index.bundles[k]):
+                    logger.info('Updating bundle %s', k)
+                    self.update_cahce_entry(self.index.bundles[k], dl.bundles[k])
+                    update_count+=1
+                else:
+                    logger.debug('Bundle %s is up to date' % k)
+                    pass
+            elif k in dl.bundles: 
+                # append
+                logger.info('Adding bundle %s', k)
+                self.index.bundles[k] = dl.bundles[k]
+                self.update_cahce_entry(self.index.bundles[k])
+                update_count+=1
+            else: 
+                # removal
+                logger.info('Removing bundle %s', k)
+                if path.exists(self.index.bundles[k].get_file_path(self.config)):
+                    remove(self.index.bundles[k].get_file_path(self.config))
+                del self.index.bundles[k]
+        logger.info('Saving AssetBundle index')
+        self.save()
+        logger.info('...Saved. Need %d updates' % update_count)
+        self.config.downloader.shutdown(wait=True)
+        logger.info('AssetBundles are now up-to-date')
+    
+    def __init__(self, config : AbCacheConfig) -> None:
+        self.config = config
+        makedirs(self.config.cache_dir,exist_ok=True)
+        try:
+            self.load()
+        except Exception as e:
+            logger.warning('Failed to load cache index. Creating new one. (%s)' % e)
+            self.index = AbCacheIndex(None, {})
+            self.save()
+    
+    def save(self):
+        with open(path.join(self.config.cache_dir, 'abindex.json'),'w',encoding='utf-8') as f:
+            dump(asdict(self.index), f, indent=4, ensure_ascii=False)
+            logger.info("Saved %d entries to cache index" % len(self.index.bundles))
+
+    def load(self):
+        with open(path.join(self.config.cache_dir, 'abindex.json'),'r',encoding='utf-8') as f:
+            self.index = AbCacheIndex(**load(f))
+            for k,v in self.index.bundles.items():
+                self.index.bundles[k] = AbCacheEntry(**v)
+            logger.info("Loaded %d entries from cache index" % len(self.index.bundles))

sssekai-0.0.1/sssekai/crypto/APIManager.py

@@ -0,0 +1,24 @@
+from Crypto.Cipher import AES
+def PKCS7_pad(data : bytes,bs) -> bytes:
+    size = (bs - len(data) % bs)
+    return data + bytes([size] * size)
+
+def PKCS7_unpad(data : bytes,bs) -> bytes:
+    pad = data[-1]
+    return data[:-pad]
+
+def decrypt_aes_cbc(data : bytes, key, iv, unpad=PKCS7_unpad) -> bytes:
+    cipher = AES.new(key, AES.MODE_CBC, iv)
+    return unpad(cipher.decrypt(data), cipher.block_size)
+
+def encrypt_aes_cbc(data : bytes, key, iv, pad=PKCS7_pad) -> bytes:
+    cipher = AES.new(key, AES.MODE_CBC, iv)
+    return cipher.encrypt(pad(data, cipher.block_size))
+
+SEKAI_APIMANAGER_KEY = b"g2fcC0ZczN9MTJ61"
+SEKAI_APIMANAGER_IV = b"msx3IV0i9XE5uYZ1"
+def encrypt(data : bytes) -> bytes:
+    return encrypt_aes_cbc(data,SEKAI_APIMANAGER_KEY,SEKAI_APIMANAGER_IV, PKCS7_pad)
+
+def decrypt(data : bytes) -> bytes:
+    return decrypt_aes_cbc(data,SEKAI_APIMANAGER_KEY,SEKAI_APIMANAGER_IV, PKCS7_unpad)

sssekai-0.0.1/sssekai/crypto/AssetBundle.py

@@ -0,0 +1,32 @@
+from io import BytesIO
+from typing import BinaryIO
+
+SEKAI_AB_MAGIC = b'\x10\x00\x00\x00'
+def decrypt_headaer_inplace(header : bytearray):
+    assert len(header) == 128
+    for i in range(0,128,8):       
+        for j in range(5):
+            header[i + j] = ~header[i + j] & 0xff
+    return header
+
+def decrypt(fin : BinaryIO, fout : BinaryIO = None) -> None:      
+    if not fout:
+        fout = BytesIO()
+    magic = fin.read(4)    
+    if magic == SEKAI_AB_MAGIC:
+        fout.write(decrypt_headaer_inplace(bytearray(fin.read(128))))
+        while (block := fin.read(65536)):
+            fout.write(block)    
+    else:
+        fin.seek(0)
+        while (block := fin.read(65536)):
+            fout.write(block)
+    return fout
+
+def encrypt(fin, fout):
+    raise NotImplementedError
+
+def has_magic(fin : BinaryIO) -> bool:
+    magic = fin.read(4)
+    fin.seek(0)
+    return magic == SEKAI_AB_MAGIC

sssekai-0.0.1/sssekai/entrypoint/abcache.py

@@ -0,0 +1,15 @@
+from sssekai.abcache import AbCache, AbCacheConfig,ThreadpoolDownloader, logger, DEFAULT_CACHE_DIR
+def main_abcache(args):
+    downloader = ThreadpoolDownloader()
+    cache_dir = args.cache_dir or DEFAULT_CACHE_DIR
+    config = AbCacheConfig(downloader, cache_dir)
+    if args.open:
+        from os import startfile
+        startfile(config.cache_dir)
+        return
+    cache = AbCache(config)
+    if not args.skip_update:
+        cache.update_cahce_index()
+    downloader.shutdown()
+    cache.save()
+    logger.info('AssetBundle cache is now ready. Visit: %s' % cache_dir)

sssekai-0.0.1/sssekai/entrypoint/abdecrypt.py

@@ -0,0 +1,9 @@
+import os
+def main_abdecrypt(args):
+    from sssekai.crypto.AssetBundle import decrypt
+    tree = os.walk(args.indir)
+    for root, dirs, files in tree:
+        for fname in files:
+            file = os.path.join(root,fname)
+            if (os.path.isfile(file)):
+                decrypt(open(file,'rb'), open(os.path.join(args.outdir, fname),'wb'))

sssekai-0.0.1/sssekai/entrypoint/apidecrypt.py

@@ -0,0 +1,9 @@
+import os
+def main_apidecrypt(args):
+    from sssekai.crypto.APIManager import decrypt
+    import json, msgpack
+    data = open(args.infile,'rb').read()
+    with open(args.outfile,'w',encoding='utf-8') as fout:
+        plain = decrypt(data)
+        msg = msgpack.unpackb(plain)
+        json.dump(msg, fout, indent=4, ensure_ascii=False)