nnnotes 0.1.0__py3-none-any.whl

nnnotes/liveui.py

@@ -0,0 +1,328 @@
+"""Live start canvas (Live/UILiveStartCanvas, LightWeight mode) -> <live>/liveui/ (liveui.json, textures/).
+
+livescene/scene.json already carries the canvas hierarchy with every component (serialized TMP texts, inline
+sprite records, the start timeline and its UILiveStartCanvasTrack clips) and livescene/shaders the shaders it
+draws with. This module adds what a renderer needs beyond that to draw the canvas the way
+UILiveStartCanvas.Initialize leaves it:
+
+  strings     the texts Initialize sets: music title, singer, the credit lines (LocalizeText master text id +
+              replacement word), music level, high score, resolved as the game resolves them
+  texts       per TMP text of the drawn part: the LocalizeText font / material / line spacing overrides; with
+              fonts "open" also its text record (textstyle: layout, colours, style values of its material)
+  fonts       fonts "game": TMP font assets reduced to the characters shown (tmpfont); FZLTH's runtime glyphs
+              generated from its source font; VibeMOPro's glyph pair adjustment lookup (kerning). fonts "open"
+              (default): the font asset names only, no font data and no atlases
+  materials   fonts "game": the localized TMP materials ("<font> - Default") and the fonts' default materials;
+              fonts "open": none
+  sprites     WhiteRect, Songtitle_base, the Difficulty_* icons of _difficultyPairs and the music jacket; their
+              texel blocks copied into textures/ (export.TexelPacker, deferred-texture mode)
+  controller  the root Animator's controller (evidence only: the start timeline drives the canvas)
+
+Texts and fonts follow the client language `language` (a catalog language code, languages.LANGUAGES): its
+LanguageMode selects the text table column, the LocalizeManager fonts and the line spacing.
+"""
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from .catalog import Catalog
+from .export import Exporter, TexelPacker, _safe
+from .jsonio import write_json
+from .player import PlayerData
+from .score import master_table
+from . import languages, textstyle, tmpfont
+from .textstyle import LANGUAGE_LINE_SPACING
+
+SCENE_KEY = "EmbScene/Live"
+CANVAS = "Live/UILiveStartCanvas"
+CONTENT = f"{CANVAS}/root/UISafeArea/ContentArea"
+# Parts drawn in LightWeight mode: root/base (a _simpleModeOnlyObjects entry), center_bottom and
+# simple_music_jacket_panel (left_top is _normalModeOnlyObjects, right_top/battle_panel is _battleLiveGameObjects).
+DRAWN = (f"{CANVAS}/root/base", f"{CONTENT}/center_bottom", f"{CONTENT}/simple_music_jacket_panel")
+JACKET_KEY = "Image/Jacket/{jacket}"
+# App.LiveBase.LiveDifficulty
+LIVE_DIFFICULTY = {"easy": 0, "normal": 1, "hard": 2, "expert": 3, "master": 4}
+TMP_STUBS = ("TMP_FontAsset", "TMP_SpriteAsset", "TMP_StyleSheet")
+TMP_CLASS = "TextMeshProUGUI"
+JAPANESE_COLUMN = languages.column("ja")
+FONT_CHOICES = ("open", "game")
+FONT_NOTE = "not exported (fonts \"open\"): the text record in `texts` describes the text without font data"
+
+
+def _drawn(path: str) -> bool:
+    return any(path == d or path.startswith(d + "/") for d in DRAWN)
+
+
+def _comp(node: dict, cls: str) -> dict | None:
+    hit = [c for c in node["components"] if c.get("class", c["type"]) == cls]
+    if len(hit) > 1:
+        raise RuntimeError(f"{node['path']}: {len(hit)} {cls}")
+    return hit[0] if hit else None
+
+
+def resolve_strings(master: Path, music_id: int, difficulty: str, column: str) -> tuple[dict, dict]:
+    """The strings UILiveStartCanvas.Initialize sets (IReadOnlyLiveMusicScore slots 2..8), for a fresh profile, with
+    the text table column `column` of the current LanguageMode. -> (strings, sources)."""
+    music = next(r for r in master_table(master, "MasterLiveMusic") if r["_id"] == music_id)
+    text = {r["_id"]: r for r in master_table(master, "MasterText")}
+
+    def localized(tid: str) -> str:
+        # MasterLoader.GetLocalizedText in the current LanguageMode: the column as it is, "" included (no
+        # fallback language); LocalizedTextTokenResolver.Resolve returns an empty text unchanged
+        return text[tid][column]
+
+    def word(tid: str) -> str:
+        # MasterLiveMusic.get_LyricistText / get_ComposerText / get_ArrangerText
+        # -> LocalizeManager.GetLocalizedText: an empty id is returned as it is, else the loader's text
+        return localized(tid) if tid else tid
+
+    def credit(label_id: str, word_id: str) -> str:
+        # LocalizeText.RegisterReplacementWord("{0}", word) stores the word and re-runs SetMasterTextId: an empty
+        # label text goes to the text as it is, otherwise String.Replace(key, word) for each registered word
+        # (ordinal, every occurrence; an empty word leaves "編曲:"). LocalizedTextTokenResolver.Resolve knows no
+        # "{0}" token. Initialize and FitUI test no text for emptiness.
+        label = localized(label_id)
+        return label.replace("{0}", word(word_id)) if label else label
+
+    # SongTitleDisplaySetting._useTranslation false (option 999 unset, GetBool default false):
+    # MasterLiveMusic.TitleText = loader.Get(_titleTextID, Japanese)
+    title = text[music["_titleTextID"]][JAPANESE_COLUMN]
+    # GetBandNameTextId: _bandNameTextID, else MasterBand(_bandIDs[0])._nameTextID
+    band_tid = music["_bandNameTextID"]
+    if not band_tid:
+        band = next(r for r in master_table(master, "MasterBand") if r["_id"] == music["_bandIDs"][0])
+        band_tid = band["_nameTextID"]
+    score_id = music[f"_{difficulty}ID"]
+    score = next(r for r in master_table(master, "MasterLiveMusicScore") if r["_id"] == score_id)
+    strings = {
+        "musicTitle": title,
+        "singerName": localized(band_tid),
+        "lyricsWriter": credit("ui_credit_lyrics", music["_lyricistTextID"]),
+        "composer": credit("ui_credit_composer", music["_composerTextID"]),
+        "arranger": credit("ui_credit_arranger", music["_arrangerTextID"]),
+        # MasterLiveMusicScore.get_MusicScoreLevel (_musicScoreLevel), UIText.SetText(int)
+        "musicLevel": str(int(score["_musicScoreLevel"])),
+        # Player.GetLiveMusicResult: a fresh profile has no entry -> new LiveMusicResult(id), get_HighScore = 0.
+        # The IFix patch checks inside these getters (patch ids 0x13e0, 0x1b5b) depend on patches loaded at run
+        # time; the code assumes none is loaded
+        "highScore": "0",
+    }
+    sources = {
+        "musicTitle": {"textId": music["_titleTextID"], "column": JAPANESE_COLUMN,
+                       "why": "SongTitleDisplaySetting._useTranslation false (option 999 default)"},
+        "singerName": {"textId": band_tid, "column": column},
+        "lyricsWriter": {"label": "ui_credit_lyrics", "word": music["_lyricistTextID"], "column": column},
+        "composer": {"label": "ui_credit_composer", "word": music["_composerTextID"], "column": column},
+        "arranger": {"label": "ui_credit_arranger", "word": music["_arrangerTextID"], "column": column},
+        "musicLevel": {"MasterLiveMusicScore": score_id, "field": "_musicScoreLevel"},
+        "highScore": {"why": "fresh profile: no LiveMusicResult entry, HighScore 0"},
+    }
+    for key in ("lyricsWriter", "composer", "arranger"):
+        if not word(sources[key]["word"]):
+            sources[key]["emptyWord"] = True        # the label is shown without the word
+    return strings, sources
+
+
+# UILiveStartCanvas field -> strings key (UIText fields and arrays set by Initialize)
+TEXT_FIELDS = {"_musicTitleText": "musicTitle", "_singerText": "singerName", "_lyricistText": "lyricsWriter",
+               "_compositterText": "composer", "_compositterText2": "composer", "_arrangerText": "arranger",
+               "_musicLevelTexts": "musicLevel", "_highScoreTexts": "highScore"}
+
+
+def _game_fonts(ex: Exporter, fontset, texts: dict, chars_by_font: dict, packer: TexelPacker):
+    """The game's TMP fonts of the texts, reduced to the characters shown (tmpfont), their atlas blocks added to
+    `packer`. -> (glyph coverage, materials, fonts)."""
+    coverage: dict[str, dict] = {}
+    needed: dict[str, set] = {}
+    runtime_units: dict[str, set] = {}
+    primaries = set()
+    for fname in sorted(chars_by_font):
+        primary = fontset.by_key(textstyle.font_dir(fname) + fname)
+        primaries.add(primary.name)
+        cov, nd, rt = tmpfont.glyph_coverage(fontset, primary, sorted(chars_by_font[fname]))
+        if cov["missing"]:
+            raise RuntimeError(f"{fname}: characters {cov['missing']} missing from the font chain")
+        coverage[fname] = cov
+        for k, v in nd.items():
+            needed.setdefault(k, set()).update(v)
+        for k, v in rt.items():
+            runtime_units.setdefault(k, set()).update(v)
+    materials, text_mats = tmpfont.text_materials(ex, fontset, [t["localized"] for t in texts.values()],
+                                                  primaries, needed, runtime_units)
+    font_out = tmpfont.export_fonts(ex, fontset, primaries, needed, runtime_units, text_mats, packer,
+                                    "characters shown by the live start canvas (plus baked control characters); "
+                                    "runtime characters generated from the source font (runtimeGlyphs)")
+    return coverage, materials, font_out
+
+
+def extract(cat: Catalog, player: PlayerData, out_dir: Path, master: Path, music_id: int,
+            difficulty: str, *, language: str, fonts: str = "open") -> dict:
+    """Write <out_dir>/liveui/ (liveui.json, textures/) for one live and return a summary. Needs
+    <out_dir>/livescene/shaders/shaders.json (livescene.extract) for the shader check. `language`: the client
+    language (a languages.LANGUAGES code); `fonts`: "open" (text records, font names only) or "game" (the game's
+    TMP fonts reduced to the characters shown; needs the `fonts` extra)."""
+    out_dir = Path(out_dir)
+    base = out_dir / "liveui"
+    base.mkdir(parents=True, exist_ok=True)
+    if difficulty not in LIVE_DIFFICULTY:
+        raise KeyError(f"difficulty {difficulty}")
+    if fonts not in FONT_CHOICES:
+        raise ValueError(f"fonts {fonts}: one of {', '.join(FONT_CHOICES)}")
+    mode, column = languages.mode(language), languages.column(language)
+    ex = Exporter(cat, base, player=player, textures="deferred", stub_assets=TMP_STUBS,
+                  follow=("AnimatorController",))
+    packer = TexelPacker(base)
+
+    # -- the canvas subtree of the scene (deferred: sprite / texture references) -------------------------
+    env, graph = ex.load(SCENE_KEY)
+    roots = [p for p in graph.tf if graph.path(p) == CANVAS]
+    if len(roots) != 1:
+        raise RuntimeError(f"{CANVAS}: {len(roots)} transforms")
+    nodes = ex.hierarchy(env, graph, roots[0])
+    by_path: dict[str, list] = {}
+    for n in nodes:
+        by_path.setdefault(n["path"], []).append(n)
+    canvas = _comp(by_path[CANVAS][0], "UILiveStartCanvas")
+    if canvas is None:
+        raise RuntimeError(f"{CANVAS}: no UILiveStartCanvas")
+
+    # -- strings, and which texts Initialize writes them to ---------------------------------------------
+    strings, sources = resolve_strings(master, music_id, difficulty, column)
+    text_of: dict[str, str] = {}
+    for field, key in TEXT_FIELDS.items():
+        refs = canvas[field] if isinstance(canvas[field], list) else [canvas[field]]
+        for r in refs:
+            text_of[r["gameObject"]] = strings[key]
+
+    # -- TMP texts of the drawn part: LocalizeText overrides and the characters they show -------------
+    styles = textstyle.TextStyles(ex, player, mode) if fonts == "open" else None
+    lang = styles.lang if styles else textstyle.language_fonts(player)
+    swap = styles.swap if styles else textstyle.font_swap(lang, mode)
+    fontset = tmpfont.FontSet(ex) if fonts == "game" else None
+    texts: dict[str, dict] = {}
+    chars_by_font: dict[str, set] = {}
+    for n in nodes:
+        if not _drawn(n["path"]):
+            continue
+        t = _comp(n, TMP_CLASS)
+        if t is None:
+            continue
+        if len(by_path[n["path"]]) != 1:
+            raise RuntimeError(f"{n['path']}: text node path not unique")
+        lt = _comp(n, "LocalizeText")
+        if not (lt and lt["m_Enabled"] and lt["_localizeEnabled"]):
+            raise NotImplementedError(f"{n['path']}: text without an enabled LocalizeText")
+        loc = textstyle.localize_text(n["path"], t["m_fontAsset"]["name"], t["m_sharedMaterial"]["material"],
+                                      swap, lang, mode)
+        shown = text_of.get(n["path"], t["m_text"])
+        texts[n["path"]] = {"localized": loc, "text": shown, "setByInitialize": n["path"] in text_of}
+        if styles:
+            record = styles.record(n["path"], t, lt)
+            record.pop("text")                        # the serialized text; `text` above is the one shown
+            texts[n["path"]]["style"] = record
+        chars_by_font.setdefault(loc["fontAsset"], set()).update(ord(ch) for ch in shown)
+    for p in text_of:
+        if _drawn(p) and p not in texts:
+            raise RuntimeError(f"{p}: Initialize writes a text that is not exported")
+
+    if fontset is not None:
+        coverage, materials, font_out = _game_fonts(ex, fontset, texts, chars_by_font, packer)
+    else:                                          # name stubs: no font data, no atlases
+        coverage, materials = None, {}
+        font_out = {f: {"name": f, "note": FONT_NOTE} for f in sorted(chars_by_font)}
+
+    # -- sprites: drawn Images, the difficulty icons, the jacket ----------------------------------------
+    replaced = {r["gameObject"] for f in ("_jacketImages", "_difficultyImages") for r in canvas[f]}
+    sprite_refs: dict[str, str] = {}               # sprite name -> spriteRef
+    for n in nodes:
+        im = _comp(n, "Image")
+        if not _drawn(n["path"]) or im is None or n["path"] in replaced or not im["m_Sprite"]:
+            continue
+        sprite_refs.setdefault(im["m_Sprite"]["name"], im["m_Sprite"]["spriteRef"])
+    difficulty_icons = {}
+    for p in canvas["_difficultyPairs"]:
+        s = p["Icon"]
+        difficulty_icons[str(p["Difficulty"])] = s["name"]
+        sprite_refs.setdefault(s["name"], s["spriteRef"])
+    music = next(r for r in master_table(master, "MasterLiveMusic") if r["_id"] == music_id)
+    jacket = ex.key_sprite(JACKET_KEY.format(jacket=music["_jacketAssetName"]))
+    sprite_refs.setdefault(jacket["name"], jacket["spriteRef"])
+    for name, ref in sorted(sprite_refs.items()):
+        s = ex.sprite_recs[ref]
+        if s["name"] != name:
+            raise RuntimeError(f"sprite {name}: record named {s['name']}")
+        ex.pack_sprite(ref, packer, f"sprites_{_safe(s['tex'].read_typetree()['m_Name'])}")
+
+    textures = packer.build()
+    if fontset is not None:
+        tmpfont.resolve_font_blocks(font_out)
+    sprites_out = {name: ex.packed_sprite(ref) for name, ref in sorted(sprite_refs.items())}
+    for m in materials.values():
+        for v in m["textures"].values():
+            if v["texture"] is not None:
+                v["texture"] = {"name": v["texture"]["name"], "width": v["texture"]["width"],
+                                "height": v["texture"]["height"], "format": v["texture"]["format"]}
+
+    # -- shaders: livescene's dump (same name and source bundle), GLES3 variant per material ------------
+    index = json.loads((out_dir / "livescene" / "shaders" / "shaders.json").read_text(encoding="utf-8"))
+    by_name = {r["name"]: r for r in index}
+    keywords = {}
+    for m in materials.values():
+        sh = m["shader"]["shader"]
+        rec = by_name.get(sh)
+        if rec is None:
+            raise RuntimeError(f"{m['material']}: shader {sh} not in livescene/shaders")
+        if sh not in ex.shaders or ex.shaders[sh].assets_file.name != rec["source"]:
+            raise RuntimeError(f"{m['material']}: shader {sh} comes from another bundle than livescene's")
+        known = {k for v in rec["variants"] if v["platform"] == "gles3" and v["type"] == "GLES3" for k in v["keywords"]}
+        want = sorted(k for k in m["keywords"] if k in known)
+        if not any(v["platform"] == "gles3" and v["type"] == "GLES3" and v["stage"] == "vertex"
+                   and sorted(v["keywords"]) == want for v in rec["variants"]):
+            raise RuntimeError(f"{sh}: no GLES3 variant for material {m['material']} keywords {want}")
+        keywords[m["material"]] = want
+    if "UI/Default" not in by_name:
+        raise RuntimeError("UI/Default not in livescene/shaders")
+
+    # -- the root Animator's controller (evidence) -----------------------------------------------------
+    an = _comp(by_path[f"{CANVAS}/root"][0], "Animator")
+    controller = an["m_Controller"] if an else None
+    tmp_settings = player.names_in(player.resource("TMP Settings"), player.mono(player.resource("TMP Settings")))
+
+    doc = {
+        "about": f"Live start canvas (Live/UILiveStartCanvas, LightWeight mode): what livescene/scene.json lacks, "
+                 f"{language}",
+        "canvas": CANVAS,
+        "slice": {"musicId": music_id, "difficulty": difficulty, "liveDifficulty": LIVE_DIFFICULTY[difficulty],
+                  "lightweight": True, "profile": "fresh (no play records)"},
+        "language": {"mode": mode, "field": column[1:], "fonts": lang, "fontSwap": swap,
+                     "lineSpacing": LANGUAGE_LINE_SPACING[mode]},
+        "strings": strings,
+        "stringSources": sources,
+        "texts": texts,
+        "sprites": sprites_out,
+        "difficultyIcons": difficulty_icons,
+        "jacket": jacket["name"],
+        "textures": textures,
+        "materials": materials,
+        "materialKeywords": keywords,
+        "fonts": font_out,
+        "glyphCoverage": coverage,
+        "tmpSettings": {k: tmp_settings[k] for k in ("m_GetFontFeaturesAtRuntime", "m_matchMaterialPreset",
+                                                     "m_enableExtraPadding", "m_missingGlyphCharacter")},
+        # images draw with Graphic.defaultGraphicMaterial (built-in UI/Default: livescene's dump of it has the
+        # same variants as the unity_builtin_extra one the ADV exports, checked byte for byte)
+        "shaders": {"index": "../livescene/shaders/shaders.json",
+                    "names": sorted({m["shader"]["shader"] for m in materials.values()} | {"UI/Default"})},
+        "controller": controller,
+    }
+    if styles:
+        doc["fontData"] = "open"
+        doc["textStyle"] = styles.summary()
+    write_json(base / "liveui.json", doc)
+    return {"liveui": str(base / "liveui.json"), "texts": len(texts), "sprites": sorted(sprites_out),
+            "textures": {k: (v["width"], v["height"]) for k, v in textures.items()},
+            "fonts": sorted(font_out), "coverage": {k: v["counts"] for k, v in (coverage or {}).items()},
+            "strings": strings,
+            "runtimeGlyphs": {k: len(v["runtimeGlyphs"]["generated"]) for k, v in font_out.items()
+                              if v.get("runtimeGlyphs")}}

nnnotes/master.py

@@ -0,0 +1,251 @@
+"""Master data files -> JSON tables.
+
+A master data file (`<Table>.bin`) is a 64-byte prefix followed by the table encrypted with Rijndael (256-bit
+block, 256-bit key, CBC, PKCS7 padding); the plaintext is gzip-compressed UTF-8 JSON (`{"_allData": [rows]}`).
+The game reads it in MasterDataModelBox<,>.Load: RijndaelEncryption.FixedDecrypt (the data after the prefix,
+the CryptProvider key and IV), then Compression.Decompress (GZipStream), then JsonUtility.FromJson.
+
+AES is Rijndael with 128-bit blocks only, so the 256-bit block cipher is implemented here (numpy, every block of a
+file at once). Key and IV come from the configuration (`[master] key`, `[master] iv`).
+
+`download` fetches a master data version from the region's CDN: `<cdn>/master/<version>/MasterManifest.json`
+(`{"version", "files": [{"name", "hash", "size"}]}`) and the files it lists, each checked against its SHA-256.
+The version a region serves now comes from the game's API (`gameapi.master_version`).
+"""
+from __future__ import annotations
+
+import gzip
+import hashlib
+import json
+import os
+import time
+import urllib.error
+import urllib.request
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import dataclass, field
+from pathlib import Path
+
+import numpy as np
+
+from . import jsonio
+
+PREFIX = 64                     # bytes before the ciphertext
+BLOCK = 32                      # 256-bit block
+NB = NK = 8                     # block / key length in 32-bit words
+NR = 14                         # rounds
+
+
+@dataclass(frozen=True)
+class MasterKey:
+    """Key and IV of the master data encryption (never shown in a repr)."""
+    key: bytes = field(repr=False)
+    iv: bytes = field(repr=False)
+
+    def __post_init__(self):
+        if len(self.key) != 32 or len(self.iv) != 32:
+            raise ValueError("master key and IV must be 32 bytes each")
+
+
+# ---------------------------------------------------------------- Rijndael-256
+def _xtime(a: int) -> int:
+    a <<= 1
+    return (a ^ 0x1B) & 0xFF if a & 0x100 else a
+
+
+def _gmul(a: int, b: int) -> int:
+    r = 0
+    while b:
+        if b & 1:
+            r ^= a
+        a = _xtime(a)
+        b >>= 1
+    return r
+
+
+def _sbox() -> list[int]:
+    """The Rijndael S-box (multiplicative inverse in GF(2^8) followed by the affine map)."""
+    box = [0] * 256
+    p = q = 1
+    while True:
+        p = p ^ ((p << 1) & 0xFF) ^ (0x1B if p & 0x80 else 0)
+        q ^= (q << 1) & 0xFF
+        q ^= (q << 2) & 0xFF
+        q ^= (q << 4) & 0xFF
+        if q & 0x80:
+            q ^= 0x09
+        x = q ^ ((q << 1 | q >> 7) & 0xFF) ^ ((q << 2 | q >> 6) & 0xFF) \
+            ^ ((q << 3 | q >> 5) & 0xFF) ^ ((q << 4 | q >> 4) & 0xFF)
+        box[p] = (x ^ 0x63) & 0xFF
+        if p == 1:
+            break
+    box[0] = 0x63
+    return box
+
+
+SBOX = _sbox()
+INV_SBOX = np.zeros(256, dtype=np.uint8)
+INV_SBOX[np.array(SBOX)] = np.arange(256, dtype=np.uint8)
+MUL = {k: np.array([_gmul(i, k) for i in range(256)], dtype=np.uint8) for k in (9, 11, 13, 14)}
+INV_SHIFT = (0, 1, 3, 4)        # InvShiftRows: row r moves right by INV_SHIFT[r] columns (Nb = 8)
+
+
+def round_keys(key: bytes) -> np.ndarray:
+    """Key expansion (Nk = 8) -> (NR + 1, 4 rows, NB columns) uint8."""
+    if len(key) != 32:
+        raise ValueError("key must be 32 bytes")
+    words = [list(key[4 * i:4 * i + 4]) for i in range(NK)]
+    rcon = 1
+    for i in range(NK, NB * (NR + 1)):
+        t = list(words[i - 1])
+        if i % NK == 0:
+            t = t[1:] + t[:1]
+            t = [SBOX[b] for b in t]
+            t[0] ^= rcon
+            rcon = _xtime(rcon)
+        elif i % NK == 4:
+            t = [SBOX[b] for b in t]
+        words.append([a ^ b for a, b in zip(words[i - NK], t)])
+    return np.array(words, dtype=np.uint8).reshape(NR + 1, NB, 4).transpose(0, 2, 1)
+
+
+def decrypt_blocks(ct: np.ndarray, rk: np.ndarray) -> np.ndarray:
+    """(n, 32) ciphertext blocks -> (n, 32) block decryptions (no CBC chaining). A block's bytes are its state
+    column by column."""
+    s = ct.reshape(-1, NB, 4).transpose(0, 2, 1).copy()          # (n, 4, NB): s[r][c]
+    s ^= rk[NR]
+    cols = np.arange(NB)
+    for rnd in range(NR - 1, -1, -1):
+        for r in range(4):                                       # InvShiftRows
+            s[:, r, :] = s[:, r, (cols - INV_SHIFT[r]) % NB]
+        s = INV_SBOX[s]                                          # InvSubBytes
+        s ^= rk[rnd]                                             # AddRoundKey
+        if rnd:                                                  # InvMixColumns
+            a0, a1, a2, a3 = s[:, 0, :], s[:, 1, :], s[:, 2, :], s[:, 3, :]
+            s = np.stack([
+                MUL[14][a0] ^ MUL[11][a1] ^ MUL[13][a2] ^ MUL[9][a3],
+                MUL[9][a0] ^ MUL[14][a1] ^ MUL[11][a2] ^ MUL[13][a3],
+                MUL[13][a0] ^ MUL[9][a1] ^ MUL[14][a2] ^ MUL[11][a3],
+                MUL[11][a0] ^ MUL[13][a1] ^ MUL[9][a2] ^ MUL[14][a3],
+            ], axis=1)
+    return s.transpose(0, 2, 1).reshape(-1, BLOCK)
+
+
+def decrypt_cbc(data: bytes, key: bytes, iv: bytes, rk: np.ndarray | None = None) -> bytes:
+    """Rijndael-256 CBC decryption, padding kept."""
+    if not data or len(data) % BLOCK:
+        raise ValueError(f"ciphertext length {len(data)} is not a positive multiple of {BLOCK}")
+    ct = np.frombuffer(data, dtype=np.uint8).reshape(-1, BLOCK)
+    pt = decrypt_blocks(ct, round_keys(key) if rk is None else rk)
+    pt ^= np.vstack([np.frombuffer(iv, dtype=np.uint8)[None, :], ct[:-1]])
+    return pt.tobytes()
+
+
+def decode(raw: bytes, key: MasterKey, rk: np.ndarray | None = None) -> bytes:
+    """A master data file as stored -> its JSON text (UTF-8 bytes)."""
+    out = decrypt_cbc(raw[PREFIX:], key.key, key.iv, rk)
+    pad = out[-1]
+    if not 1 <= pad <= BLOCK or out[-pad:] != bytes([pad]) * pad:
+        raise ValueError("bad PKCS7 padding (wrong key or IV, or not a master data file)")
+    return gzip.decompress(out[:-pad])
+
+
+def table_json(text: bytes) -> str:
+    """The stored form of a decoded table: parsed and written again by jsonio.dumps (UTF-8, one-space indent,
+    non-finite numbers as 1e999 / -1e999; a NaN raises ValueError naming where it is)."""
+    return jsonio.dumps(json.loads(text.decode("utf-8")), ensure_ascii=False, indent=1)
+
+
+def decode_files(files: list[Path], out_dir: Path, key: MasterKey, workers: int = 8) -> dict:
+    """Decode master data files into <out_dir>/<stem>.json. Returns {decoded, failed: [{file, error}]}."""
+    out_dir = Path(out_dir)
+    out_dir.mkdir(parents=True, exist_ok=True)
+    rk = round_keys(key.key)
+
+    def one(p: Path):
+        try:
+            text = table_json(decode(p.read_bytes(), key, rk))
+        except Exception as e:                       # report and continue with the other tables
+            return {"file": p.name, "error": f"{type(e).__name__}: {e}"}
+        (out_dir / f"{p.stem}.json").write_text(text, encoding="utf-8", newline="\n")
+        return None
+
+    with ThreadPoolExecutor(max(1, workers)) as ex:
+        failed = [r for r in ex.map(one, files) if r is not None]
+    return {"decoded": len(files) - len(failed), "failed": failed, "out": str(out_dir)}
+
+
+def table(master_dir: Path, name: str) -> list[dict]:
+    """The rows of a decoded master table: `_allData` of <master_dir>/<name>.json."""
+    return json.loads((Path(master_dir) / f"{name}.json").read_text(encoding="utf-8"))["_allData"]
+
+
+def has_row(master_dir: Path, name: str, row_id) -> bool:
+    """Whether the decoded master table `name` has a row whose `_id` is `row_id`."""
+    return any(r.get("_id") == row_id for r in table(master_dir, name))
+
+
+def input_files(inputs: list[Path]) -> list[Path]:
+    """Files given, and the `*.bin` files of directories given, sorted by name per directory."""
+    out: list[Path] = []
+    for p in map(Path, inputs):
+        if p.is_dir():
+            out += sorted(f for f in p.iterdir() if f.is_file() and f.suffix.lower() == ".bin")
+        elif p.is_file():
+            out.append(p)
+        else:
+            raise FileNotFoundError(p)
+    return out
+
+
+# ---------------------------------------------------------------- download
+class DownloadError(RuntimeError):
+    """A CDN download failed (the message names the file, not the URL)."""
+
+
+def _get(url: str, retries: int = 3, timeout: int = 60) -> bytes:
+    name = url.rsplit("/", 1)[-1]
+    for i in range(retries):
+        try:
+            with urllib.request.urlopen(url, timeout=timeout) as r:
+                return r.read()
+        except urllib.error.HTTPError as e:
+            if e.code < 500 or i == retries - 1:
+                raise DownloadError(f"download of {name} failed: HTTP {e.code}") from None
+        except (urllib.error.URLError, TimeoutError, OSError) as e:
+            if i == retries - 1:
+                raise DownloadError(f"download of {name} failed: {type(e).__name__}") from None
+        time.sleep(0.5 * (i + 1))
+    raise AssertionError("unreachable")
+
+
+def download(cdn: str, version: str, out_dir: Path, workers: int = 16) -> dict:
+    """Master data `version` from the CDN into <out_dir>/ (the manifest and every listed file, SHA-256 checked;
+    files already present with the right hash are kept)."""
+    out_dir = Path(out_dir)
+    out_dir.mkdir(parents=True, exist_ok=True)
+    base = f"{cdn.rstrip('/')}/master/{version}"
+    manifest_raw = _get(f"{base}/MasterManifest.json")
+    manifest = json.loads(manifest_raw.decode("utf-8"))
+    (out_dir / "MasterManifest.json").write_bytes(manifest_raw)
+    files = manifest.get("files", [])
+
+    def one(f: dict):
+        name, sha = f["name"], (f.get("hash") or "").lower()
+        if "/" in name or "\\" in name or name in ("", ".", ".."):
+            return {"file": name, "error": "unexpected file name"}
+        dst = out_dir / name
+        if sha and dst.is_file() and hashlib.sha256(dst.read_bytes()).hexdigest() == sha:
+            return "kept"
+        data = _get(f"{base}/{name}")
+        if sha and hashlib.sha256(data).hexdigest() != sha:
+            return {"file": name, "error": "sha256 differs from the manifest"}
+        tmp = dst.with_name(f"{name}.{os.getpid()}.part")
+        tmp.write_bytes(data)
+        os.replace(tmp, dst)
+        return "downloaded"
+
+    with ThreadPoolExecutor(max(1, workers)) as ex:
+        results = list(ex.map(one, files))
+    return {"version": manifest.get("version", version), "files": len(files),
+            "downloaded": results.count("downloaded"), "kept": results.count("kept"),
+            "failed": [r for r in results if isinstance(r, dict)], "out": str(out_dir)}