nnnotes 0.1.0__py3-none-any.whl

nnnotes/webaudio.py

@@ -0,0 +1,14 @@
+"""The audio formats of a web build (`web --format`): per format the file suffix and the ffmpeg encoder flags (None:
+the FLAC of the decode itself). Imports nothing, so the command line lists the formats without loading the
+exporters."""
+
+# BGM encodings for the web (ffmpeg). Chromium's decodeAudioData decodes each sample-aligned with the FLAC (encoder
+# delay / pre-skip removed). Default AAC-LC 160 kbit/s in MP4: decodes in Safari / iOS as well.
+WEB_AUDIO = {
+    "aac": (".m4a", ["-c:a", "aac", "-b:a", "160k"]),
+    "opus": (".opus", ["-c:a", "libopus", "-b:a", "128k"]),
+    "vorbis": (".ogg", ["-c:a", "libvorbis", "-q:a", "5"]),
+    "mp3": (".mp3", ["-c:a", "libmp3lame", "-b:a", "192k"]),
+    "flac": (".flac", None),
+}
+DEFAULT_AUDIO_FORMAT = "aac"

nnnotes/webmodel.py

@@ -0,0 +1,397 @@
+"""Live2D models -> the model part of an ournotes-player site (web.py builds the charts of the same site).
+
+    <site>/models.json             model index: id, manifest path, key, listing facts, sizes
+    <site>/models/<id>.json        model manifest: every path of the model's files -> {asset, size}, or, for a large
+                                   JSON object, {parts: [[key, asset, size], ...], size} (as a chart manifest)
+    <site>/assets/<sha256>.<ext>   content-addressed files, shared with the charts (the models share their shaders)
+
+The files of one model (the paths in its manifest), only those the player reads (read_files):
+
+    model.json                     index: format (MODEL_FORMAT), name, key, moc3, prefab, textures, canvas, shaders,
+                                   resources
+    <name>.moc3                    the moc3 (as live2d.extract_runtime writes it)
+    <name>.prefab.json             the whole prefab, clips, expressions and controllers inlined (the same)
+    textures/*.png                 the atlas pages the drawables draw with (the same)
+    shaders/shaders.json           the shader index (shader.py layout, filtered as in the chart sites: web.collect),
+    shaders/<shader>.json          the parsed forms and the GLES3 programs (GLSL ES 3.00) the drawables' materials
+    shaders/<shader>/gles3/*.glsl  select by their keywords; the mask shader's only when a drawable is masked
+
+`resources` in model.json: the materials the Cubism mask pass draws with (the Resources.Load paths of
+advscene.RESOURCES, read from the APK's boot data), exported as the story scene exports them.
+
+Models are the catalog keys Character/Live2D/<group>/<name>/model/<name>; a model's id is <name> (unique in the
+catalog, URL-safe). Each model is exported into a temporary directory, stored, and the directory deleted; models run in
+parallel worker processes (catalog cache writes serialized by a lock). A model whose manifest exists is skipped unless
+`force`. Same inputs and library versions give byte-identical outputs. A model's files read no master data and the
+regions serve the same catalog, so one build serves every region of a site (the bundles are fetched from the CDN of
+`region`).
+
+Names (optional): when master data is configured for `region`, the listing of each model the master data maps to a
+character (model_names: MasterCharacterCostume -> MasterCharacter -> MasterText) gets `character`, `names` and
+`label`; the manifests of skipped models get the same fields of this build. Without master data the fields are not
+written (and skipped manifests keep theirs).
+"""
+from __future__ import annotations
+
+import json
+import multiprocessing as mp
+import re
+import shutil
+import tempfile
+import time
+import traceback
+from pathlib import Path
+
+from . import advscene, jsonio, languages, live2d, master
+from . import shader as shader_mod
+from .config import Config, usable_cpus, use
+from .export import Exporter
+from .web import (MODELS_DIR, MODELS_INDEX, SHADER_PLATFORM, SHADER_TYPE, SITE_FORMAT, Store, _dump, _lock_fetches,
+                  _log, check_player, collect, entry_assets, text_asset, write_index, write_player)
+
+LIVE2D_PREFIX = "Character/Live2D/"
+MODEL_KEY = re.compile(r"Character/Live2D/(?P<group>[^/]+)/(?P<name>[^/]+)/model/(?P=name)")
+MODEL_ID = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]*")
+MODEL_INDEX = "model.json"
+MODEL_FORMAT = 1                                   # model.json "format"
+SHADER_DIR = "shaders"
+MASK_KEYWORD = "CUBISM_MASK_ON"                    # a drawable material's keyword: the drawable is masked
+COSTUME_TABLE = "MasterCharacterCostume"           # _live2dPath (the key after LIVE2D_PREFIX) -> _characterID
+NAME_FIELDS = ("character", "names", "label")      # listing fields from the master data (model_names)
+
+
+# ---------------------------------------------------------------- models and ids
+def model_id(key: str) -> str:
+    """The id of a model key: its <name>."""
+    m = MODEL_KEY.fullmatch(key)
+    if not m:
+        raise ValueError(f"{key}: not a Live2D model key (Character/Live2D/<group>/<name>/model/<name>)")
+    if not MODEL_ID.fullmatch(m["name"]):
+        raise ValueError(f"{key}: model name {m['name']!r} is not URL-safe")
+    return m["name"]
+
+
+def model_keys(keys) -> dict[str, str]:
+    """{id: key} of the model keys among `keys`, in id order; two keys with the same id raise."""
+    out: dict[str, str] = {}
+    for k in keys:
+        if MODEL_KEY.fullmatch(k):
+            i = model_id(k)
+            if out.get(i, k) != k:
+                raise ValueError(f"model id {i}: keys {out[i]} and {k}")
+            out[i] = k
+    return dict(sorted(out.items()))
+
+
+def select(specs, models: dict[str, str]) -> dict[str, str]:
+    """The models `specs` names (ids or keys; None: all) as {id: key} in id order, from `models` ({id: key})."""
+    if specs is None:
+        return dict(models)
+    by_key = {k: i for i, k in models.items()}
+    out, unknown = {}, []
+    for s in specs:
+        i = s if s in models else by_key.get(s)
+        if i is None:
+            unknown.append(s)
+        else:
+            out[i] = models[i]
+    if unknown:
+        raise ValueError(f"not a Live2D model of the catalog: {', '.join(unknown)} "
+                         f"(give a model id or a key Character/Live2D/<group>/<name>/model/<name>)")
+    return dict(sorted(out.items()))
+
+
+def catalog_models(cat, specs=None) -> dict[str, str]:
+    """`select` over the model keys of the catalog `cat`."""
+    return select(specs, model_keys(cat.keys(LIVE2D_PREFIX)))
+
+
+# ---------------------------------------------------------------- names from the master data
+def model_names(master_dir: Path, language: str | None = None) -> dict[str, dict]:
+    """{model key: {"character", "names", "label"}} of the models the master data maps to one character: every
+    MasterCharacterCostume row maps the key LIVE2D_PREFIX + `_live2dPath` to `_characterID` (the MasterCharacter id,
+    `character`); `names`: {language code: the MasterText of the character's `_nameTextID`}, the languages with a
+    text; `label`: the name in `language` (left out without `language` or a text in it). A key the rows map to two
+    characters, or a character without a name text, is left out."""
+    chars_of: dict[str, set] = {}
+    for r in master.table(master_dir, COSTUME_TABLE):
+        if r.get("_live2dPath") and r.get("_characterID") is not None:
+            chars_of.setdefault(LIVE2D_PREFIX + r["_live2dPath"], set()).add(r["_characterID"])
+    chars = {r["_id"]: r for r in master.table(master_dir, "MasterCharacter")}
+    texts = {r["_id"]: r for r in master.table(master_dir, "MasterText")}
+    out = {}
+    for key, ids in sorted(chars_of.items()):
+        if len(ids) != 1:
+            continue
+        (cid,) = ids
+        row = texts.get((chars.get(cid) or {}).get("_nameTextID"))
+        names = {c: t for c, t in languages.texts(row).items() if isinstance(t, str) and t} if row else {}
+        if not names:
+            continue
+        entry = {"character": cid, "names": names}
+        if language in names:
+            entry["label"] = names[language]
+        out[key] = entry
+    return out
+
+
+def master_names(cfg: Config, region: str | None = None, log=None) -> dict[str, dict] | None:
+    """model_names of the master data of `region` (default: [catalog] region; the --master flag, else
+    [servers.<region>] master, else [paths] master), labels in [catalog] language; None when no master data is
+    configured or it lacks a table model_names reads (logged)."""
+    section, key = cfg.master(region or cfg.get("catalog", "region"))
+    if cfg.path(section, key) is None:
+        return None
+    from .cli import master_dir
+    code = cfg.get("catalog", "language")
+    try:
+        return model_names(master_dir(cfg, region), languages.check(code) if code else None)
+    except FileNotFoundError as e:
+        (log or _log)(f"model names left out: {Path(e.filename).name if e.filename else e} not in the master data")
+        return None
+
+
+def refresh_names(site: Path, mid: str, entry: dict | None) -> bool:
+    """Set the NAME_FIELDS of the model manifest of `mid` to those of `entry` (None: none); True when it changed."""
+    p = Path(site) / MODELS_DIR / f"{mid}.json"
+    man = json.loads(p.read_text(encoding="utf-8"))
+    model = {k: v for k, v in man.get("model", {}).items() if k not in NAME_FIELDS}
+    model.update({k: entry[k] for k in NAME_FIELDS if entry and k in entry})
+    if model == man.get("model"):
+        return False
+    man["model"] = model
+    p.write_bytes(_dump(man))
+    return True
+
+
+# ---------------------------------------------------------------- one model
+def shader_names(doc) -> set[str]:
+    """Names of the shaders an export references (its {"shader": <name>} objects)."""
+    out: set[str] = set()
+    stack = [doc]
+    while stack:
+        v = stack.pop()
+        if isinstance(v, dict):
+            s = v.get("shader")
+            if isinstance(s, str) and len(v) == 1:
+                out.add(s)
+            stack.extend(v.values())
+        elif isinstance(v, list):
+            stack.extend(v)
+    return out
+
+
+def drawable_materials(prefab: dict) -> list[dict]:
+    """The material of every drawable: MeshRenderer.m_Materials[0] of the prefab's nodes."""
+    return [c["m_Materials"][0] for n in prefab["nodes"] for c in n["components"]
+            if c.get("type") == "MeshRenderer" and c.get("m_Materials") and c["m_Materials"][0]]
+
+
+def drawable_textures(prefab: dict) -> list[str]:
+    """The atlas pages the drawables draw with (every distinct CubismRenderer._mainTexture), sorted."""
+    return sorted({c["_mainTexture"]["texture"] for n in prefab["nodes"] for c in n["components"]
+                   if c.get("class") == "CubismRenderer" and c.get("_mainTexture")})
+
+
+def shader_files(index: list, materials: list[dict]) -> list[str]:
+    """The files of a shader directory (paths relative to it) that `materials` draw with: per shader its parsed file
+    and, for each distinct keyword set of its materials, the GLES3 program of subshader 0 pass 0 whose keywords are
+    exactly the material's keywords among those the shader's programs there use (a global keyword such as
+    _ADDITIONAL_LIGHTS is never in a material's set, so its variants are never picked)."""
+    recs = {r["name"]: r for r in index}
+    files = set()
+    for name, kws in sorted({(m["shader"]["shader"], tuple(m["keywords"])) for m in materials}):
+        if name not in recs:
+            raise RuntimeError(f"shader {name}: not in the shader index")
+        variants = [v for v in recs[name]["variants"] if v["platform"] == SHADER_PLATFORM and v["type"] == SHADER_TYPE
+                    and v["subShader"] == 0 and v["pass"] == 0]
+        want = set(kws) & {k for v in variants for k in v["keywords"]}
+        hits = [v for v in variants if set(v["keywords"]) == want]
+        if len(hits) != 1:
+            raise RuntimeError(f"shader {name}: {len(hits)} GLES3 programs with the keywords {sorted(want)}")
+        files |= {recs[name]["parsed"], hits[0]["file"]}
+    return sorted(files)
+
+
+def read_files(doc: dict, prefab: dict, index: list) -> list[str]:
+    """The files of a model the player reads (model.json `doc`, its prefab, its shader index): model.json, the moc3,
+    the prefab, the drawables' atlas pages, the shader index and the shader files of the drawables' materials, plus
+    those of the mask materials when a drawable is masked (MASK_KEYWORD)."""
+    materials = drawable_materials(prefab)
+    if any(MASK_KEYWORD in m["keywords"] for m in materials):
+        materials += list(doc["resources"].values())
+    sdir = doc["shaders"].rpartition("/")[0]
+    return sorted({MODEL_INDEX, doc["moc3"], doc["prefab"], *doc["textures"], doc["shaders"],
+                   *(f"{sdir}/{f}" if sdir else f for f in shader_files(index, materials))})
+
+
+def export_model(cat, player, key: str, out_dir: Path) -> dict:
+    """The files of one model (module docstring) into `out_dir`; returns the runtime export summary with `canvas`,
+    `shaders` (names) and `files` (read_files: the paths to store)."""
+    out_dir = Path(out_dir)
+    rt = live2d.export_runtime(cat, key, out_dir)
+    res, ex = rt.summary, rt.exporter
+    prefab = json.loads((out_dir / res["prefab"]).read_text(encoding="utf-8"))
+    rex = Exporter(cat, out_dir, player=player)
+    resources = {name: rex.material(player.resource(path)) for name, path in advscene.RESOURCES.items()}
+    objs = {**rex.shaders, **ex.shaders}
+    names = sorted(shader_names(prefab) | shader_names(resources))
+    missing = [n for n in names if n not in objs]
+    if missing:
+        raise RuntimeError(f"{key}: shaders neither in the model's bundles nor in the player data: {missing}")
+    index: list = []
+    for n in names:
+        shader_mod.dump_objects([objs[n]], out_dir / SHADER_DIR, objs[n].assets_file.name, index)
+    shader_mod.write_index(index, out_dir / SHADER_DIR)
+    doc = {"format": MODEL_FORMAT, "name": res["name"], "key": key, "moc3": res["moc3"], "prefab": res["prefab"],
+           "textures": drawable_textures(prefab), "canvas": prefab["canvas"], "shaders": f"{SHADER_DIR}/shaders.json",
+           "resources": resources}
+    jsonio.write_json(out_dir / MODEL_INDEX, doc)
+    return {**res, "textures": doc["textures"], "canvas": prefab["canvas"], "shaders": names,
+            "files": read_files(doc, prefab, index)}
+
+
+def model_facts(key: str, summary: dict, names: dict | None = None) -> dict:
+    """What a model listing needs, from the key and the export summary, plus the NAME_FIELDS of `names` (an entry
+    of model_names)."""
+    facts = {"group": MODEL_KEY.fullmatch(key)["group"], "canvas": summary["canvas"],
+             "textures": len(summary["textures"]), "nodes": summary["nodes"]}
+    facts.update({k: names[k] for k in NAME_FIELDS if names and k in names})
+    return facts
+
+
+def ingest(store: Store, site: Path, mid: str, key: str, model_dir: Path, summary: dict,
+           names: dict | None = None) -> dict:
+    """One exported model's files (summary["files"]) into the store + its manifest (`names`: model_facts)."""
+    text, binary = collect(Path(model_dir), summary["files"])
+    entries = {p: store.put_file(p, text_asset(p, s)) for p, s in text.items()}
+    entries.update({p: store.put(p, b) for p, b in binary.items()})
+    manifest = {"format": SITE_FORMAT, "id": mid, "key": key, "model": model_facts(key, summary, names),
+                "files": dict(sorted(entries.items()))}
+    (Path(site) / MODELS_DIR / f"{mid}.json").write_bytes(_dump(manifest))
+    return {"id": mid, "ok": True, "files": len(entries), "bytes": sum(e["size"] for e in entries.values())}
+
+
+# ---------------------------------------------------------------- index
+def write_models_index(site: Path) -> tuple[int, set[str]]:
+    """site/models.json from every model manifest present (no models directory: no models.json); returns the number
+    of models and the assets their manifests reference."""
+    site = Path(site)
+    mdir = site / MODELS_DIR
+    if not mdir.is_dir():
+        (site / MODELS_INDEX).unlink(missing_ok=True)
+        return 0, set()
+    models, used = [], set()
+    for p in sorted(mdir.glob("*.json")):
+        man = json.loads(p.read_text(encoding="utf-8"))
+        for e in man["files"].values():
+            used.update(entry_assets(e))
+        models.append({"id": p.stem, "manifest": f"{MODELS_DIR}/{p.name}", "key": man["key"],
+                       "files": len(man["files"]),
+                       "bytes": sum(f["size"] for f in man["files"].values()), **man["model"]})
+    (site / MODELS_INDEX).write_bytes(_dump({"format": SITE_FORMAT, "models": models}))
+    return len(models), used
+
+
+# ---------------------------------------------------------------- per model (in a worker or in this process)
+_W: dict = {}
+
+
+def _open(cfg: Config, region: str | None = None):
+    from .cli import open_catalog, player_data
+    return open_catalog(cfg, region=region), player_data(cfg)
+
+
+def _worker_init(cfg: Config, lock, job: dict) -> None:
+    use(cfg)
+    cat, player = _open(cfg, job.get("region"))
+    _lock_fetches(cat, lock)
+    _W.update(cat=cat, player=player, job=job)
+
+
+def model_task(mid: str, key: str, job: dict | None = None, data=None) -> dict:
+    """One model: export into a temporary directory, ingest; the directory is deleted."""
+    cat, player = data or (_W["cat"], _W["player"])
+    job = job or _W["job"]
+    site = Path(job["site"])
+    work = Path(tempfile.mkdtemp(prefix=f"{mid}-", dir=job["tmp"]))
+    t0, stage = time.time(), "export"
+    try:
+        summary = export_model(cat, player, key, work)
+        stage = "ingest"
+        r = ingest(Store(site), site, mid, key, work, summary, (job.get("names") or {}).get(key))
+        return {**r, "seconds": round(time.time() - t0, 1)}
+    except Exception as e:
+        cause = f"{type(e).__name__}: {e}"
+        _log(f"{mid}: {stage} failed: {cause[:300]}")
+        return {"id": mid, "ok": False, "stage": stage, "error": cause[:2000], "trace": traceback.format_exc()[-3000:]}
+    finally:
+        shutil.rmtree(work, ignore_errors=True)
+
+
+def _pool_task(args):
+    return model_task(*args)
+
+
+# ---------------------------------------------------------------- build
+def build(out_dir, models: dict[str, str] | None, cfg: Config, player_dir, force: bool = False, *, tmp_dir=None,
+          log=None, workers: int | None = None, region: str | None = None) -> dict:
+    """Add the Live2D models `models` ({id: key}, see `catalog_models`; None: every model of the catalog) to the site
+    at `out_dir`, with the player of the ournotes-player checkout or package at `player_dir` (its page files are
+    written, models.json and charts.json rebuilt). The data comes from the settings `cfg` (each worker process opens
+    its own), bundles from the CDN of `region` (default: [catalog] region), names from the master data of `region`
+    when it is configured (master_names). `workers`: parallel model processes (default up to 4)."""
+    player_dir = check_player(player_dir)
+    site = Path(out_dir).resolve()
+    (site / MODELS_DIR).mkdir(parents=True, exist_ok=True)
+    Store(site)
+    tmp_root = Path(tmp_dir).resolve() if tmp_dir else site.parent / f"{site.name}.tmp"
+    tmp_root.mkdir(parents=True, exist_ok=True)
+    log = log or _log
+    t0 = time.time()
+    if models is None:
+        from .cli import open_catalog
+        models = catalog_models(open_catalog(cfg, bundles=False, region=region))
+    todo, skipped = [], []
+    for mid, key in sorted(models.items()):
+        if model_id(key) != mid:
+            raise ValueError(f"model {mid}: key {key} has the id {model_id(key)}")
+        if (site / MODELS_DIR / f"{mid}.json").exists() and not force:
+            skipped.append(mid)
+        else:
+            todo.append((mid, key))
+    if workers is None:
+        workers = max(1, min(4, len(todo), usable_cpus() // 2))
+    names = master_names(cfg, region, log)
+    if names is not None:
+        for mid in skipped:
+            refresh_names(site, mid, names.get(models[mid]))
+    job = {"site": str(site), "tmp": str(tmp_root), "region": region, "names": names or {}}
+    results = []
+    if todo:
+        log(f"{len(todo)} models, {workers} worker(s)")
+        if workers <= 1:
+            data = _open(cfg) if region is None else _open(cfg, region)
+            for mid, key in todo:
+                results.append(model_task(mid, key, job, data=data))
+        else:
+            ctx = mp.get_context("spawn")
+            with ctx.Manager() as mgr:
+                lock = mgr.Lock()
+                with ctx.Pool(workers, initializer=_worker_init, initargs=(cfg, lock, job)) as pool:
+                    for r in pool.imap_unordered(_pool_task, [(mid, key) for mid, key in todo]):
+                        results.append(r)
+                        if len(results) % 20 == 0:
+                            log(f"{len(results)}/{len(todo)} models")
+    v = write_player(site, player_dir)
+    idx = write_index(site)
+    results.sort(key=lambda r: r["id"])
+    failed = [r for r in results if not r["ok"]]
+    if failed:
+        (site.parent / f"{site.name}.model-failures.json").write_bytes(_dump(failed))
+    return {"site": str(site),
+            "modelsBuilt": [{k: r[k] for k in ("id", "files", "bytes")} for r in results if r["ok"]],
+            "modelsFailed": [{k: r.get(k) for k in ("id", "stage", "error")} for r in failed],
+            "modelsSkipped": skipped, "modelSeconds": round(time.time() - t0, 1), "modelWorkers": workers,
+            "modelNames": None if names is None else sum(1 for k in models.values() if k in names),
+            **v, **idx}

nnnotes-0.1.0.dist-info/METADATA

@@ -0,0 +1,200 @@
+Metadata-Version: 2.4
+Name: nnnotes
+Version: 0.1.0
+Summary: Offline data toolkit for BanG Dream! Our Notes game files: catalogs, bundles, master data, stories, Live2D, audio and live charts
+Author: MetaMiku, emptysekai
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/MetaSekaiLab/nnnotes
+Project-URL: Source, https://github.com/MetaSekaiLab/nnnotes
+Project-URL: Issues, https://github.com/MetaSekaiLab/nnnotes/issues
+Classifier: Programming Language :: Python :: 3
+Classifier: Operating System :: OS Independent
+Classifier: Topic :: Multimedia
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: UnityPy>=1.25
+Requires-Dist: numpy>=2.0
+Requires-Dist: Pillow>=10.0
+Requires-Dist: pycryptodome>=3.20
+Requires-Dist: grpcio>=1.49.1
+Provides-Extra: test
+Requires-Dist: pytest>=8; extra == "test"
+Requires-Dist: pyflakes>=3.2; extra == "test"
+Requires-Dist: jsonschema>=4.18; extra == "test"
+Provides-Extra: fonts
+Requires-Dist: fonttools>=4.50; extra == "fonts"
+Requires-Dist: freetype-py>=2.5; extra == "fonts"
+Requires-Dist: scipy>=1.13; extra == "fonts"
+Dynamic: license-file
+
+# nnnotes？！
+
+[简体中文](https://github.com/MetaSekaiLab/nnnotes/blob/main/README.md) | [English](https://github.com/MetaSekaiLab/nnnotes/blob/main/README.en.md)
+
+nnnotes is an offline data toolkit for the game files of BanG Dream! Our Notes: it reads Addressables catalogs,
+downloads and decrypts asset bundles, downloads and decodes master data, and exports stories, Live2D models, spots,
+shaders, CRI audio and live charts as structured JSON and common file formats. The chart export is the data that
+[ournotes-player](https://github.com/empty-sekai/ournotes-player) reads. The naming was inspired by [mos9527/sssekai](https://github.com/mos9527/sssekai).
+
+This is an unofficial fan project, not affiliated with the game's developer or operator. The repository contains no
+game assets, keys or server addresses: users supply the game files, the keys needed for decryption and the server
+addresses in their own configuration, and the exports stay in local directories the user chooses.
+
+## Features
+
+| Command | Input | Output |
+|---|---|---|
+| `catalog` | the region's catalog | addressable keys (optionally by prefix) |
+| `browse` | the configured regions' catalogs | a local web page to browse catalogs and bundles (`--host` / `--port`) |
+| `pull` | addressable keys | every bundle of each key's dependency closure, decrypted into the local cache |
+| `servers` | the bootstrap API root `[bootstrap] api` | the game API's server list: each region's name, area id and CDN / API roots (the roots only with `--show-hosts`) |
+| `master version` | region | the master data version and resource version the region serves now (anonymous game API call) |
+| `master download` | master data version, or `--latest` (the region's current one) | that version's `MasterManifest.json` and every `.bin` file (SHA-256 checked) |
+| `master decode` | master data `.bin` files or directories | one JSON per table (Rijndael-256 CBC decryption + gzip) |
+| `adv` | episode ID | `episode.json`: command list, lines in five languages, voice / sound / video index |
+| `story` | episode ID | a full story directory: episode, every Live2D model, audio, stage scene and shaders, story UI, frames / particle effects / post effects / stills / talk windows / chat assets, videos (WebM) |
+| `live2d` | model key or model id | a Live2D (Cubism) runtime directory: moc3, textures, motion3, physics, expressions, prefab parameters |
+| `spot` | spot ID | `spot.json` + Spine characters + the room as `room.glb` + shaders |
+| `room` | background prefab key | the room model (binary glTF) |
+| `shader` | key or APK bundles | every platform variant of the shaders (GLSL ES and others) with an index |
+| `audio` | CRI cue sheet | one audio file per cue (FLAC / Ogg / WAV) + cue metadata |
+| `crikey` | APK | the CRI HCA keycode from the game's boot data (shows whether it was found; can write a `.hcakey`) |
+| `player` | APK | render-related global settings (color space, quality levels, renderers) as JSON |
+| `live` | music ID + difficulty | a full chart directory: chart and runtime notes, 3D scene, note and effect assets, BGM and sounds, sound routing |
+| `web` | `--pair music:difficulty` (repeatable) or `--all`; `--live2d model` (repeatable) or `--all-live2d`; `--story episode` (repeatable) or `--all-stories`; `--region region` (repeatable) or `--all-regions` | an ournotes-player static site: shared player + per-chart / per-model / per-episode manifests + content-addressed assets; one site can serve several regions, with listing texts in five languages; a story's interface texts are grouped by language, with TextMeshPro font assets generated from open fonts (the game's fonts with `--fonts game`) |
+| `deck-data` | master data files (`--master-files` directory or `--apk-master`) | one JSON file for deck-building tools: every chart's runtime notes, skill events and fever ranges, and the master data tables about cards, skills, bonuses, scores and events ([format](https://github.com/MetaSekaiLab/nnnotes/blob/main/docs/deck-data.md)) |
+
+Export conventions:
+
+- Commands that write files take the output path with `-o` (required); `web` takes the site directory as a
+  positional argument.
+- JSON is always UTF-8 with LF line endings; infinity is written as `1e999`.
+- Exports are deterministic for a given installation: the same input with the same versions of nnnotes, its Python
+  dependencies (UnityPy, Pillow, numpy, ...) and the external tools (vgmstream, FFmpeg) gives byte-identical files.
+  Other versions can encode the same content into other bytes: PNG files written by another Pillow version can
+  differ in their bytes while their pixels are equal.
+- Values keep Unity's serialized values and field names (`m_LocalPosition`, `_bandIDs`, ...), so they can be
+  compared with the game data.
+- Textures are exported as PNG, shaders keep the game's own compiled programs, audio is decoded from the CRI formats
+  to common formats.
+
+## Status
+
+Based on all data of the Taiwan server, version 1.0.1 (zh-Hant):
+
+| Part | Status |
+|---|---|
+| catalog / bundle decryption / dependency closure | working |
+| master data decoding | working |
+| stories, `adv` | 946 / 946 episodes |
+| stories, `story` (full directory) | 946 / 946 episodes have all their resources in the catalog and of supported kinds (the resource closure equals the game's own per-episode download list); 759 exported one by one, the other 187 (with frames, effects, post effects, stills and the like) not yet one by one |
+| Live2D models | 239 / 239 (every model of the catalog; episodes use 185 of them) |
+| CRI audio | 681 / 681 cue sheets |
+| charts, `live` | 336 / 336 (music, difficulty) pairs |
+| web site, `web` | 336 / 336 charts, 239 / 239 Live2D models, 946 / 946 story episodes (two full builds in English without audio are byte-identical and 946 / 946 pass the data validation; a 27-episode sample in five languages with AAC audio is likewise identical across two builds and valid) |
+| spots, `spot` / `room` | one spot verified, the others not individually checked |
+| other regions (en / kr) and languages | checked: the regions serve the same catalog for a language and the same bundles, and the keys are shared; the master tables the charts use are the same in the three regions, and the text tables have all five languages; the chart exports checked match the Taiwan server's. A full multi-region site build is not verified yet |
+
+## Requirements
+
+- Python 3.11+, 3.13 recommended (a `web` build reads and writes a lot of JSON, and the standard library encodes
+  JSON faster on 3.13); `pip install nnnotes` (for development `pip install -e .` in the repository)
+- the game's `base.apk`: APK-local bundles, the CRI keycode, boot settings. `player`, `story`, `live` and `web` read
+  MonoBehaviours of its boot data with type trees that ship with nnnotes; supported now: game version 1.0.1
+  (Unity 6000.3.12f1). With an APK of another version whose classes do not match, these commands stop with an
+  error naming the class, the game version and the Unity version
+- a decoded master data directory (from `master download` + `master decode`): needed by `adv`, `story`, `spot`,
+  `live` and the charts of `web`; the Live2D models of `web` use it only for their character names (optional)
+- external tools: [vgmstream](https://vgmstream.org/) (CRI HCA decoding), [FFmpeg](https://ffmpeg.org/)
+  (transcoding, including the WebM muxing and Opus audio of story videos); `web` also needs Node.js 20+ and a built
+  ournotes-player
+
+## Configuration
+
+The code contains no keys, server addresses or default paths. Settings are read in this order, later sources
+overriding earlier ones:
+
+1. config file: `--config <file>`, else `NNNOTES_CONFIG`, else `nnnotes.toml` in the working directory
+2. environment variables: `NNNOTES_<SECTION>_<KEY>` (e.g. `NNNOTES_BUNDLE_KEY`, `NNNOTES_SERVERS_TW_CDN`)
+3. command-line flags: `--region`, `--language`, `--catalog`, `--cache`, `--master`, `--apk`, `--ffmpeg`,
+   `--vgmstream`, `--node` go before the command name; `--player` is an option of `web`
+
+Copy [`nnnotes.example.toml`](https://github.com/MetaSekaiLab/nnnotes/blob/main/nnnotes.example.toml) to `nnnotes.toml` and fill it in. The settings are: the bundle
+key and nonce seed, the master data key and IV, the region in use (`[catalog] region`) and the catalog language
+(`[catalog] language`), the CDN base and API root of each region (`cdn`, `api` of `[servers.<region>]`), the client
+version (`[client] version`; unset: the APK's versionName), optionally the bootstrap API root (`[bootstrap] api`, for
+`servers`), and the paths of the cache, the APK, the master data directory, ournotes-player and vgmstream / FFmpeg /
+Node.js (the three tools are looked up on `PATH` when unset). All values come from the game client you own.
+
+A missing or malformed setting stops the command with exit status 2 and one line naming the TOML key, the
+environment variable and the flag; no setting value is printed. `nnnotes.toml` is in `.gitignore`; do not commit it.
+
+See [docs/configuration.md](https://github.com/MetaSekaiLab/nnnotes/blob/main/docs/configuration.md) for the full reference.
+
+## Examples
+
+```bash
+nnnotes catalog --prefix Live/MusicScore/ --limit 20
+nnnotes browse --port 8000
+nnnotes pull Live/MusicScore/0001/0001_03
+nnnotes master version
+nnnotes master download --latest -o work/master-bin
+nnnotes master decode work/master-bin -o work/master
+nnnotes --master work/master adv 10462 -o out/adv_10462.json
+nnnotes story 10462 -o out/story_10462
+nnnotes live 100001 --difficulty expert -o out/live_100001
+nnnotes web out/site --all --player <ournotes-player dir> --workers 5
+nnnotes web out/site --pair 100001:expert --pair 100001:hard --player <ournotes-player dir>
+nnnotes web out/site --all --region tw --region en --region kr --player <ournotes-player dir>
+nnnotes web out/site --live2d adv_live2d_rana_003_casual_spring_01 --player <ournotes-player dir>
+```
+
+`web` builds incrementally: charts and models whose manifest exists are skipped (`--force` rebuilds them) and assets
+no longer referenced are removed. Common options:
+
+- `--format aac|opus|vorbis|mp3|flac`: BGM format, default AAC; `--no-audio`: no audio files
+- `--band` / `--leader-card`: the band of the LightWeight background and the start timeline; default: the band of
+  the music's first vocal character
+- `--workers`: parallel music processes (default a quarter of the CPUs, up to 8) and model processes (default up
+  to 4); `--read-workers`: chart read sets at a time (Node.js processes; default half the CPUs, up to 16); `--tmp`:
+  temporary build directory (default `<site>.tmp`)
+- `--live2d MODEL` (model id or key, repeatable) / `--all-live2d`: add Live2D models (every model of the catalog);
+  charts and models can be added in the same run
+- `--region REGION` (repeatable) / `--all-regions`: the regions the site serves (default: `[catalog] region`);
+  each region's master data is `[servers.<region>] master`. Regions with the same chart data share one manifest;
+  the chart list page switches region and language with `?region=&lang=`
+- `--player-only`: rewrite the player files, `charts.json` and `models.json` only; `--reingest-json`: store every
+  chart's and model's JSON files again under the current rules
+
+See [docs/commands.md](https://github.com/MetaSekaiLab/nnnotes/blob/main/docs/commands.md) for every command's options and output layout.
+
+## Architecture
+
+```
+settings (TOML / environment / flags)
+  └─ access    addressables (catalog parsing, bundle decryption, local browser)
+               catalog (dependency closures, remote and APK bundles, local cache)
+               master (master data download and decoding)
+               gameapi (anonymous game API calls: current master data version, server list)
+       └─ reading  unity (UnityPy type trees and TextAssets)
+                   export (prefabs / components / references resolved to JSON, textures and shaders alongside)
+                   shader, textstyle (text layout and style), tmpfont (TextMesh Pro fonts), player (boot settings), cri + crikey (CRI audio)
+            └─ content  stories: adv, advscene, advmedia (frames / effects / post effects / stills / chat),
+                                 advvideo (USM videos), advui, story
+                        Live2D: live2d, motion
+                        spots: spot, room
+                        charts: score (chart parsing and a reimplementation of the game's chart converter),
+                                livescene, livenotes, liveui, liveaudio, live
+                        site: web, webmodel (ournotes-player data)
+```
+
+Every JSON file is written by one writer (`jsonio`), so encoding, line endings and number formatting are the same
+everywhere.
+
+See [docs/performance.md](https://github.com/MetaSekaiLab/nnnotes/blob/main/docs/performance.md) for where a
+`web` build spends its time and which parts of it run in compiled code.
+
+## License
+
+MIT, see [LICENSE](https://github.com/MetaSekaiLab/nnnotes/blob/main/LICENSE).