nnnotes 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nnnotes/__init__.py +7 -0
- nnnotes/__main__.py +4 -0
- nnnotes/acb.py +190 -0
- nnnotes/addressables.py +413 -0
- nnnotes/adv.py +243 -0
- nnnotes/advcommand.py +29 -0
- nnnotes/advmedia.py +178 -0
- nnnotes/advscene.py +113 -0
- nnnotes/advui.py +1154 -0
- nnnotes/advvideo.py +684 -0
- nnnotes/atoms/__init__.py +151 -0
- nnnotes/atoms/astc.py +52 -0
- nnnotes/atoms/gltf.py +290 -0
- nnnotes/atoms/mesh.py +289 -0
- nnnotes/atoms/png.py +28 -0
- nnnotes/atoms/reader.py +269 -0
- nnnotes/atoms/sniff.py +198 -0
- nnnotes/atoms/sprite.py +236 -0
- nnnotes/atoms/texture.py +80 -0
- nnnotes/cache.py +289 -0
- nnnotes/catalog.py +373 -0
- nnnotes/catalogdb.py +457 -0
- nnnotes/census.py +276 -0
- nnnotes/cli.py +744 -0
- nnnotes/cli_assets.py +1539 -0
- nnnotes/config.py +238 -0
- nnnotes/contract.py +497 -0
- nnnotes/cri.py +371 -0
- nnnotes/crikey.py +71 -0
- nnnotes/crilips.py +265 -0
- nnnotes/cristages.py +558 -0
- nnnotes/deckdata.py +462 -0
- nnnotes/export.py +1462 -0
- nnnotes/gameapi.py +318 -0
- nnnotes/jsonio.py +98 -0
- nnnotes/languages.py +44 -0
- nnnotes/layout.py +596 -0
- nnnotes/link.py +418 -0
- nnnotes/live.py +115 -0
- nnnotes/live2d.py +235 -0
- nnnotes/liveaudio.py +331 -0
- nnnotes/livenotes.py +291 -0
- nnnotes/liveoptions.py +250 -0
- nnnotes/livescene.py +450 -0
- nnnotes/liveui.py +328 -0
- nnnotes/master.py +251 -0
- nnnotes/modelstages.py +424 -0
- nnnotes/motion.py +131 -0
- nnnotes/objexport.py +1564 -0
- nnnotes/orchestrate.py +630 -0
- nnnotes/plan.py +218 -0
- nnnotes/player.py +392 -0
- nnnotes/room.py +391 -0
- nnnotes/score.py +1139 -0
- nnnotes/shader.py +217 -0
- nnnotes/spot.py +201 -0
- nnnotes/stages.py +285 -0
- nnnotes/store.py +518 -0
- nnnotes/story.py +142 -0
- nnnotes/storyfonts.py +1258 -0
- nnnotes/storyhost.py +1190 -0
- nnnotes/storysite.py +662 -0
- nnnotes/storyvoices.py +102 -0
- nnnotes/textstyle.py +355 -0
- nnnotes/tmpfont.py +636 -0
- nnnotes/typetrees/6000.3.12f1.json +19 -0
- nnnotes/unity.py +368 -0
- nnnotes/viewrules.json +441 -0
- nnnotes/views.py +966 -0
- nnnotes/voices.py +1363 -0
- nnnotes/web.py +1624 -0
- nnnotes/webaudio.py +14 -0
- nnnotes/webmodel.py +397 -0
- nnnotes-0.1.0.dist-info/METADATA +200 -0
- nnnotes-0.1.0.dist-info/RECORD +79 -0
- nnnotes-0.1.0.dist-info/WHEEL +5 -0
- nnnotes-0.1.0.dist-info/entry_points.txt +2 -0
- nnnotes-0.1.0.dist-info/licenses/LICENSE +21 -0
- nnnotes-0.1.0.dist-info/top_level.txt +1 -0
nnnotes/plan.py
ADDED
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
"""Plan: what a run of a pipeline would do, and why, without running anything.
|
|
2
|
+
|
|
3
|
+
Every subject of every stage is a node. A node is a `hit` (its result is in the store), `run` (it would run), or
|
|
4
|
+
`unknown` (it reads the result of a task that would run first, or of one that has not run: its inputs are not known
|
|
5
|
+
yet). Why a node runs is the difference between its key parts and those of the same task id in a previous run
|
|
6
|
+
manifest (the latest of the same selection, or one named with --since):
|
|
7
|
+
|
|
8
|
+
new-input the previous run had no task with this id
|
|
9
|
+
input-changed{role,old,new,name} an input's content id (null: added / removed), the new input's name
|
|
10
|
+
stage-version{old,new} the stage's version
|
|
11
|
+
atom-changed{atom,old,new} an atom's implementation id (null: added / removed)
|
|
12
|
+
params-changed{path,old,new} an output-affecting parameter (path: /name/sub...)
|
|
13
|
+
context-changed{entry} a consumed context entry
|
|
14
|
+
previous-failed the task failed in the previous run
|
|
15
|
+
result-missing same key as before, but the result is not in the store
|
|
16
|
+
forced asked to run again
|
|
17
|
+
Hit reasons: `unchanged` (same key as before), `cached` (the key's result is in the store from another run, e.g.
|
|
18
|
+
content that changed back). Unknown: `pending{task}`.
|
|
19
|
+
|
|
20
|
+
The plan document (nnnotes.plan/1) lists the nodes in pipeline order with key, status, reasons and cost estimate,
|
|
21
|
+
the edges (Stage.depends) and a summary: counts, estimated CPU seconds and peak memory of what would run with the
|
|
22
|
+
given workers and memory budget, and per output layout the files it would write, remove and keep.
|
|
23
|
+
"""
|
|
24
|
+
from __future__ import annotations
|
|
25
|
+
|
|
26
|
+
from pathlib import Path
|
|
27
|
+
|
|
28
|
+
from . import contract
|
|
29
|
+
from .orchestrate import WORKER_BYTES
|
|
30
|
+
from .stages import Env, Pending, Stage, describe
|
|
31
|
+
from .store import write_file
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _leaves(old, new, path: str = ""):
|
|
35
|
+
"""(path, old, new) of every differing leaf of two JSON values (dicts are walked, other values compared)."""
|
|
36
|
+
if isinstance(old, dict) and isinstance(new, dict):
|
|
37
|
+
for k in sorted(set(old) | set(new)):
|
|
38
|
+
yield from _leaves(old.get(k), new.get(k), f"{path}/{k}")
|
|
39
|
+
elif contract.key_text(old) != contract.key_text(new):
|
|
40
|
+
yield path or "/", old, new
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def why(old_parts: dict, new_parts: dict) -> list[dict]:
|
|
44
|
+
"""The reasons two key parts differ (empty when they do not)."""
|
|
45
|
+
out = []
|
|
46
|
+
ov, nv = old_parts["stage"]["version"], new_parts["stage"]["version"]
|
|
47
|
+
if ov != nv:
|
|
48
|
+
out.append({"code": "stage-version", "old": ov, "new": nv})
|
|
49
|
+
oi, ni = dict(map(tuple, old_parts["inputs"])), dict(map(tuple, new_parts["inputs"]))
|
|
50
|
+
for role in sorted(set(oi) | set(ni)):
|
|
51
|
+
if oi.get(role) != ni.get(role):
|
|
52
|
+
out.append({"code": "input-changed", "role": role, "old": oi.get(role), "new": ni.get(role)})
|
|
53
|
+
oa, na = old_parts["atoms"], new_parts["atoms"]
|
|
54
|
+
for atom in sorted(set(oa) | set(na)):
|
|
55
|
+
if oa.get(atom) != na.get(atom):
|
|
56
|
+
out.append({"code": "atom-changed", "atom": atom, "old": oa.get(atom), "new": na.get(atom)})
|
|
57
|
+
for path, o, n in _leaves(old_parts["params"], new_parts["params"]):
|
|
58
|
+
out.append({"code": "params-changed", "path": path, "old": o, "new": n})
|
|
59
|
+
oc, nc = old_parts["context"], new_parts["context"]
|
|
60
|
+
for entry in sorted(set(oc) | set(nc)):
|
|
61
|
+
if contract.key_text(oc.get(entry)) != contract.key_text(nc.get(entry)):
|
|
62
|
+
out.append({"code": "context-changed", "entry": entry})
|
|
63
|
+
return out
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
class Plan:
|
|
67
|
+
"""A computed plan: `doc` (nnnotes.plan/1) and the runnable tasks (status `run`)."""
|
|
68
|
+
|
|
69
|
+
def __init__(self, doc: dict, tasks: list):
|
|
70
|
+
self.doc, self.tasks = doc, tasks
|
|
71
|
+
|
|
72
|
+
def check(self) -> int:
|
|
73
|
+
"""1 when anything would run or change (a node not a hit, a layout file to write or remove), else 0."""
|
|
74
|
+
if any(n["status"] != "hit" for n in self.doc["nodes"]):
|
|
75
|
+
return 1
|
|
76
|
+
return int(any(v["write"] or v["remove"] for v in self.doc["summary"]["layouts"].values()))
|
|
77
|
+
|
|
78
|
+
def emit(self, directory) -> list[Path]:
|
|
79
|
+
"""Write the runnable tasks as task descriptions <directory>/<key>.json (one per distinct key)."""
|
|
80
|
+
d = Path(directory)
|
|
81
|
+
d.mkdir(parents=True, exist_ok=True)
|
|
82
|
+
out = {}
|
|
83
|
+
for t in self.tasks:
|
|
84
|
+
if t.key not in out:
|
|
85
|
+
out[t.key] = d / f"{t.key}.json"
|
|
86
|
+
write_file(out[t.key], contract.encode(t.to_json()))
|
|
87
|
+
return sorted(out.values())
|
|
88
|
+
|
|
89
|
+
def text(self, limit: int | None = 20) -> str:
|
|
90
|
+
"""Per stage its counts and a table of its first `limit` nodes that are not hits (None: all), then the
|
|
91
|
+
summary."""
|
|
92
|
+
rows, stages, shown = [], {}, {}
|
|
93
|
+
for n in self.doc["nodes"]:
|
|
94
|
+
c = stages.setdefault(n["stage"], {"hit": 0, "run": 0, "unknown": 0})
|
|
95
|
+
c[n["status"]] += 1
|
|
96
|
+
if n["status"] == "hit":
|
|
97
|
+
continue
|
|
98
|
+
shown[n["stage"]] = shown.get(n["stage"], 0) + 1
|
|
99
|
+
if limit is not None and shown[n["stage"]] > limit:
|
|
100
|
+
continue
|
|
101
|
+
reasons = ", ".join(_reason_text(r) for r in n["reasons"])
|
|
102
|
+
cpu = f"{n['cost']['cpuSeconds']:.1f}" if n.get("cost") else "-"
|
|
103
|
+
rows.append((n["status"], n["id"], cpu, reasons))
|
|
104
|
+
width = max((len(r[1]) for r in rows), default=4)
|
|
105
|
+
lines = [f"{'status':<8} {'task':<{width}} {'cpu s':>8} why"] if rows else []
|
|
106
|
+
lines += [f"{s:<8} {i:<{width}} {c:>8} {w}" for s, i, c, w in rows]
|
|
107
|
+
for stage, c in stages.items():
|
|
108
|
+
more = shown.get(stage, 0) - (limit if limit is not None else shown.get(stage, 0))
|
|
109
|
+
lines.append(f"{stage}: {c['hit']} hits, {c['run']} to run, {c['unknown']} unknown"
|
|
110
|
+
+ (f" ({more} more not listed)" if more > 0 else ""))
|
|
111
|
+
s = self.doc["summary"]
|
|
112
|
+
lines.append(f"{s['tasks']} tasks: {s['hit']} hits, {s['run']} to run, {s['unknown']} unknown")
|
|
113
|
+
gib = f"{s['peakBytes'] / (1 << 30):.2f} GiB" if s["peakBytes"] else "0"
|
|
114
|
+
lines.append(f"estimated: {s['cpuSeconds']:.1f} CPU s, peak memory {gib}"
|
|
115
|
+
+ (f" ({len(s['oversize'])} tasks above the budget run alone)" if s["oversize"] else ""))
|
|
116
|
+
for name, v in s["layouts"].items():
|
|
117
|
+
lines.append(f"layout {name}: write {v['write']} / remove {v['remove']} / keep {v['keep']}")
|
|
118
|
+
return "\n".join(lines) + "\n"
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def _reason_text(r: dict) -> str:
|
|
122
|
+
extra = {k: v for k, v in r.items() if k != "code"}
|
|
123
|
+
if not extra:
|
|
124
|
+
return r["code"]
|
|
125
|
+
return r["code"] + "{" + ",".join(f"{k}={_short(v)}" for k, v in extra.items()) + "}"
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _short(v) -> str:
|
|
129
|
+
s = v if isinstance(v, str) else contract.key_text(v)
|
|
130
|
+
return s[:12] if contract.is_sha256(s) else s
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def plan(stages: dict[str, Stage], pipeline: list[str], store, *, params: dict | None = None,
|
|
134
|
+
facts: dict | None = None, previous: dict | None = None, force=(), workers: int = 1,
|
|
135
|
+
memory: int | None = None, worker_bytes: int = WORKER_BYTES, layouts: dict | None = None) -> Plan:
|
|
136
|
+
"""The plan of running `pipeline` (stage names, in order) with `params` ({stage: parameters}) over `facts`.
|
|
137
|
+
`previous`: the run manifest to explain against; `force`: task ids or stage names ("*": all); `workers`,
|
|
138
|
+
`memory`, `worker_bytes`: as the orchestrator's, for the peak-memory estimate; `layouts`: {name: {write,
|
|
139
|
+
remove, keep}} (layout.diff)."""
|
|
140
|
+
params = params or {}
|
|
141
|
+
env = Env(store, facts)
|
|
142
|
+
force = set(force)
|
|
143
|
+
prev = {t["id"]: t for t in (previous or {}).get("tasks", [])}
|
|
144
|
+
nodes, edges, runnable = [], set(), []
|
|
145
|
+
for name in pipeline:
|
|
146
|
+
stage = stages[name]
|
|
147
|
+
p = stage.normalize(params.get(name))
|
|
148
|
+
forced = "*" in force or name in force
|
|
149
|
+
try:
|
|
150
|
+
subjects = stage.subjects(env)
|
|
151
|
+
except Pending as e:
|
|
152
|
+
nodes.append({"id": contract.task_id(name, "*"), "stage": name, "key": None, "status": "unknown",
|
|
153
|
+
"reasons": [{"code": "pending", "task": e.task}], "cost": None})
|
|
154
|
+
continue
|
|
155
|
+
for subject in subjects:
|
|
156
|
+
tid = contract.task_id(name, subject)
|
|
157
|
+
try:
|
|
158
|
+
edges.update((d, tid) for d in stage.depends(subject, env))
|
|
159
|
+
except Pending:
|
|
160
|
+
pass
|
|
161
|
+
try:
|
|
162
|
+
t = describe(stage, subject, p, env)
|
|
163
|
+
except Pending as e:
|
|
164
|
+
env.waiting(tid)
|
|
165
|
+
nodes.append({"id": tid, "stage": name, "key": None, "status": "unknown",
|
|
166
|
+
"reasons": [{"code": "pending", "task": e.task}], "cost": None})
|
|
167
|
+
continue
|
|
168
|
+
old = prev.get(tid)
|
|
169
|
+
if not (forced or tid in force) and store.has_result(t.key):
|
|
170
|
+
env.done(tid, t.key)
|
|
171
|
+
reasons = [{"code": "unchanged" if old is not None and old.get("key") == t.key else "cached"}]
|
|
172
|
+
status = "hit"
|
|
173
|
+
else:
|
|
174
|
+
env.waiting(tid)
|
|
175
|
+
reasons = _run_reasons(t, old, store, forced or tid in force)
|
|
176
|
+
status = "run"
|
|
177
|
+
runnable.append(t)
|
|
178
|
+
nodes.append({"id": tid, "stage": name, "key": t.key, "status": status, "reasons": reasons,
|
|
179
|
+
"cost": t.cost.to_json() if t.cost is not None else None})
|
|
180
|
+
return Plan(_doc(nodes, edges, runnable, workers, memory, worker_bytes, layouts or {}), runnable)
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def _run_reasons(t, old: dict | None, store, forced: bool) -> list[dict]:
|
|
184
|
+
if forced:
|
|
185
|
+
return [{"code": "forced"}]
|
|
186
|
+
if old is None:
|
|
187
|
+
return [{"code": "new-input"}]
|
|
188
|
+
out = []
|
|
189
|
+
if old["status"] == "failed":
|
|
190
|
+
out.append({"code": "previous-failed"})
|
|
191
|
+
if old.get("key") == t.key:
|
|
192
|
+
return out or [{"code": "result-missing"}]
|
|
193
|
+
prior = store.result(old["key"]) if old.get("key") else None
|
|
194
|
+
if prior is not None:
|
|
195
|
+
reasons = why(prior["keyParts"], t.key_parts()) or [{"code": "key-changed"}]
|
|
196
|
+
for r in reasons:
|
|
197
|
+
if r["code"] == "input-changed" and r["new"] is not None and t.input(r["role"]).name:
|
|
198
|
+
r["name"] = t.input(r["role"]).name
|
|
199
|
+
out += reasons
|
|
200
|
+
return out or [{"code": "previous-failed"}]
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def _doc(nodes: list, edges: set, runnable: list, workers: int, memory: int | None, worker_bytes: int,
|
|
204
|
+
layouts: dict) -> dict:
|
|
205
|
+
count = {s: sum(n["status"] == s for n in nodes) for s in ("hit", "run", "unknown")}
|
|
206
|
+
peaks = sorted((t.cost.peak_bytes if t.cost else 0 for t in runnable), reverse=True)
|
|
207
|
+
procs = max(1, workers)
|
|
208
|
+
budget = None if memory is None else max(0, memory - procs * worker_bytes)
|
|
209
|
+
peak = sum(peaks[:procs])
|
|
210
|
+
if budget is not None and peaks:
|
|
211
|
+
peak = min(peak, max(budget, peaks[0]))
|
|
212
|
+
peak += procs * worker_bytes if peaks else 0
|
|
213
|
+
oversize = sorted({t.id for t in runnable if budget is not None and t.cost and t.cost.peak_bytes > budget})
|
|
214
|
+
summary = {"tasks": len(nodes), **count,
|
|
215
|
+
"cpuSeconds": round(sum(t.cost.cpu_seconds for t in runnable if t.cost), 3),
|
|
216
|
+
"peakBytes": int(peak), "budget": memory, "oversize": oversize,
|
|
217
|
+
"layouts": {k: dict(v) for k, v in sorted(layouts.items())}}
|
|
218
|
+
return {"schema": contract.PLAN, "nodes": nodes, "edges": sorted([a, b] for a, b in edges), "summary": summary}
|
nnnotes/player.py
ADDED
|
@@ -0,0 +1,392 @@
|
|
|
1
|
+
"""Player-wide configuration from the game's boot data in a user-supplied APK.
|
|
2
|
+
|
|
3
|
+
`assets/bin/Data/data.unity3d` holds the global managers (graphics, quality,
|
|
4
|
+
player settings) and `globalgamemanagers.assets` with the URP pipeline assets,
|
|
5
|
+
renderer data, renderer features and post-process data. IL2CPP player builds
|
|
6
|
+
strip script typetrees there, so MonoBehaviours are read with typetrees that
|
|
7
|
+
ship with nnnotes: `typetrees/<Unity version>.json`, generated from the game's
|
|
8
|
+
managed assembly stubs by `scripts/gen_typetrees.py`. Each entry records the
|
|
9
|
+
type hash the serialized file stores for its class; a class without an entry,
|
|
10
|
+
a different type hash or a read that does not consume exactly the object's
|
|
11
|
+
serialized size raises UnsupportedVersion. `unity default resources`
|
|
12
|
+
(built-in meshes) is read from the APK as well.
|
|
13
|
+
"""
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import copy
|
|
17
|
+
import io
|
|
18
|
+
import json
|
|
19
|
+
import re
|
|
20
|
+
import struct
|
|
21
|
+
import zipfile
|
|
22
|
+
from importlib import resources
|
|
23
|
+
from pathlib import Path
|
|
24
|
+
|
|
25
|
+
import UnityPy
|
|
26
|
+
from UnityPy.helpers.TypeTreeNode import TypeTreeNode
|
|
27
|
+
|
|
28
|
+
from .config import ConfigError
|
|
29
|
+
from .unity import DEFAULT_RESOURCES, deref, external_path, is_pptr
|
|
30
|
+
|
|
31
|
+
DATA_IN_APK = "assets/bin/Data/data.unity3d"
|
|
32
|
+
DEFAULT_RESOURCES_IN_APK = "assets/bin/Data/Resources/unity default resources"
|
|
33
|
+
MANIFEST_IN_APK = "AndroidManifest.xml"
|
|
34
|
+
COLOR_SPACE = {0: "Gamma", 1: "Linear"}
|
|
35
|
+
HEADER = ("m_GameObject", "m_Script", "m_Enabled")
|
|
36
|
+
TYPETREE_FORMAT = 1
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class UnsupportedVersion(ConfigError):
|
|
40
|
+
"""The APK's boot data has a MonoBehaviour class that nnnotes has no matching typetree for."""
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
# ---------------------------------------------------------------- embedded typetrees
|
|
44
|
+
def typetree_key(assembly: str, namespace: str, cls: str) -> str:
|
|
45
|
+
"""Key of a class in a typetree file: `assembly|namespace|class`, as its MonoScript names them."""
|
|
46
|
+
return f"{assembly}|{namespace}|{cls}"
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def typetree_file(unity_version: str):
|
|
50
|
+
"""The embedded typetree file of a Unity version (an importlib.resources Traversable; it may not exist)."""
|
|
51
|
+
return resources.files(__package__).joinpath("typetrees", f"{unity_version}.json")
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def load_typetrees(unity_version: str) -> dict | None:
|
|
55
|
+
"""The embedded typetrees of a Unity version, or None when nnnotes has none for it."""
|
|
56
|
+
if not re.fullmatch(r"[0-9A-Za-z.]+", unity_version or ""):
|
|
57
|
+
return None
|
|
58
|
+
f = typetree_file(unity_version)
|
|
59
|
+
if not f.is_file():
|
|
60
|
+
return None
|
|
61
|
+
doc = json.loads(f.read_text(encoding="utf-8"))
|
|
62
|
+
if doc.get("format") != TYPETREE_FORMAT or doc.get("unityVersion") != unity_version:
|
|
63
|
+
raise ValueError(f"typetree file {unity_version}.json: unexpected format or Unity version")
|
|
64
|
+
return doc
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def typetree_node(rows: list) -> TypeTreeNode:
|
|
68
|
+
"""TypeTreeNode from [level, type, name, meta flag] rows (byte size and version 0, as generated)."""
|
|
69
|
+
return TypeTreeNode.from_list([TypeTreeNode(lv, ty, nm, 0, 0, m_MetaFlag=mf) for lv, ty, nm, mf in rows])
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def dump_typetrees(doc: dict) -> str:
|
|
73
|
+
"""Text of a typetree file: sorted keys, one class per line, LF."""
|
|
74
|
+
head = {k: v for k, v in doc.items() if k != "classes"}
|
|
75
|
+
lines = [json.dumps(head, sort_keys=True, ensure_ascii=False, separators=(",", ":"))[:-1] + ',"classes":{']
|
|
76
|
+
items = sorted(doc["classes"].items())
|
|
77
|
+
for i, (k, v) in enumerate(items):
|
|
78
|
+
lines.append(json.dumps(k, ensure_ascii=False) + ":"
|
|
79
|
+
+ json.dumps(v, sort_keys=True, ensure_ascii=False, separators=(",", ":"))
|
|
80
|
+
+ ("," if i + 1 < len(items) else ""))
|
|
81
|
+
lines.append("}}")
|
|
82
|
+
return "\n".join(lines) + "\n"
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def _manifest_attribute(data: bytes, attribute: str) -> tuple[str | None, int | None, int | None] | None:
|
|
86
|
+
"""An attribute of the <manifest> element of a binary AndroidManifest.xml: (its string, None when it has no
|
|
87
|
+
string value; the Res_value data type; the Res_value data). None when absent or unreadable."""
|
|
88
|
+
try:
|
|
89
|
+
pos, strings = 8, []
|
|
90
|
+
while pos + 8 <= len(data):
|
|
91
|
+
typ, hsize, size = struct.unpack_from("<HHI", data, pos)
|
|
92
|
+
if typ == 0x0001: # string pool
|
|
93
|
+
count, _styles, flags, start = struct.unpack_from("<IIII", data, pos + 8)
|
|
94
|
+
for off in struct.unpack_from(f"<{count}I", data, pos + hsize):
|
|
95
|
+
q = pos + start + off
|
|
96
|
+
if flags & 0x100: # UTF-8: UTF-16 length, UTF-8 length, bytes
|
|
97
|
+
q += 2 if data[q] & 0x80 else 1
|
|
98
|
+
n = data[q]
|
|
99
|
+
q += 1
|
|
100
|
+
if n & 0x80:
|
|
101
|
+
n = ((n & 0x7F) << 8) | data[q]
|
|
102
|
+
q += 1
|
|
103
|
+
strings.append(data[q:q + n].decode("utf-8"))
|
|
104
|
+
else: # UTF-16LE: length in code units, chars
|
|
105
|
+
n, = struct.unpack_from("<H", data, q)
|
|
106
|
+
q += 2
|
|
107
|
+
if n & 0x8000:
|
|
108
|
+
n = ((n & 0x7FFF) << 16) | struct.unpack_from("<H", data, q)[0]
|
|
109
|
+
q += 2
|
|
110
|
+
strings.append(data[q:q + 2 * n].decode("utf-16-le"))
|
|
111
|
+
elif typ == 0x0102: # start element
|
|
112
|
+
_ns, name, astart, asize, acount = struct.unpack_from("<IIHHH", data, pos + 16)
|
|
113
|
+
if strings[name] == "manifest":
|
|
114
|
+
for i in range(acount):
|
|
115
|
+
a = pos + 16 + astart + i * asize
|
|
116
|
+
_ans, aname, raw = struct.unpack_from("<III", data, a)
|
|
117
|
+
if strings[aname] == attribute:
|
|
118
|
+
text = strings[raw] if raw != 0xFFFFFFFF else None
|
|
119
|
+
if a + 20 > len(data): # no typed value
|
|
120
|
+
return text, None, None
|
|
121
|
+
_vsize, _res0, vtype, value = struct.unpack_from("<HBBI", data, a + 12)
|
|
122
|
+
return text, vtype, value
|
|
123
|
+
return None
|
|
124
|
+
if size < 8:
|
|
125
|
+
return None
|
|
126
|
+
pos += size
|
|
127
|
+
except (IndexError, struct.error, UnicodeDecodeError):
|
|
128
|
+
return None
|
|
129
|
+
return None
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def manifest_version_name(data: bytes) -> str | None:
|
|
133
|
+
"""`android:versionName` of a binary AndroidManifest.xml (None when absent or unreadable)."""
|
|
134
|
+
a = _manifest_attribute(data, "versionName")
|
|
135
|
+
return a[0] if a is not None else None
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def manifest_version_code(data: bytes) -> int | None:
|
|
139
|
+
"""`android:versionCode` of a binary AndroidManifest.xml: an integer (decimal or hex type), or a string of
|
|
140
|
+
digits (None when absent or unreadable)."""
|
|
141
|
+
a = _manifest_attribute(data, "versionCode")
|
|
142
|
+
if a is None:
|
|
143
|
+
return None
|
|
144
|
+
text, vtype, value = a
|
|
145
|
+
if vtype in (0x10, 0x11): # TYPE_INT_DEC, TYPE_INT_HEX
|
|
146
|
+
return value
|
|
147
|
+
return int(text) if text is not None and text.isdecimal() else None
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
class PlayerData:
|
|
151
|
+
def __init__(self, apk: Path):
|
|
152
|
+
with zipfile.ZipFile(apk) as z:
|
|
153
|
+
self.env = UnityPy.load(io.BytesIO(z.read(DATA_IN_APK)))
|
|
154
|
+
self.defaults = UnityPy.load(io.BytesIO(z.read(DEFAULT_RESOURCES_IN_APK)))
|
|
155
|
+
self.game_version = (manifest_version_name(z.read(MANIFEST_IN_APK))
|
|
156
|
+
if MANIFEST_IN_APK in z.namelist() else None)
|
|
157
|
+
self._by_type: dict[str, list] = {}
|
|
158
|
+
for o in self.env.objects:
|
|
159
|
+
self._by_type.setdefault(o.type.name, []).append(o)
|
|
160
|
+
self.unity_version = self._by_type["GraphicsSettings"][0].assets_file.unity_version
|
|
161
|
+
self.typetrees = load_typetrees(self.unity_version)
|
|
162
|
+
self._nodes: dict[str, tuple[TypeTreeNode, str]] = {}
|
|
163
|
+
self._clear_memos()
|
|
164
|
+
|
|
165
|
+
def _clear_memos(self) -> None:
|
|
166
|
+
"""Empty the memos of script, mono, shader, resource and graphics (an object is kept with its entry, so its
|
|
167
|
+
id is not reused while the entry exists; the objects belong to `env` / `defaults` anyway)."""
|
|
168
|
+
self._scripts: dict[int, tuple] = {} # id(object) -> (object, (assembly, namespace, class))
|
|
169
|
+
self._monos: dict[int, tuple] = {} # id(object) -> (object, fields)
|
|
170
|
+
self._shader_names: dict[str, list] | None = None
|
|
171
|
+
self._container: dict[str, list] | None = None
|
|
172
|
+
self._graphics: dict | None = None
|
|
173
|
+
|
|
174
|
+
# --- object access ---------------------------------------------------
|
|
175
|
+
def one(self, type_name: str):
|
|
176
|
+
objs = self._by_type.get(type_name, [])
|
|
177
|
+
if len(objs) != 1:
|
|
178
|
+
raise RuntimeError(f"{type_name}: {len(objs)} objects")
|
|
179
|
+
return objs[0]
|
|
180
|
+
|
|
181
|
+
def resource(self, path: str):
|
|
182
|
+
"""The object `Resources.Load(path)` returns (ResourceManager container, case-insensitive)."""
|
|
183
|
+
rm = self.one("ResourceManager")
|
|
184
|
+
if self._container is None: # lower-case path -> PPtrs, in container order
|
|
185
|
+
self._container = {}
|
|
186
|
+
for p, pp in rm.read_typetree()["m_Container"]:
|
|
187
|
+
self._container.setdefault(p.lower(), []).append(pp)
|
|
188
|
+
hits = self._container.get(path.lower(), [])
|
|
189
|
+
if len(hits) != 1:
|
|
190
|
+
raise KeyError(f"Resources path {path}: {len(hits)} entries")
|
|
191
|
+
o = self.deref(rm, hits[0])
|
|
192
|
+
if o is None:
|
|
193
|
+
raise KeyError(f"Resources path {path}: null reference")
|
|
194
|
+
return o
|
|
195
|
+
|
|
196
|
+
def shader(self, name: str, file: str | None = None):
|
|
197
|
+
"""The Shader object named `name` (parsed-form name, as Shader.Find sees it); `file` (serialized file
|
|
198
|
+
name, e.g. "unity_builtin_extra") narrows the search. Exactly one must match."""
|
|
199
|
+
if self._shader_names is None: # parsed-form name -> Shader objects, in object order
|
|
200
|
+
self._shader_names = {}
|
|
201
|
+
for o in self._by_type.get("Shader", []):
|
|
202
|
+
self._shader_names.setdefault(o.read().m_ParsedForm.m_Name, []).append(o)
|
|
203
|
+
hits = [o for o in self._shader_names.get(name, []) if file is None or o.assets_file.name == file]
|
|
204
|
+
if len(hits) != 1:
|
|
205
|
+
raise KeyError(f"shader {name!r}{f' in {file}' if file else ''}: {len(hits)} objects")
|
|
206
|
+
return hits[0]
|
|
207
|
+
|
|
208
|
+
def deref(self, owner, pptr: dict):
|
|
209
|
+
if not pptr["m_PathID"]:
|
|
210
|
+
return None
|
|
211
|
+
if external_path(owner, pptr) == DEFAULT_RESOURCES:
|
|
212
|
+
return self.defaults.files[next(iter(self.defaults.files))].objects[pptr["m_PathID"]]
|
|
213
|
+
return deref(owner, pptr)
|
|
214
|
+
|
|
215
|
+
def script(self, o) -> tuple[str, str, str]:
|
|
216
|
+
"""(assembly, namespace, class) of a MonoBehaviour from its raw m_Script PPtr (read once per object)."""
|
|
217
|
+
hit = self._scripts.get(id(o))
|
|
218
|
+
if hit is not None and hit[0] is o:
|
|
219
|
+
return hit[1]
|
|
220
|
+
fid, pid = struct.unpack_from("<iq", o.get_raw_data(), 16)
|
|
221
|
+
ms = self.deref(o, {"m_FileID": fid, "m_PathID": pid}).read()
|
|
222
|
+
r = ms.m_AssemblyName, ms.m_Namespace, ms.m_ClassName
|
|
223
|
+
self._scripts[id(o)] = (o, r)
|
|
224
|
+
return r
|
|
225
|
+
|
|
226
|
+
def mono(self, o) -> dict:
|
|
227
|
+
"""A MonoBehaviour's fields (header removed), read with the embedded typetree of its class; read once per
|
|
228
|
+
object, each call returns its own deep copy (callers may change what they get)."""
|
|
229
|
+
hit = self._monos.get(id(o))
|
|
230
|
+
if hit is None or hit[0] is not o:
|
|
231
|
+
hit = self._monos[id(o)] = (o, self._read_mono(o))
|
|
232
|
+
return copy.deepcopy(hit[1])
|
|
233
|
+
|
|
234
|
+
def _read_mono(self, o) -> dict:
|
|
235
|
+
"""A MonoBehaviour's fields (header removed), read with the embedded typetree of its class."""
|
|
236
|
+
asm, ns, cls = self.script(o)
|
|
237
|
+
key = typetree_key(asm, ns, cls)
|
|
238
|
+
if key not in self._nodes:
|
|
239
|
+
entry = (self.typetrees or {}).get("classes", {}).get(key)
|
|
240
|
+
if entry is None:
|
|
241
|
+
raise self.unsupported(asm, ns, cls, "no embedded typetree for this class" if self.typetrees
|
|
242
|
+
else f"no embedded typetrees for Unity {self.unity_version}")
|
|
243
|
+
self._nodes[key] = typetree_node(entry["nodes"]), entry["typeHash"]
|
|
244
|
+
node, type_hash = self._nodes[key]
|
|
245
|
+
st = o.serialized_type
|
|
246
|
+
if st is not None and st.old_type_hash.hex() != type_hash:
|
|
247
|
+
raise self.unsupported(asm, ns, cls, "its serialized type hash differs from the embedded typetree's")
|
|
248
|
+
try:
|
|
249
|
+
tt = o.read_typetree(node, check_read=True)
|
|
250
|
+
except Exception as e:
|
|
251
|
+
raise self.unsupported(asm, ns, cls, f"the embedded typetree does not fit the serialized data ({e})"
|
|
252
|
+
) from e
|
|
253
|
+
return {k: v for k, v in tt.items() if k not in HEADER}
|
|
254
|
+
|
|
255
|
+
def unsupported(self, asm: str, ns: str, cls: str, why: str) -> UnsupportedVersion:
|
|
256
|
+
made = f" (embedded typetrees from game version {self.typetrees['gameVersion']})" if self.typetrees else ""
|
|
257
|
+
return UnsupportedVersion(
|
|
258
|
+
f"MonoBehaviour {ns + '.' if ns else ''}{cls} ({asm}): {why}{made}. "
|
|
259
|
+
f"Game version {self.game_version or 'unknown'} (Unity {self.unity_version}) is not supported yet.")
|
|
260
|
+
|
|
261
|
+
def name_of(self, owner, pptr: dict) -> str | None:
|
|
262
|
+
"""`m_Name` of a referenced object (shader names come from the parsed form)."""
|
|
263
|
+
o = self.deref(owner, pptr)
|
|
264
|
+
if o is None:
|
|
265
|
+
return None
|
|
266
|
+
if o.type.name == "Shader":
|
|
267
|
+
return o.read().m_ParsedForm.m_Name
|
|
268
|
+
if o.type.name == "MonoBehaviour":
|
|
269
|
+
return self.mono(o).get("m_Name")
|
|
270
|
+
return o.read().m_Name
|
|
271
|
+
|
|
272
|
+
def names_in(self, owner, tt):
|
|
273
|
+
"""Typetree with every PPtr replaced by the referenced object's name."""
|
|
274
|
+
if is_pptr(tt):
|
|
275
|
+
return self.name_of(owner, tt)
|
|
276
|
+
if isinstance(tt, dict):
|
|
277
|
+
return {k: self.names_in(owner, v) for k, v in tt.items()}
|
|
278
|
+
if isinstance(tt, list):
|
|
279
|
+
return [self.names_in(owner, v) for v in tt]
|
|
280
|
+
return tt
|
|
281
|
+
|
|
282
|
+
# --- settings --------------------------------------------------------
|
|
283
|
+
def color_space(self) -> str:
|
|
284
|
+
"""PlayerSettings.m_ActiveColorSpace.
|
|
285
|
+
|
|
286
|
+
The built-in PlayerSettings typetree diverges from this build's layout
|
|
287
|
+
after the colour space field, so only the prefix up to it is read.
|
|
288
|
+
"""
|
|
289
|
+
o = self.one("PlayerSettings")
|
|
290
|
+
node = o._get_typetree_node()
|
|
291
|
+
names = [c.m_Name for c in node.m_Children]
|
|
292
|
+
i = names.index("m_ActiveColorSpace")
|
|
293
|
+
prefix = TypeTreeNode(node.m_Level, node.m_Type, node.m_Name, node.m_ByteSize,
|
|
294
|
+
node.m_Version, m_MetaFlag=node.m_MetaFlag,
|
|
295
|
+
m_Children=node.m_Children[:i + 1])
|
|
296
|
+
return COLOR_SPACE[o.read_typetree(prefix, check_read=False)["m_ActiveColorSpace"]]
|
|
297
|
+
|
|
298
|
+
def graphics(self) -> dict:
|
|
299
|
+
"""The render settings tree (_read_graphics), built once; each call returns its own deep copy."""
|
|
300
|
+
if self._graphics is None:
|
|
301
|
+
self._graphics = self._read_graphics()
|
|
302
|
+
return copy.deepcopy(self._graphics)
|
|
303
|
+
|
|
304
|
+
def _read_graphics(self) -> dict:
|
|
305
|
+
gs_o, qs_o = self.one("GraphicsSettings"), self.one("QualitySettings")
|
|
306
|
+
gs, qs = gs_o.read_typetree(), qs_o.read_typetree()
|
|
307
|
+
pipelines, renderers, post = {}, {}, {}
|
|
308
|
+
|
|
309
|
+
def pipeline(owner, pptr) -> str:
|
|
310
|
+
o = self.deref(owner, pptr)
|
|
311
|
+
tt = self.mono(o)
|
|
312
|
+
name = tt["m_Name"]
|
|
313
|
+
if name not in pipelines:
|
|
314
|
+
tt["m_RendererDataList"] = [renderer(o, p) for p in tt["m_RendererDataList"]]
|
|
315
|
+
pipelines[name] = self.names_in(o, tt)
|
|
316
|
+
return name
|
|
317
|
+
|
|
318
|
+
def renderer(owner, pptr) -> str:
|
|
319
|
+
o = self.deref(owner, pptr)
|
|
320
|
+
tt = self.mono(o)
|
|
321
|
+
name = tt["m_Name"]
|
|
322
|
+
if name not in renderers:
|
|
323
|
+
feats = []
|
|
324
|
+
for fp in tt["m_RendererFeatures"]:
|
|
325
|
+
fo = self.deref(o, fp)
|
|
326
|
+
asm, ns, cls = self.script(fo)
|
|
327
|
+
feats.append({"class": f"{ns}.{cls}" if ns else cls, "assembly": asm,
|
|
328
|
+
**self.names_in(fo, self.mono(fo))})
|
|
329
|
+
tt["m_RendererFeatures"] = feats
|
|
330
|
+
pp = tt.get("postProcessData")
|
|
331
|
+
if pp and pp["m_PathID"]:
|
|
332
|
+
po = self.deref(o, pp)
|
|
333
|
+
pt = self.mono(po)
|
|
334
|
+
post.setdefault(pt["m_Name"], self.names_in(po, pt))
|
|
335
|
+
tt["postProcessData"] = pt["m_Name"]
|
|
336
|
+
renderers[name] = self.names_in(o, tt)
|
|
337
|
+
return name
|
|
338
|
+
|
|
339
|
+
levels = []
|
|
340
|
+
for q in qs["m_QualitySettings"]:
|
|
341
|
+
q = dict(q)
|
|
342
|
+
q["customRenderPipeline"] = pipeline(qs_o, q["customRenderPipeline"])
|
|
343
|
+
levels.append(q)
|
|
344
|
+
return {
|
|
345
|
+
"colorSpace": self.color_space(),
|
|
346
|
+
"defaultPipeline": pipeline(gs_o, gs["m_CustomRenderPipeline"]),
|
|
347
|
+
"qualityLevels": levels,
|
|
348
|
+
"qualityPerPlatform": qs.get("m_PerPlatformDefaultQuality"),
|
|
349
|
+
"pipelines": pipelines,
|
|
350
|
+
"renderers": renderers,
|
|
351
|
+
"postProcessData": post,
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
def renderer_shaders(self, renderer_name: str) -> list:
|
|
355
|
+
"""Shader objects referenced by a renderer's features and its post-process data."""
|
|
356
|
+
out = []
|
|
357
|
+
for o in self._by_type.get("MonoBehaviour", []):
|
|
358
|
+
if self.script(o)[2] != "UniversalRendererData":
|
|
359
|
+
continue
|
|
360
|
+
tt = self.mono(o)
|
|
361
|
+
if tt["m_Name"] != renderer_name:
|
|
362
|
+
continue
|
|
363
|
+
owners = [(self.deref(o, fp)) for fp in tt["m_RendererFeatures"]]
|
|
364
|
+
if tt["postProcessData"]["m_PathID"]:
|
|
365
|
+
owners.append(self.deref(o, tt["postProcessData"]))
|
|
366
|
+
for fo in owners:
|
|
367
|
+
self._collect_shaders(fo, self.mono(fo), out)
|
|
368
|
+
return out
|
|
369
|
+
|
|
370
|
+
def post_textures(self, renderer_name: str, group: str) -> list:
|
|
371
|
+
"""Texture objects of a renderer's post-process data `textures.<group>` list (index order)."""
|
|
372
|
+
for o in self._by_type.get("MonoBehaviour", []):
|
|
373
|
+
if self.script(o)[2] != "UniversalRendererData":
|
|
374
|
+
continue
|
|
375
|
+
tt = self.mono(o)
|
|
376
|
+
if tt["m_Name"] != renderer_name:
|
|
377
|
+
continue
|
|
378
|
+
po = self.deref(o, tt["postProcessData"])
|
|
379
|
+
return [self.deref(po, p) for p in self.mono(po)["textures"][group]]
|
|
380
|
+
raise KeyError(f"renderer {renderer_name} not in player data")
|
|
381
|
+
|
|
382
|
+
def _collect_shaders(self, owner, tt, out: list):
|
|
383
|
+
if is_pptr(tt):
|
|
384
|
+
o = self.deref(owner, tt)
|
|
385
|
+
if o is not None and o.type.name == "Shader" and o not in out:
|
|
386
|
+
out.append(o)
|
|
387
|
+
elif isinstance(tt, dict):
|
|
388
|
+
for v in tt.values():
|
|
389
|
+
self._collect_shaders(owner, v, out)
|
|
390
|
+
elif isinstance(tt, list):
|
|
391
|
+
for v in tt:
|
|
392
|
+
self._collect_shaders(owner, v, out)
|