sonolus.py 0.1.8__py3-none-any.whl → 0.2.0__py3-none-any.whl

sonolus/build/collection.py

@@ -36,7 +36,7 @@ SINGULAR_CATEGORY_NAMES: dict[Category, str] = {
 }
 BASE_PATH = "/sonolus/"
 RESERVED_FILENAMES = {"info", "list"}
-LOCALIZED_KEYS = {"title", "subtitle", "author", "description"}
+LOCALIZED_KEYS = {"title", "subtitle", "author", "description", "artists"}
 CATEGORY_SORT_ORDER = {
     "levels": 0,
     "engines": 1,
@@ -69,10 +69,10 @@ class Collection:
     def add_item(self, category: Category, name: str, item: Any) -> None:
         self.categories.setdefault(category, {})[name] = self._make_item_details(item)
 
-    @staticmethod
-    def _make_item_details(item: dict[str, Any]) -> dict[str, Any]:
+    @classmethod
+    def _make_item_details(cls, item: dict[str, Any]) -> dict[str, Any]:
         return {
-            "item": item,
+            "item": cls._localize_item(item),
             "actions": [],
             "hasCommunity": False,
             "leaderboards": [],
@@ -81,16 +81,7 @@ class Collection:
 
     @staticmethod
     def _load_data(value: Asset) -> bytes:
-        match value:
-            case str() if value.startswith(("http://", "https://")):
-                with urllib.request.urlopen(value) as response:
-                    return response.read()
-            case PathLike():
-                return Path(value).read_bytes()
-            case bytes():
-                return value
-            case _:
-                raise TypeError("value must be a URL, a path, or bytes")
+        return load_asset(value)
 
     def add_asset(self, value: Asset, /) -> Srl:
         data = self._load_data(value)
@@ -123,7 +114,7 @@ class Collection:
                     continue
 
                 try:
-                    item_data = json.loads(item_json_path.read_text())
+                    item_data = json.loads(item_json_path.read_text(encoding="utf-8"))
                 except json.JSONDecodeError:
                     continue
 
@@ -149,19 +140,32 @@ class Collection:
 
                 self.add_item(category_name, item_dir.name, item_data)
 
-    @staticmethod
-    def _localize_item(item: dict[str, Any]) -> dict[str, Any]:
+    @classmethod
+    def _localize_item(cls, item: dict[str, Any]) -> dict[str, Any]:
         localized_item = item.copy()
         for key in LOCALIZED_KEYS:
-            match localized_item.get(key):
-                case {"en": localized_value}:
-                    localized_item[key] = localized_value
-                case {**other_languages} if other_languages:
-                    localized_item[key] = localized_item[key][min(other_languages)]
-                case _:
-                    localized_item[key] = ""
+            if key not in localized_item:
+                continue
+            localized_item[key] = cls._localize_text(localized_item[key])
+        if "tags" in localized_item:
+            localized_item["tags"] = [
+                {**tag, "title": cls._localize_text(tag["title"])} for tag in localized_item["tags"]
+            ]
+        localized_item.pop("meta", None)
         return localized_item
 
+    @staticmethod
+    def _localize_text(text: str | dict[str, str]) -> str:
+        match text:
+            case str():
+                return text
+            case {"en": localized_text}:
+                return localized_text
+            case {**other_languages} if other_languages:
+                return text[min(other_languages)]
+            case _:
+                return ""
+
     def _group_zip_entries_by_directory(self, file_list: list[zipfile.ZipInfo]) -> dict[str, list[zipfile.ZipInfo]]:
         files_by_dir: dict[str, list[zipfile.ZipInfo]] = {}
 
@@ -207,7 +211,7 @@ class Collection:
     ) -> None:
         for zip_entry in zip_entries:
             try:
-                item_details = json.loads(zf.read(zip_entry))
+                item_details = json.loads(zf.read(zip_entry).decode("utf-8"))
             except json.JSONDecodeError:
                 continue
 
@@ -220,11 +224,28 @@ class Collection:
                 self.categories[dir_name][item_name] = item_details
 
     def write(self, path: Asset) -> None:
+        self.link()
         base_dir = self._create_base_directory(path)
         self._write_main_info(base_dir)
         self._write_category_items(base_dir)
         self._write_repository_items(base_dir)
 
+    def link(self):
+        for level_details in self.categories.get("levels", {}).values():
+            level = level_details["item"]
+            if isinstance(level["engine"], str):
+                level["engine"] = self.get_item("engines", level["engine"])
+            for key, category in (
+                ("useSkin", "skins"),
+                ("useBackground", "backgrounds"),
+                ("useEffect", "effects"),
+                ("useParticle", "particles"),
+            ):
+                use_item = level[key]
+                if "item" not in use_item:
+                    continue
+                use_item["item"] = self.get_item(category, use_item["item"])
+
     def _create_base_directory(self, path: Asset) -> Path:
         base_dir = Path(path) / BASE_PATH.strip("/")
         base_dir.mkdir(parents=True, exist_ok=True)
@@ -280,7 +301,7 @@ class Collection:
 
     @staticmethod
     def _write_json(path: Path, content: Any) -> None:
-        path.write_text(json.dumps(content))
+        path.write_text(json.dumps(content), encoding="utf-8")
 
     def update(self, other: Collection) -> None:
         self.repository.update(other.repository)
@@ -291,3 +312,24 @@ class Collection:
 class Srl(TypedDict):
     hash: str
     url: str
+
+
+def load_asset(value: Asset) -> bytes:
+    match value:
+        case str() if value.startswith(("http://", "https://")):
+            headers = {
+                "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+                "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+                "Accept": "*/*",
+                "Accept-Language": "en-US,en;q=0.9",
+                "Connection": "keep-alive",
+            }
+            request = urllib.request.Request(value, headers=headers)
+            with urllib.request.urlopen(request) as response:
+                return response.read()
+        case PathLike():
+            return Path(value).read_bytes()
+        case bytes():
+            return value
+        case _:
+            raise TypeError("value must be a URL, a path, or bytes")

sonolus/build/compile.py

@@ -1,11 +1,13 @@
-from collections.abc import Callable
+from collections.abc import Callable, Sequence
+from concurrent.futures import Executor, Future
 
 from sonolus.backend.finalize import cfg_to_engine_node
 from sonolus.backend.ir import IRConst, IRInstr
 from sonolus.backend.mode import Mode
 from sonolus.backend.ops import Op
 from sonolus.backend.optimize.flow import BasicBlock
-from sonolus.backend.optimize.optimize import optimize_and_allocate
+from sonolus.backend.optimize.optimize import STANDARD_PASSES
+from sonolus.backend.optimize.passes import CompilerPass, run_passes
 from sonolus.backend.visitor import compile_and_call
 from sonolus.build.node import OutputNodeGenerator
 from sonolus.script.archetype import _BaseArchetype
@@ -27,54 +29,109 @@ def compile_mode(
     rom: ReadOnlyMemory,
     archetypes: list[type[_BaseArchetype]] | None,
     global_callbacks: list[tuple[CallbackInfo, Callable]] | None,
+    passes: Sequence[CompilerPass] | None = None,
+    thread_pool: Executor | None = None,
 ) -> dict:
+    if passes is None:
+        passes = STANDARD_PASSES
+
     global_state = GlobalContextState(
-        mode, {a: i for i, a in enumerate(archetypes)} if archetypes is not None else None, rom
+        mode,
+        {a: i for i, a in enumerate(archetypes)} if archetypes is not None else None,
+        rom,
     )
     nodes = OutputNodeGenerator()
     results = {}
+
+    def process_callback(
+        cb_info: CallbackInfo,
+        cb: Callable,
+        arch: type[_BaseArchetype] | None = None,
+    ) -> tuple[str, int] | tuple[str, dict]:
+        """Compile a single callback to a node.
+
+        Returns either:
+        - (cb_info.name, node_index) for global callbacks, or
+        - (cb_info.name, {"index": node_index, "order": cb_order}) for archetype callbacks.
+        """
+        cfg = callback_to_cfg(global_state, cb, cb_info.name, arch)
+        cfg = run_passes(cfg, passes)
+        node = cfg_to_engine_node(cfg)
+        node_index = nodes.add(node)
+
+        if arch is not None:
+            cb_order = getattr(cb, "_callback_order_", 0)
+            return cb_info.name, {"index": node_index, "order": cb_order}
+        else:
+            return cb_info.name, node_index
+
+    all_futures = {}
+    archetype_entries = []
+
     if archetypes is not None:
-        archetype_entries = []
         for archetype in archetypes:
+            archetype._init_fields()
+
             archetype_data = {
                 "name": archetype.name,
                 "hasInput": archetype.is_scored,
                 "imports": [{"name": name, "index": index} for name, index in archetype._imported_keys_.items()],
             }
             if mode == Mode.PLAY:
-                archetype_data["exports"] = [
-                    {"name": name, "index": index} for name, index in archetype._exported_keys_.items()
-                ]
-            for cb_name, cb_info in archetype._supported_callbacks_.items():
-                cb = getattr(archetype, cb_name)
-                if cb in archetype._default_callbacks_:
-                    continue
-                cb_order = getattr(cb, "_callback_order_", 0)
-                if not cb_info.supports_order and cb_order != 0:
-                    raise ValueError(f"Callback '{cb_name}' does not support a non-zero order")
-                cfg = callback_to_cfg(global_state, cb, cb_info.name, archetype)
-                cfg = optimize_and_allocate(cfg)
-                node = cfg_to_engine_node(cfg)
-                node_index = nodes.add(node)
-                archetype_data[cb_info.name] = {
-                    "index": node_index,
-                    "order": cb_order,
-                }
+                archetype_data["exports"] = [*archetype._exported_keys_]
+
+            callback_items = [
+                (cb_name, cb_info, getattr(archetype, cb_name))
+                for cb_name, cb_info in archetype._supported_callbacks_.items()
+                if getattr(archetype, cb_name) not in archetype._default_callbacks_
+            ]
+
+            if thread_pool is not None:
+                for cb_name, cb_info, cb in callback_items:
+                    cb_order = getattr(cb, "_callback_order_", 0)
+                    if not cb_info.supports_order and cb_order != 0:
+                        raise ValueError(f"Callback '{cb_name}' does not support a non-zero order")
+                    f: Future = thread_pool.submit(process_callback, cb_info, cb, archetype)
+                    all_futures[f] = ("archetype", archetype_data, cb_name)
+            else:
+                for cb_name, cb_info, cb in callback_items:
+                    cb_order = getattr(cb, "_callback_order_", 0)
+                    if not cb_info.supports_order and cb_order != 0:
+                        raise ValueError(f"Callback '{cb_name}' does not support a non-zero order")
+                    cb_name, result_data = process_callback(cb_info, cb, archetype)
+                    archetype_data[cb_name] = result_data
+
             archetype_entries.append(archetype_data)
-        results["archetypes"] = archetype_entries
-    if global_callbacks is not None:
+
+    if global_callbacks is not None and thread_pool is not None:
+        for cb_info, cb in global_callbacks:
+            f: Future = thread_pool.submit(process_callback, cb_info, cb, None)
+            all_futures[f] = ("global", None, cb_info.name)
+
+    if thread_pool is not None:
+        for f, (callback_type, archetype_data, cb_name) in all_futures.items():
+            cb_name, result_data = f.result()
+            if callback_type == "archetype":
+                archetype_data[cb_name] = result_data
+            else:  # global callback
+                results[cb_name] = result_data
+    elif global_callbacks is not None:
         for cb_info, cb in global_callbacks:
-            cfg = callback_to_cfg(global_state, cb, cb_info.name)
-            cfg = optimize_and_allocate(cfg)
-            node = cfg_to_engine_node(cfg)
-            node_index = nodes.add(node)
-            results[cb_info.name] = node_index
+            cb_name, node_index = process_callback(cb_info, cb, None)
+            results[cb_name] = node_index
+
+    if archetypes is not None:
+        results["archetypes"] = archetype_entries
+
     results["nodes"] = nodes.get()
     return results
 
 
 def callback_to_cfg(
-    global_state: GlobalContextState, callback: Callable, name: str, archetype: type[_BaseArchetype] | None = None
+    global_state: GlobalContextState,
+    callback: Callable,
+    name: str,
+    archetype: type[_BaseArchetype] | None = None,
 ) -> BasicBlock:
     callback_state = CallbackContextState(name)
     context = Context(global_state, callback_state)

sonolus/build/engine.py

@@ -1,8 +1,12 @@
 import gzip
 import json
 import struct
+import sys
 from collections.abc import Callable
+from concurrent.futures import Executor
+from concurrent.futures.thread import ThreadPoolExecutor
 from dataclasses import dataclass
+from os import process_cpu_count
 from pathlib import Path
 
 from sonolus.backend.mode import Mode
@@ -10,7 +14,7 @@ from sonolus.build.compile import compile_mode
 from sonolus.script.archetype import _BaseArchetype
 from sonolus.script.bucket import Buckets
 from sonolus.script.effect import Effects
-from sonolus.script.engine import EngineData
+from sonolus.script.engine import EngineData, empty_play_mode, empty_preview_mode, empty_tutorial_mode, empty_watch_mode
 from sonolus.script.instruction import (
     TutorialInstructionIcons,
     TutorialInstructions,
@@ -24,6 +28,7 @@ from sonolus.script.internal.callbacks import (
 from sonolus.script.internal.context import ReadOnlyMemory
 from sonolus.script.options import Options
 from sonolus.script.particle import Particles
+from sonolus.script.project import BuildConfig
 from sonolus.script.sprite import Skin
 from sonolus.script.ui import UiConfig
 
@@ -48,43 +53,132 @@ class PackagedEngine:
         (path / "EngineTutorialData").write_bytes(self.tutorial_data)
         (path / "EngineRom").write_bytes(self.rom)
 
+    def read(self, path: Path):
+        self.configuration = (path / "EngineConfiguration").read_bytes()
+        self.play_data = (path / "EnginePlayData").read_bytes()
+        self.watch_data = (path / "EngineWatchData").read_bytes()
+        self.preview_data = (path / "EnginePreviewData").read_bytes()
+        self.tutorial_data = (path / "EngineTutorialData").read_bytes()
+        self.rom = (path / "EngineRom").read_bytes()
 
-def package_engine(engine: EngineData):
+
+def no_gil() -> bool:
+    return sys.version_info >= (3, 13) and not sys._is_gil_enabled()
+
+
+def package_engine(
+    engine: EngineData,
+    config: BuildConfig | None = None,
+):
+    config = config or BuildConfig()
     rom = ReadOnlyMemory()
     configuration = build_engine_configuration(engine.options, engine.ui)
-    play_data = build_play_mode(
-        archetypes=engine.play.archetypes,
-        skin=engine.play.skin,
-        effects=engine.play.effects,
-        particles=engine.play.particles,
-        buckets=engine.play.buckets,
-        rom=rom,
-    )
-    watch_data = build_watch_mode(
-        archetypes=engine.watch.archetypes,
-        skin=engine.watch.skin,
-        effects=engine.watch.effects,
-        particles=engine.watch.particles,
-        buckets=engine.watch.buckets,
-        rom=rom,
-        update_spawn=engine.watch.update_spawn,
-    )
-    preview_data = build_preview_mode(
-        archetypes=engine.preview.archetypes,
-        skin=engine.preview.skin,
-        rom=rom,
-    )
-    tutorial_data = build_tutorial_mode(
-        skin=engine.tutorial.skin,
-        effects=engine.tutorial.effects,
-        particles=engine.tutorial.particles,
-        instructions=engine.tutorial.instructions,
-        instruction_icons=engine.tutorial.instruction_icons,
-        preprocess=engine.tutorial.preprocess,
-        navigate=engine.tutorial.navigate,
-        update=engine.tutorial.update,
-        rom=rom,
-    )
+    if no_gil():
+        thread_pool = ThreadPoolExecutor(process_cpu_count() or 1)
+    else:
+        thread_pool = None
+
+    play_mode = engine.play if config.build_play else empty_play_mode()
+    watch_mode = engine.watch if config.build_watch else empty_watch_mode()
+    preview_mode = engine.preview if config.build_preview else empty_preview_mode()
+    tutorial_mode = engine.tutorial if config.build_tutorial else empty_tutorial_mode()
+
+    if thread_pool is not None:
+        futures = {
+            "play": thread_pool.submit(
+                build_play_mode,
+                archetypes=play_mode.archetypes,
+                skin=play_mode.skin,
+                effects=play_mode.effects,
+                particles=play_mode.particles,
+                buckets=play_mode.buckets,
+                rom=rom,
+                config=config,
+                thread_pool=thread_pool,
+            ),
+            "watch": thread_pool.submit(
+                build_watch_mode,
+                archetypes=watch_mode.archetypes,
+                skin=watch_mode.skin,
+                effects=watch_mode.effects,
+                particles=watch_mode.particles,
+                buckets=watch_mode.buckets,
+                rom=rom,
+                update_spawn=watch_mode.update_spawn,
+                config=config,
+                thread_pool=thread_pool,
+            ),
+            "preview": thread_pool.submit(
+                build_preview_mode,
+                archetypes=preview_mode.archetypes,
+                skin=preview_mode.skin,
+                rom=rom,
+                config=config,
+                thread_pool=thread_pool,
+            ),
+            "tutorial": thread_pool.submit(
+                build_tutorial_mode,
+                skin=tutorial_mode.skin,
+                effects=tutorial_mode.effects,
+                particles=tutorial_mode.particles,
+                instructions=tutorial_mode.instructions,
+                instruction_icons=tutorial_mode.instruction_icons,
+                preprocess=tutorial_mode.preprocess,
+                navigate=tutorial_mode.navigate,
+                update=tutorial_mode.update,
+                rom=rom,
+                config=config,
+                thread_pool=thread_pool,
+            ),
+        }
+
+        play_data = futures["play"].result()
+        watch_data = futures["watch"].result()
+        preview_data = futures["preview"].result()
+        tutorial_data = futures["tutorial"].result()
+    else:
+        play_data = build_play_mode(
+            archetypes=play_mode.archetypes,
+            skin=play_mode.skin,
+            effects=play_mode.effects,
+            particles=play_mode.particles,
+            buckets=play_mode.buckets,
+            rom=rom,
+            config=config,
+            thread_pool=None,
+        )
+        watch_data = build_watch_mode(
+            archetypes=watch_mode.archetypes,
+            skin=watch_mode.skin,
+            effects=watch_mode.effects,
+            particles=watch_mode.particles,
+            buckets=watch_mode.buckets,
+            rom=rom,
+            update_spawn=watch_mode.update_spawn,
+            config=config,
+            thread_pool=None,
+        )
+        preview_data = build_preview_mode(
+            archetypes=preview_mode.archetypes,
+            skin=preview_mode.skin,
+            rom=rom,
+            config=config,
+            thread_pool=None,
+        )
+        tutorial_data = build_tutorial_mode(
+            skin=tutorial_mode.skin,
+            effects=tutorial_mode.effects,
+            particles=tutorial_mode.particles,
+            instructions=tutorial_mode.instructions,
+            instruction_icons=tutorial_mode.instruction_icons,
+            preprocess=tutorial_mode.preprocess,
+            navigate=tutorial_mode.navigate,
+            update=tutorial_mode.update,
+            rom=rom,
+            config=config,
+            thread_pool=None,
+        )
+
     return PackagedEngine(
         configuration=package_output(configuration),
         play_data=package_output(play_data),
@@ -112,9 +206,18 @@ def build_play_mode(
     particles: Particles,
     buckets: Buckets,
     rom: ReadOnlyMemory,
+    config: BuildConfig,
+    thread_pool: Executor | None = None,
 ):
     return {
-        **compile_mode(mode=Mode.PLAY, rom=rom, archetypes=archetypes, global_callbacks=None),
+        **compile_mode(
+            mode=Mode.PLAY,
+            rom=rom,
+            archetypes=archetypes,
+            global_callbacks=None,
+            passes=config.passes,
+            thread_pool=thread_pool,
+        ),
         "skin": build_skin(skin),
         "effect": build_effects(effects),
         "particle": build_particles(particles),
@@ -130,10 +233,17 @@ def build_watch_mode(
     buckets: Buckets,
     rom: ReadOnlyMemory,
     update_spawn: Callable[[], float],
+    config: BuildConfig,
+    thread_pool: Executor | None = None,
 ):
     return {
         **compile_mode(
-            mode=Mode.WATCH, rom=rom, archetypes=archetypes, global_callbacks=[(update_spawn_callback, update_spawn)]
+            mode=Mode.WATCH,
+            rom=rom,
+            archetypes=archetypes,
+            global_callbacks=[(update_spawn_callback, update_spawn)],
+            passes=config.passes,
+            thread_pool=thread_pool,
         ),
         "skin": build_skin(skin),
         "effect": build_effects(effects),
@@ -146,9 +256,18 @@ def build_preview_mode(
     archetypes: list[type[_BaseArchetype]],
     skin: Skin,
     rom: ReadOnlyMemory,
+    config: BuildConfig,
+    thread_pool: Executor | None = None,
 ):
     return {
-        **compile_mode(mode=Mode.PREVIEW, rom=rom, archetypes=archetypes, global_callbacks=None),
+        **compile_mode(
+            mode=Mode.PREVIEW,
+            rom=rom,
+            archetypes=archetypes,
+            global_callbacks=None,
+            passes=config.passes,
+            thread_pool=thread_pool,
+        ),
         "skin": build_skin(skin),
     }
 
@@ -160,9 +279,11 @@ def build_tutorial_mode(
     instructions: TutorialInstructions,
     instruction_icons: TutorialInstructionIcons,
     preprocess: Callable[[], None],
-    navigate: Callable[[int], None],
+    navigate: Callable[[], None],
     update: Callable[[], None],
     rom: ReadOnlyMemory,
+    config: BuildConfig,
+    thread_pool: Executor | None = None,
 ):
     return {
         **compile_mode(
@@ -174,6 +295,8 @@ def build_tutorial_mode(
                 (navigate_callback, navigate),
                 (update_callback, update),
             ],
+            passes=config.passes,
+            thread_pool=thread_pool,
         ),
         "skin": build_skin(skin),
         "effect": build_effects(effects),
@@ -183,7 +306,10 @@ def build_tutorial_mode(
 
 
 def build_skin(skin: Skin) -> JsonValue:
-    return {"sprites": [{"name": name, "id": i} for i, name in enumerate(skin._sprites_)]}
+    return {
+        "renderMode": skin.render_mode,
+        "sprites": [{"name": name, "id": i} for i, name in enumerate(skin._sprites_)],
+    }
 
 
 def build_effects(effects: Effects) -> JsonValue:

sonolus/build/level.py

@@ -11,11 +11,12 @@ def package_level_data(
 def build_level_data(
     level_data: LevelData,
 ):
-    level_refs = {entity: i for i, entity in enumerate(level_data.entities)}
+    level_refs = {entity: f"{i}_{entity.name}" for i, entity in enumerate(level_data.entities)}
     return {
         "bgmOffset": level_data.bgm_offset,
         "entities": [
             {
+                "name": level_refs[entity],
                 "archetype": entity.name,
                 "data": entity._level_data_entries(level_refs),
             }

sonolus/build/node.py

@@ -1,3 +1,4 @@
+from threading import Lock
 from typing import TypedDict
 
 from sonolus.backend.node import ConstantNode, EngineNode, FunctionNode
@@ -15,12 +16,18 @@ class FunctionOutputNode(TypedDict):
 class OutputNodeGenerator:
     nodes: list[ValueOutputNode | FunctionOutputNode]
     indexes: dict[EngineNode, int]
+    lock: Lock
 
     def __init__(self):
         self.nodes = []
         self.indexes = {}
+        self.lock = Lock()
 
     def add(self, node: EngineNode):
+        with self.lock:
+            return self._add(node)
+
+    def _add(self, node: EngineNode):
         if node in self.indexes:
             return self.indexes[node]
 
@@ -31,7 +38,7 @@ class OutputNodeGenerator:
                 self.indexes[node] = index
                 return index
             case FunctionNode(func, args):
-                arg_indexes = [self.add(arg) for arg in args]
+                arg_indexes = [self._add(arg) for arg in args]
                 index = len(self.nodes)
                 self.nodes.append({"func": func.value, "args": arg_indexes})
                 self.indexes[node] = index