splat64 0.36.4__tar.gz → 0.37.1__tar.gz

{splat64-0.36.4 → splat64-0.37.1}/CHANGELOG.md

@@ -1,12 +1,31 @@
 # splat Release Notes
 
+### 0.37.1
+
+* Fix `make_full_disasm_for_code` not extracting data-only TUs.
+* Fix `make_full_disasm_for_code` and `disassemble_all` combo
+  * Fixes not writing other sections to asm file when both options are enabled.
+
+### 0.37.0
+
+* Add define check to allow using `macro.inc` instead of `labels.inc` in `include_asm.h`.
+  * The macro must be `INCLUDE_ASM_USE_MACRO_INC` and have a non zero value.
+* New option: `generated_asm_macros_directory`
+  * Allow changing the directory for the generated assembly macros files.
+* Add workaround for the short loop bug that happens in SN PS2 compilers.
+  * This workaround is only used when the selected compiler is `EEGCC`.
+    * This can be manually overriden with the option `align_on_branch_labels`.
+  * Emit an `.align` directive in branch labels to avoid triggering the bug.
+  * The bug in question adds a non-zero number of nops as a workaround for a hardware defect, producing non-matching builds.
+
 ### 0.36.4
 
 * Fix missing dollar signs (`$`) on specific ps2 registers on the extra assembly files generated by `make_full_disasm_for_code`.
 * Pin versions of dependencies to avoid breaking because they broke.
 * Add more checks for texture and palette segments to have a correct size in their yaml entries, ensuring their `width`/`height` or `size` attributes match the sizes relative to other segments.
 * Fix `gfx` segments sometimes not picking up properly the corresponding typed reference.
-* * Fix sometimes picking a name symbol from a different segment instead of the owned segment when both symbols have the same address but are from completely different segments.
+* Fix sometimes picking a name symbol from a different segment instead of the owned segment when both symbols have the same address but are from completely different segments.
+* Add PS2 compatibility to `create_config`.
 
 ### 0.36.3
 

{splat64-0.36.4 → splat64-0.37.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: splat64
-Version: 0.36.4
+Version: 0.37.1
 Summary: A binary splitting tool to assist with decompilation and modding projects
 Project-URL: Repository, https://github.com/ethteck/splat
 Project-URL: Issues, https://github.com/ethteck/splat/issues
@@ -42,7 +42,7 @@ Requires-Dist: n64img>=0.3.3; extra == 'dev'
 Requires-Dist: pygfxd>=1.0.5; extra == 'dev'
 Requires-Dist: rabbitizer<2.0.0,>=1.12.0; extra == 'dev'
 Requires-Dist: ruff; extra == 'dev'
-Requires-Dist: spimdisasm<2.0.0,>=1.38.0; extra == 'dev'
+Requires-Dist: spimdisasm<2.0.0,>=1.39.0; extra == 'dev'
 Requires-Dist: types-colorama; extra == 'dev'
 Requires-Dist: types-pyyaml; extra == 'dev'
 Provides-Extra: mips
@@ -50,7 +50,7 @@ Requires-Dist: crunch64<1.0.0,>=0.5.1; extra == 'mips'
 Requires-Dist: n64img>=0.3.3; extra == 'mips'
 Requires-Dist: pygfxd>=1.0.5; extra == 'mips'
 Requires-Dist: rabbitizer<2.0.0,>=1.12.0; extra == 'mips'
-Requires-Dist: spimdisasm<2.0.0,>=1.38.0; extra == 'mips'
+Requires-Dist: spimdisasm<2.0.0,>=1.39.0; extra == 'mips'
 Description-Content-Type: text/markdown
 
 # splat
@@ -76,7 +76,7 @@ The brackets corresponds to the optional dependencies to install while installin
 If you use a `requirements.txt` file in your repository, then you can add this library with the following line:
 
 ```txt
-splat64[mips]>=0.36.4,<1.0.0
+splat64[mips]>=0.37.1,<1.0.0
 ```
 
 ### Optional dependencies

{splat64-0.36.4 → splat64-0.37.1}/README.md

@@ -21,7 +21,7 @@ The brackets corresponds to the optional dependencies to install while installin
 If you use a `requirements.txt` file in your repository, then you can add this library with the following line:
 
 ```txt
-splat64[mips]>=0.36.4,<1.0.0
+splat64[mips]>=0.37.1,<1.0.0
 ```
 
 ### Optional dependencies

{splat64-0.36.4 → splat64-0.37.1}/docs/Configuration.md

@@ -147,6 +147,12 @@ Allows configuring the style used by the generated `INCLUDE_ASM` macro. It curre
 
 Defaults to `default`.
 
+### generated_asm_macros_directory
+
+Change the directory where the assembly macros files are written to.
+
+Defaults to `include`.
+
 ### o_as_suffix
 
 Used to determine the file extension of the built files that will be listed on the linker script.
@@ -842,6 +848,16 @@ Emit a full `.s` file for each `c`/`cpp` segment containing disassembly of .text
 
 Can be used to generate "target" or "expected" objects for asm diffing.
 
+To generate `.s` files for data-only TUs (or rodata/bss only, or any other combination that doesn't have a text section) a dummy `c` or `cpp` section with the same name as the data section must be defined in the yaml with an start value of `auto` instead of a number. For example:
+
+```yaml
+      - [auto, c, boot/rom_offsets]
+
+      # -- Other c or data subsegments --
+
+      - [0x00F340, .data, boot/rom_offsets]
+```
+
 ### global_vram_start and global_vram_end
 
 Allow specifying that the global memory range may be larger than what was automatically detected.
@@ -932,3 +948,21 @@ use_legacy_include_asm: True
 
 #### Default
 `False`
+
+## align_on_branch_labels
+
+If enabled emit no-op alignment directives on all branch labels.
+
+This is useful as a workaround for the "short loop bug" present in SN PS2 compilers, in which the compiler may decide to insert extra `nop` instructions due to a hardware defect, producing non-matching builds.
+
+This option is enabled automatically if the selected compiler is known to have this issue, but this option allows to override the default selection in case this setting ends up producing problems.
+
+#### Usage
+
+```yaml
+align_on_branch_labels: False
+```
+
+#### Default
+
+`True` if [`compiler`](#compiler) is set to `EEGCC`, `False` otherwise

{splat64-0.36.4 → splat64-0.37.1}/pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "splat64"
 # Should be synced with src/splat/__init__.py
-version = "0.36.4"
+version = "0.37.1"
 description = "A binary splitting tool to assist with decompilation and modding projects"
 readme = "README.md"
 license = {file = "LICENSE"}
@@ -20,7 +20,7 @@ dependencies = [
 
 [project.optional-dependencies]
 mips = [
-    "spimdisasm>=1.38.0,<2.0.0", # This value should be keep in sync with the version listed on disassembler/spimdisasm_disassembler.py
+    "spimdisasm>=1.39.0,<2.0.0", # This value should be keep in sync with the version listed on disassembler/spimdisasm_disassembler.py
     "rabbitizer>=1.12.0,<2.0.0",
     "pygfxd>=1.0.5",
     "n64img>=0.3.3",

{splat64-0.36.4 → splat64-0.37.1}/requirements.txt

@@ -4,7 +4,7 @@ tqdm==4.67.1
 intervaltree==3.1.0
 colorama==0.4.6
 # This value should be keep in sync with the version listed on disassembler/spimdisasm_disassembler.py and pyproject.toml
-spimdisasm>=1.38.0
+spimdisasm>=1.39.0
 rabbitizer>=1.10.0
 pygfxd>=1.0.5
 n64img>=0.1.4

{splat64-0.36.4 → splat64-0.37.1}/src/splat/__init__.py

@@ -1,7 +1,7 @@
 __package_name__ = __name__
 
 # Should be synced with pyproject.toml
-__version__ = "0.36.4"
+__version__ = "0.37.1"
 __author__ = "ethteck"
 
 from . import util as util

{splat64-0.36.4 → splat64-0.37.1}/src/splat/disassembler/spimdisasm_disassembler.py

@@ -7,7 +7,7 @@ from typing import Set
 
 class SpimdisasmDisassembler(disassembler.Disassembler):
     # This value should be kept in sync with the version listed on requirements.txt and pyproject.toml
-    SPIMDISASM_MIN = (1, 38, 0)
+    SPIMDISASM_MIN = (1, 39, 0)
 
     def configure(self):
         # Configure spimdisasm
@@ -74,6 +74,8 @@ class SpimdisasmDisassembler(disassembler.Disassembler):
             spimdisasm.common.GlobalConfig.SYMBOL_FINDER_FILTERED_ADDRESSES_AS_HILO = (
                 False
             )
+        if options.opts.align_on_branch_labels:
+            spimdisasm.common.GlobalConfig.ASM_EMIT_ALIGN_BRANCH_LABELS = True
         rabbitizer.config.toolchainTweaks_treatJAsUnconditionalBranch = (
             selected_compiler.j_as_branch
         )

{splat64-0.36.4 → splat64-0.37.1}/src/splat/scripts/split.py

@@ -581,7 +581,7 @@ def add_arguments_to_parser(parser: argparse.ArgumentParser):
         nargs="+",
         type=Path,
     )
-    parser.add_argument("--modes", nargs="+", default="all")
+    parser.add_argument("--modes", nargs="+", default=["all"])
     parser.add_argument("--verbose", action="store_true", help="Enable debug logging")
     parser.add_argument(
         "--use-cache", action="store_true", help="Only split changed segments in config"

{splat64-0.36.4 → splat64-0.37.1}/src/splat/segtypes/common/c.py

@@ -152,6 +152,8 @@ class CommonSegC(CommonSegCodeSubsegment):
 
     def split(self, rom_bytes: bytes):
         if self.is_auto_segment:
+            if options.opts.make_full_disasm_for_code:
+                self.split_as_asmtu_file(self.asm_out_path())
             return
 
         if self.rom_start != self.rom_end:

{splat64-0.36.4 → splat64-0.37.1}/src/splat/segtypes/common/codesubsegment.py

@@ -252,22 +252,21 @@ class CommonSegCodeSubsegment(Segment):
             f.write(self.spim_section.disassemble())
 
     # Same as above but write all sections from siblings
-    def split_as_asmtu_file(self, out_path: Optional[Path]):
-        if self.spim_section is None:
-            return
-
-        if not out_path:
-            return
-
+    def split_as_asmtu_file(self, out_path: Path):
         out_path.parent.mkdir(parents=True, exist_ok=True)
 
         self.print_file_boundaries()
 
         with open(out_path, "w", newline="\n") as f:
-            # Write `.text` contents
-            for line in self.get_asm_file_header():
-                f.write(line + "\n")
-            f.write(self.spim_section.disassemble())
+            # self.spim_section would be None if the current section was
+            # declared `auto` in the yaml.
+            # This can be useful when there are TUs with only data/rodata/bss/etc
+            # sections but not text section.
+            if self.spim_section is not None:
+                # Write `.text` contents
+                for line in self.get_asm_file_header():
+                    f.write(line + "\n")
+                f.write(self.spim_section.disassemble())
 
             # Disassemble the siblings to this file by respecting the `section_order`
             for sect in self.section_order:
@@ -281,25 +280,31 @@ class CommonSegCodeSubsegment(Segment):
                 if (
                     isinstance(sibling, CommonSegCodeSubsegment)
                     and sibling.spim_section is not None
-                    and not sibling.should_self_split()
+                    and (
+                        not sibling.should_self_split()
+                        or options.opts.make_full_disasm_for_code
+                    )
                 ):
                     f.write("\n")
                     f.write(f"{sibling.get_section_asm_line()}\n\n")
                     f.write(sibling.spim_section.disassemble())
 
-            # Another loop to check anything that somehow may not be present on the `section_order`
+            # Another loop to check anything that somehow may not be present in the `section_order`
             for sect, sibling in self.siblings.items():
                 if sect == self.get_linker_section_linksection():
                     continue
 
                 if sect in self.section_order:
-                    # Already handled on the above loop
+                    # Already handled in the above loop
                     continue
 
                 if (
                     isinstance(sibling, CommonSegCodeSubsegment)
                     and sibling.spim_section is not None
-                    and not sibling.should_self_split()
+                    and (
+                        not sibling.should_self_split()
+                        or options.opts.make_full_disasm_for_code
+                    )
                 ):
                     f.write("\n")
                     f.write(f"{sibling.get_section_asm_line()}\n\n")

{splat64-0.36.4 → splat64-0.37.1}/src/splat/util/compiler.py

@@ -18,6 +18,7 @@ class Compiler:
     asm_emit_size_directive: Optional[bool] = None
     j_as_branch: bool = False
     uses_include_asm: bool = True
+    align_on_branch_labels: bool = False
 
 
 GCC = Compiler(
@@ -61,7 +62,7 @@ PSYQ = Compiler(
 
 # PS2
 MWCCPS2 = Compiler("MWCCPS2", uses_include_asm=False)
-EEGCC = Compiler("EEGCC")
+EEGCC = Compiler("EEGCC", align_on_branch_labels=True)
 
 compiler_for_name: Dict[str, Compiler] = {
     x.name: x

{splat64-0.36.4 → splat64-0.37.1}/src/splat/util/file_presets.py

@@ -5,10 +5,12 @@ This includes writing files like:
 - include/macro.inc
 - include/labels.inc
 - include/gte_macros.inc
+
+The directory where these files are written to can be controlled with
+`options.opts.generated_asm_macros_directory`.
 """
 
 from pathlib import Path
-import os
 
 from . import options, log
 
@@ -22,7 +24,7 @@ def write_all_files():
 
 
 def _write(filepath: str, contents: str):
-    p = Path(os.path.normpath(options.opts.base_path / filepath))
+    p = Path(filepath)
     p.parent.mkdir(parents=True, exist_ok=True)
 
     if p.exists():
@@ -39,6 +41,8 @@ def write_include_asm_h():
         # These compilers do not use the `INCLUDE_ASM` macro.
         return
 
+    directory = options.opts.generated_asm_macros_directory.as_posix()
+
     if options.opts.include_asm_macro_style == "maspsx_hack":
         include_asm_macro = """\
 #define INCLUDE_ASM(FOLDER, NAME) \\
@@ -88,7 +92,12 @@ def write_include_asm_h():
 #ifndef INCLUDE_RODATA
 {include_rodata_macro}\
 #endif
-__asm__(".include \\"include/labels.inc\\"\\n");
+
+#if INCLUDE_ASM_USE_MACRO_INC
+__asm__(".include \\"{directory}/macro.inc\\"\\n");
+#else
+__asm__(".include \\"{directory}/labels.inc\\"\\n");
+#endif
 
 #else
 
@@ -103,10 +112,12 @@ __asm__(".include \\"include/labels.inc\\"\\n");
 
 #endif /* INCLUDE_ASM_H */
 """
-    _write("include/include_asm.h", file_data)
+    _write(f"{directory}/include_asm.h", file_data)
 
 
 def write_assembly_inc_files():
+    directory = options.opts.generated_asm_macros_directory.as_posix()
+
     func_macros = f"""\
 # A function symbol.
 .macro {options.opts.asm_function_macro} label, visibility=global
@@ -230,7 +241,7 @@ def write_assembly_inc_files():
 {data_macros}
 {nm_macros}\
 """
-        _write("include/labels.inc", labels_inc)
+        _write(f"{directory}/labels.inc", labels_inc)
 
     if options.opts.platform in {"n64", "psx"}:
         gas = macros_inc
@@ -338,7 +349,7 @@ def write_assembly_inc_files():
 {gas}
 .endif
 """
-    _write("include/macro.inc", f"{preamble}\n{gas}")
+    _write(f"{directory}/macro.inc", f"{preamble}\n{gas}")
 
 
 def write_gte_macros():
@@ -762,4 +773,5 @@ def write_gte_macros():
 .endif
 """
 
-    _write("include/gte_macros.inc", gte_macros)
+    directory = options.opts.generated_asm_macros_directory.as_posix()
+    _write(f"{directory}/gte_macros.inc", gte_macros)

{splat64-0.36.4 → splat64-0.37.1}/src/splat/util/options.py

@@ -43,6 +43,8 @@ class SplatOpts:
     # default: The default one.
     # maspsx_hack: Use the maspsx hack workaround definition https://github.com/mkst/maspsx?tab=readme-ov-file#include_asm-reordering-workaround-hack
     include_asm_macro_style: Literal["default", "maspsx_hack"]
+    # Directory to place the generated asm macros files.
+    generated_asm_macros_directory: Path
     # Determines whether to use .o as the suffix for all binary files?... TODO document
     use_o_as_suffix: bool
     # the value of the $gp register to correctly calculate offset to %gp_rel relocs
@@ -278,6 +280,8 @@ class SplatOpts:
     # Determines whether to use a legacy INCLUDE_ASM macro format in c files
     # only applies to GCC/SN64
     use_legacy_include_asm: bool
+    # Emit alignment directives in branch labels, as a way to workaround the short loop bug present in SN PS2 compilers
+    align_on_branch_labels: bool
 
     # Returns whether the given mode is currently enabled
     def is_mode_active(self, mode: str) -> bool:
@@ -392,6 +396,11 @@ def _parse_yaml(
     if asm_emit_size_directive is None:
         asm_emit_size_directive = comp.asm_emit_size_directive
 
+    align_on_branch_labels = p.parse_optional_opt("align_on_branch_labels", bool)
+    # If option not provided then use the compiler default
+    if align_on_branch_labels is None:
+        align_on_branch_labels = comp.align_on_branch_labels
+
     def parse_endianness() -> Literal["big", "little"]:
         endianness = p.parse_opt_within(
             "endianness",
@@ -454,6 +463,9 @@ def _parse_yaml(
         ),
         generate_asm_macros_files=p.parse_opt("generate_asm_macros_files", bool, True),
         include_asm_macro_style=parse_include_asm_macro_style(),
+        generated_asm_macros_directory=p.parse_path(
+            base_path, "generated_asm_macros_directory", "include"
+        ),
         use_o_as_suffix=p.parse_opt("o_as_suffix", bool, False),
         gp=p.parse_optional_opt("gp_value", int),
         check_consecutive_segment_types=p.parse_opt(
@@ -608,6 +620,7 @@ def _parse_yaml(
         hardware_regs=p.parse_opt("hardware_regs", bool, False),
         image_type_in_extension=p.parse_opt("image_type_in_extension", bool, False),
         use_legacy_include_asm=p.parse_opt("use_legacy_include_asm", bool, False),
+        align_on_branch_labels=align_on_branch_labels,
         disasm_unknown=p.parse_opt("disasm_unknown", bool, False),
         detect_redundant_function_end=p.parse_opt(
             "detect_redundant_function_end", bool, True

{splat64-0.36.4 → splat64-0.37.1}/test/basic_app/expected/asm/main.s

@@ -86,3 +86,51 @@ glabel func_800004A0
     /* 10E8 800004E8 27BD0018 */   addiu     $sp, $sp, 0x18
 endlabel func_800004A0
     /* 10EC 800004EC 00000000 */  nop
+
+.section .data, "wa"
+
+nonmatching D_80000500
+
+dlabel D_80000500
+    /* 1100 80000500 00000001 */ .word 0x00000001
+enddlabel D_80000500
+
+nonmatching D_80000504
+
+dlabel D_80000504
+    /* 1104 80000504 00000000 */ .word 0x00000000
+    /* 1108 80000508 00000000 */ .word 0x00000000
+    /* 110C 8000050C 00000000 */ .word 0x00000000
+enddlabel D_80000504
+
+.section .rodata, "a"
+
+nonmatching D_80000510
+
+dlabel D_80000510
+    /* 1110 80000510 00010203 */ .word 0x00010203
+    /* 1114 80000514 04050607 */ .word 0x04050607
+enddlabel D_80000510
+
+.align 3
+nonmatching jtbl_80000518
+
+dlabel jtbl_80000518
+    /* 1118 80000518 80000448 */ .word .L80000448
+    /* 111C 8000051C 80000450 */ .word .L80000450
+    /* 1120 80000520 80000458 */ .word .L80000458
+    /* 1124 80000524 80000460 */ .word .L80000460
+    /* 1128 80000528 80000468 */ .word .L80000468
+    /* 112C 8000052C 80000470 */ .word .L80000470
+    /* 1130 80000530 80000478 */ .word .L80000478
+    /* 1134 80000534 80000480 */ .word .L80000480
+    /* 1138 80000538 00000000 */ .word 0x00000000
+    /* 113C 8000053C 00000000 */ .word 0x00000000
+enddlabel jtbl_80000518
+
+.section .bss, "wa"
+
+nonmatching D_80000540, 0x80
+
+dlabel D_80000540
+    /* 80000540 */ .space 0x80

{splat64-0.36.4 → splat64-0.37.1}/test/basic_app/expected/include/include_asm.h

@@ -22,7 +22,12 @@
         ".section .text" \
     )
 #endif
-__asm__(".include \"include/labels.inc\"\n");
+
+#if INCLUDE_ASM_USE_MACRO_INC
+__asm__(".include \"test/basic_app/split/include/macro.inc\"\n");
+#else
+__asm__(".include \"test/basic_app/split/include/labels.inc\"\n");
+#endif
 
 #else