splat64 0.37.0__tar.gz → 0.37.2__tar.gz

{splat64-0.37.0 → splat64-0.37.2}/.github/workflows/unit_tests.yml

@@ -80,3 +80,4 @@ jobs:
         run: |
           ${{ matrix.os.python_venv_activate }}
           python3 test.py
+          python3 test_n64_entrypoints.py

{splat64-0.37.0 → splat64-0.37.2}/CHANGELOG.md

@@ -1,5 +1,15 @@
 # splat Release Notes
 
+### 0.37.2
+* Add new option `sort_segments_by_vram_dependency` to help with non-matching builds for binaries with complicated memory layouts. See the wiki for details.
+* Fix create_config missing bss segments due to unsigned LO instructions.
+
+### 0.37.1
+
+* Fix `make_full_disasm_for_code` not extracting data-only TUs.
+* Fix `make_full_disasm_for_code` and `disassemble_all` combo
+  * Fixes not writing other sections to asm file when both options are enabled.
+
 ### 0.37.0
 
 * Add define check to allow using `macro.inc` instead of `labels.inc` in `include_asm.h`.

{splat64-0.37.0 → splat64-0.37.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: splat64
-Version: 0.37.0
+Version: 0.37.2
 Summary: A binary splitting tool to assist with decompilation and modding projects
 Project-URL: Repository, https://github.com/ethteck/splat
 Project-URL: Issues, https://github.com/ethteck/splat/issues
@@ -76,7 +76,7 @@ The brackets corresponds to the optional dependencies to install while installin
 If you use a `requirements.txt` file in your repository, then you can add this library with the following line:
 
 ```txt
-splat64[mips]>=0.37.0,<1.0.0
+splat64[mips]>=0.37.2,<1.0.0
 ```
 
 ### Optional dependencies

{splat64-0.37.0 → splat64-0.37.2}/README.md

@@ -21,7 +21,7 @@ The brackets corresponds to the optional dependencies to install while installin
 If you use a `requirements.txt` file in your repository, then you can add this library with the following line:
 
 ```txt
-splat64[mips]>=0.37.0,<1.0.0
+splat64[mips]>=0.37.2,<1.0.0
 ```
 
 ### Optional dependencies

{splat64-0.37.0 → splat64-0.37.2}/docs/Configuration.md

@@ -526,6 +526,10 @@ Generates a discard section like this:
 
 Determines whether to add wildcards for section linking in the linker script (.rodata* for example)
 
+### ld_sort_segments_by_vram_class_dependency
+
+Ensures segments of vram classes with dependencies (`vram_symbol` / `follows_classes`) are written to the linker script AFTER the segments they depend on. This is intended to preserve vram class ordering for shifted builds (e.g., mods) and is not expected to produce a matching build. Disable this when a matching build is desired.
+
 ### ld_use_symbolic_vram_addresses
 
 Determines whether to use `follows_vram` (segment option) and `vram_symbol` / `follows_classes` (vram_class options) to calculate vram addresses in the linker script.
@@ -848,6 +852,16 @@ Emit a full `.s` file for each `c`/`cpp` segment containing disassembly of .text
 
 Can be used to generate "target" or "expected" objects for asm diffing.
 
+To generate `.s` files for data-only TUs (or rodata/bss only, or any other combination that doesn't have a text section) a dummy `c` or `cpp` section with the same name as the data section must be defined in the yaml with an start value of `auto` instead of a number. For example:
+
+```yaml
+      - [auto, c, boot/rom_offsets]
+
+      # -- Other c or data subsegments --
+
+      - [0x00F340, .data, boot/rom_offsets]
+```
+
 ### global_vram_start and global_vram_end
 
 Allow specifying that the global memory range may be larger than what was automatically detected.

{splat64-0.37.0 → splat64-0.37.2}/pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "splat64"
 # Should be synced with src/splat/__init__.py
-version = "0.37.0"
+version = "0.37.2"
 description = "A binary splitting tool to assist with decompilation and modding projects"
 readme = "README.md"
 license = {file = "LICENSE"}

{splat64-0.37.0 → splat64-0.37.2}/src/splat/__init__.py

@@ -1,7 +1,7 @@
 __package_name__ = __name__
 
 # Should be synced with pyproject.toml
-__version__ = "0.37.0"
+__version__ = "0.37.2"
 __author__ = "ethteck"
 
 from . import util as util

{splat64-0.37.0 → splat64-0.37.2}/src/splat/scripts/split.py

@@ -6,6 +6,8 @@ import importlib
 from typing import Any, Dict, List, Optional, Set, Tuple, Union
 from pathlib import Path
 
+from collections import defaultdict, deque
+
 from .. import __package_name__, __version__
 from ..disassembler import disassembler_instance
 from ..util import cache_handler, progress_bar, vram_classes, statistics, file_presets
@@ -222,6 +224,47 @@ def calc_segment_dependences(
     return vram_class_to_follows_segments
 
 
+def sort_segments_by_vram_class_dependency(
+    all_segments: List[Segment],
+) -> List[Segment]:
+    # map all "_VRAM_END" strings to segments
+    end_sym_to_seg: Dict[str, Segment] = {}
+    for seg in all_segments:
+        end_sym_to_seg[get_segment_vram_end_symbol_name(seg)] = seg
+
+    # build dependency graph: A -> B means "A must come before B"
+    graph: Dict[Segment, List[Segment]] = defaultdict(list)
+    indeg: Dict[Segment, int] = {seg: 0 for seg in all_segments}
+
+    for seg in all_segments:
+        sym = seg.vram_symbol
+        if sym is None:
+            continue
+        dep = end_sym_to_seg.get(sym)
+        if dep is None or dep is seg:
+            continue
+        graph[dep].append(seg)
+        indeg[seg] += 1
+
+    # stable topo sort with queue seeded in original order
+    q = deque([seg for seg in all_segments if indeg[seg] == 0])
+    out: List[Segment] = []
+
+    while q:
+        n = q.popleft()
+        out.append(n)
+        for m in graph.get(n, []):
+            indeg[m] -= 1
+            if indeg[m] == 0:
+                q.append(m)
+
+    assert len(out) == len(all_segments), (
+        "Encountered cyclic dependency when reordering segments by vram class."
+    )
+
+    return out
+
+
 def read_target_binary() -> bytes:
     rom_bytes = options.opts.target_path.read_bytes()
 
@@ -317,6 +360,9 @@ def do_split(
 
 
 def write_linker_script(all_segments: List[Segment]) -> LinkerWriter:
+    if options.opts.ld_sort_segments_by_vram_class_dependency:
+        all_segments = sort_segments_by_vram_class_dependency(all_segments)
+
     vram_class_dependencies = calc_segment_dependences(all_segments)
     vram_classes_to_search = set(vram_class_dependencies.keys())
 

{splat64-0.37.0 → splat64-0.37.2}/src/splat/segtypes/common/c.py

@@ -51,7 +51,7 @@ class CommonSegC(CommonSegCodeSubsegment):
 
     @staticmethod
     def get_funcs_defined_in_c(c_file: Path) -> Set[str]:
-        with open(c_file, "r") as f:
+        with open(c_file, "r", encoding="utf-8") as f:
             text = CommonSegC.strip_c_comments(f.read())
 
         return set(m.group(1) for m in C_FUNC_RE.finditer(text))
@@ -105,7 +105,7 @@ class CommonSegC(CommonSegCodeSubsegment):
 
     @staticmethod
     def get_global_asm_funcs(c_file: Path) -> Set[str]:
-        with c_file.open() as f:
+        with c_file.open(encoding="utf-8") as f:
             text = CommonSegC.strip_c_comments(f.read())
         if options.opts.compiler == IDO:
             return set(m.group(2) for m in C_GLOBAL_ASM_IDO_RE.finditer(text))
@@ -114,7 +114,7 @@ class CommonSegC(CommonSegCodeSubsegment):
 
     @staticmethod
     def get_global_asm_rodata_syms(c_file: Path) -> Set[str]:
-        with c_file.open() as f:
+        with c_file.open(encoding="utf-8") as f:
             text = CommonSegC.strip_c_comments(f.read())
         if options.opts.compiler == IDO:
             return set(m.group(2) for m in C_GLOBAL_ASM_IDO_RE.finditer(text))
@@ -152,6 +152,8 @@ class CommonSegC(CommonSegCodeSubsegment):
 
     def split(self, rom_bytes: bytes):
         if self.is_auto_segment:
+            if options.opts.make_full_disasm_for_code:
+                self.split_as_asmtu_file(self.asm_out_path())
             return
 
         if self.rom_start != self.rom_end:

{splat64-0.37.0 → splat64-0.37.2}/src/splat/segtypes/common/codesubsegment.py

@@ -252,22 +252,21 @@ class CommonSegCodeSubsegment(Segment):
             f.write(self.spim_section.disassemble())
 
     # Same as above but write all sections from siblings
-    def split_as_asmtu_file(self, out_path: Optional[Path]):
-        if self.spim_section is None:
-            return
-
-        if not out_path:
-            return
-
+    def split_as_asmtu_file(self, out_path: Path):
         out_path.parent.mkdir(parents=True, exist_ok=True)
 
         self.print_file_boundaries()
 
         with open(out_path, "w", newline="\n") as f:
-            # Write `.text` contents
-            for line in self.get_asm_file_header():
-                f.write(line + "\n")
-            f.write(self.spim_section.disassemble())
+            # self.spim_section would be None if the current section was
+            # declared `auto` in the yaml.
+            # This can be useful when there are TUs with only data/rodata/bss/etc
+            # sections but not text section.
+            if self.spim_section is not None:
+                # Write `.text` contents
+                for line in self.get_asm_file_header():
+                    f.write(line + "\n")
+                f.write(self.spim_section.disassemble())
 
             # Disassemble the siblings to this file by respecting the `section_order`
             for sect in self.section_order:
@@ -281,25 +280,31 @@ class CommonSegCodeSubsegment(Segment):
                 if (
                     isinstance(sibling, CommonSegCodeSubsegment)
                     and sibling.spim_section is not None
-                    and not sibling.should_self_split()
+                    and (
+                        not sibling.should_self_split()
+                        or options.opts.make_full_disasm_for_code
+                    )
                 ):
                     f.write("\n")
                     f.write(f"{sibling.get_section_asm_line()}\n\n")
                     f.write(sibling.spim_section.disassemble())
 
-            # Another loop to check anything that somehow may not be present on the `section_order`
+            # Another loop to check anything that somehow may not be present in the `section_order`
             for sect, sibling in self.siblings.items():
                 if sect == self.get_linker_section_linksection():
                     continue
 
                 if sect in self.section_order:
-                    # Already handled on the above loop
+                    # Already handled in the above loop
                     continue
 
                 if (
                     isinstance(sibling, CommonSegCodeSubsegment)
                     and sibling.spim_section is not None
-                    and not sibling.should_self_split()
+                    and (
+                        not sibling.should_self_split()
+                        or options.opts.make_full_disasm_for_code
+                    )
                 ):
                     f.write("\n")
                     f.write(f"{sibling.get_section_asm_line()}\n\n")

{splat64-0.37.0 → splat64-0.37.2}/src/splat/util/file_presets.py

@@ -42,9 +42,6 @@ def write_include_asm_h():
         return
 
     directory = options.opts.generated_asm_macros_directory.as_posix()
-    print()
-    print(directory)
-    print()
 
     if options.opts.include_asm_macro_style == "maspsx_hack":
         include_asm_macro = """\

{splat64-0.37.0 → splat64-0.37.2}/src/splat/util/n64/rominfo.py

@@ -162,8 +162,7 @@ class N64EntrypointInfo:
                         register_values[insn.rs.value] + insn.getProcessedImmediate()
                     )
                     completed_pair[insn.rt.value] = True
-                    if not insn.isUnsigned():
-                        lo_assignments[insn.rt.value] = current_rom
+                    lo_assignments[insn.rt.value] = current_rom
                 elif insn.doesStore():
                     if insn.rt == rabbitizer.RegGprO32.zero:
                         # Try to detect the zero-ing bss algorithm

{splat64-0.37.0 → splat64-0.37.2}/src/splat/util/options.py

@@ -127,6 +127,8 @@ class SplatOpts:
     # `vram_symbol` / `follows_classes` (vram_class options) to calculate vram addresses in the linker script.
     # If disabled, this uses the plain integer values for vram addresses defined in the yaml.
     ld_use_symbolic_vram_addresses: bool
+    # Ensures segments of vram classes with dependencies (`vram_symbol` / `follows_classes`) are written to the linker script AFTER the segments they depend on.
+    ld_sort_segments_by_vram_class_dependency: bool
     # Change linker script generation to allow partially linking segments. Requires both `ld_partial_scripts_path` and `ld_partial_build_segments_path` to be set.
     ld_partial_linking: bool
     # Folder were each intermediary linker script will be written to.
@@ -512,6 +514,9 @@ def _parse_yaml(
         ld_sections_allowlist=p.parse_opt("ld_sections_allowlist", list, []),
         ld_sections_denylist=p.parse_opt("ld_sections_denylist", list, []),
         ld_wildcard_sections=p.parse_opt("ld_wildcard_sections", bool, False),
+        ld_sort_segments_by_vram_class_dependency=p.parse_opt(
+            "ld_sort_segments_by_vram_class_dependency", bool, False
+        ),
         ld_use_symbolic_vram_addresses=p.parse_opt(
             "ld_use_symbolic_vram_addresses", bool, True
         ),

{splat64-0.37.0 → splat64-0.37.2}/test/basic_app/expected/asm/main.s

@@ -86,3 +86,51 @@ glabel func_800004A0
     /* 10E8 800004E8 27BD0018 */   addiu     $sp, $sp, 0x18
 endlabel func_800004A0
     /* 10EC 800004EC 00000000 */  nop
+
+.section .data, "wa"
+
+nonmatching D_80000500
+
+dlabel D_80000500
+    /* 1100 80000500 00000001 */ .word 0x00000001
+enddlabel D_80000500
+
+nonmatching D_80000504
+
+dlabel D_80000504
+    /* 1104 80000504 00000000 */ .word 0x00000000
+    /* 1108 80000508 00000000 */ .word 0x00000000
+    /* 110C 8000050C 00000000 */ .word 0x00000000
+enddlabel D_80000504
+
+.section .rodata, "a"
+
+nonmatching D_80000510
+
+dlabel D_80000510
+    /* 1110 80000510 00010203 */ .word 0x00010203
+    /* 1114 80000514 04050607 */ .word 0x04050607
+enddlabel D_80000510
+
+.align 3
+nonmatching jtbl_80000518
+
+dlabel jtbl_80000518
+    /* 1118 80000518 80000448 */ .word .L80000448
+    /* 111C 8000051C 80000450 */ .word .L80000450
+    /* 1120 80000520 80000458 */ .word .L80000458
+    /* 1124 80000524 80000460 */ .word .L80000460
+    /* 1128 80000528 80000468 */ .word .L80000468
+    /* 112C 8000052C 80000470 */ .word .L80000470
+    /* 1130 80000530 80000478 */ .word .L80000478
+    /* 1134 80000534 80000480 */ .word .L80000480
+    /* 1138 80000538 00000000 */ .word 0x00000000
+    /* 113C 8000053C 00000000 */ .word 0x00000000
+enddlabel jtbl_80000518
+
+.section .bss, "wa"
+
+nonmatching D_80000540, 0x80
+
+dlabel D_80000540
+    /* 80000540 */ .space 0x80