splat64 0.37.1__tar.gz → 0.37.3__tar.gz

{splat64-0.37.1 → splat64-0.37.3}/.github/workflows/lint.yml

@@ -45,14 +45,8 @@ jobs:
           pip install -r requirements.txt
           pip install types-PyYAML
 
-      - name: ruff split
-        run: ruff check split.py
-
-      - name: ruff create_config
-        run: ruff check create_config.py
-
-      - name: ruff test
-        run: ruff check test.py
+      - name: ruff
+        run: ruff check .
 
   mypy_splat_checks:
     runs-on: ubuntu-latest
@@ -94,11 +88,5 @@ jobs:
         pip install -r requirements.txt
         pip install types-PyYAML
 
-    - name: mypy split
-      run: mypy --show-column-numbers --hide-error-context split.py
-
-    - name: mypy create_config
-      run: mypy --show-column-numbers --hide-error-context create_config.py
-
-    - name: mypy test
-      run: mypy --show-column-numbers --hide-error-context test.py
+    - name: mypy
+      run: mypy --show-column-numbers --hide-error-context *.py

{splat64-0.37.1 → splat64-0.37.3}/.github/workflows/unit_tests.yml

@@ -80,3 +80,4 @@ jobs:
         run: |
           ${{ matrix.os.python_venv_activate }}
           python3 test.py
+          python3 test_n64_entrypoints.py

{splat64-0.37.1 → splat64-0.37.3}/CHANGELOG.md

@@ -1,10 +1,20 @@
 # splat Release Notes
 
+### 0.37.3
+
+* create_config: Avoid emitting relocations for addresses paired by using `ori`
+
+### 0.37.2
+
+* Add new option `sort_segments_by_vram_dependency` to help with non-matching builds for binaries with complicated memory layouts. See the wiki for details.
+* Fix create_config missing bss segments due to unsigned LO instructions.
+
 ### 0.37.1
 
 * Fix `make_full_disasm_for_code` not extracting data-only TUs.
 * Fix `make_full_disasm_for_code` and `disassemble_all` combo
   * Fixes not writing other sections to asm file when both options are enabled.
+* Add UTF-8 encoding to file reads in c.py.
 
 ### 0.37.0
 

{splat64-0.37.1 → splat64-0.37.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: splat64
-Version: 0.37.1
+Version: 0.37.3
 Summary: A binary splitting tool to assist with decompilation and modding projects
 Project-URL: Repository, https://github.com/ethteck/splat
 Project-URL: Issues, https://github.com/ethteck/splat/issues
@@ -76,7 +76,7 @@ The brackets corresponds to the optional dependencies to install while installin
 If you use a `requirements.txt` file in your repository, then you can add this library with the following line:
 
 ```txt
-splat64[mips]>=0.37.1,<1.0.0
+splat64[mips]>=0.37.3,<1.0.0
 ```
 
 ### Optional dependencies

{splat64-0.37.1 → splat64-0.37.3}/README.md

@@ -21,7 +21,7 @@ The brackets corresponds to the optional dependencies to install while installin
 If you use a `requirements.txt` file in your repository, then you can add this library with the following line:
 
 ```txt
-splat64[mips]>=0.37.1,<1.0.0
+splat64[mips]>=0.37.3,<1.0.0
 ```
 
 ### Optional dependencies

{splat64-0.37.1 → splat64-0.37.3}/docs/Configuration.md

@@ -526,6 +526,10 @@ Generates a discard section like this:
 
 Determines whether to add wildcards for section linking in the linker script (.rodata* for example)
 
+### ld_sort_segments_by_vram_class_dependency
+
+Ensures segments of vram classes with dependencies (`vram_symbol` / `follows_classes`) are written to the linker script AFTER the segments they depend on. This is intended to preserve vram class ordering for shifted builds (e.g., mods) and is not expected to produce a matching build. Disable this when a matching build is desired.
+
 ### ld_use_symbolic_vram_addresses
 
 Determines whether to use `follows_vram` (segment option) and `vram_symbol` / `follows_classes` (vram_class options) to calculate vram addresses in the linker script.

{splat64-0.37.1 → splat64-0.37.3}/pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "splat64"
 # Should be synced with src/splat/__init__.py
-version = "0.37.1"
+version = "0.37.3"
 description = "A binary splitting tool to assist with decompilation and modding projects"
 readme = "README.md"
 license = {file = "LICENSE"}

{splat64-0.37.1 → splat64-0.37.3}/src/splat/__init__.py

@@ -1,7 +1,7 @@
 __package_name__ = __name__
 
 # Should be synced with pyproject.toml
-__version__ = "0.37.1"
+__version__ = "0.37.3"
 __author__ = "ethteck"
 
 from . import util as util

{splat64-0.37.1 → splat64-0.37.3}/src/splat/scripts/create_config.py

@@ -193,32 +193,37 @@ segments:
     file_presets.write_all_files()
 
     # Write reloc_addrs.txt file
-    reloc_addrs = []
-    if rom.entrypoint_info.bss_start_address is not None:
-        reloc_addrs.append(
-            f"rom:0x{rom.entrypoint_info.bss_start_address.rom_hi:06X} reloc:MIPS_HI16 symbol:main_BSS_START"
-        )
-        reloc_addrs.append(
-            f"rom:0x{rom.entrypoint_info.bss_start_address.rom_lo:06X} reloc:MIPS_LO16 symbol:main_BSS_START"
-        )
-        reloc_addrs.append("")
-    if rom.entrypoint_info.bss_size is not None:
-        reloc_addrs.append(
-            f"rom:0x{rom.entrypoint_info.bss_size.rom_hi:06X} reloc:MIPS_HI16 symbol:main_BSS_SIZE"
-        )
-        reloc_addrs.append(
-            f"rom:0x{rom.entrypoint_info.bss_size.rom_lo:06X} reloc:MIPS_LO16 symbol:main_BSS_SIZE"
-        )
-        reloc_addrs.append("")
-    if rom.entrypoint_info.bss_end_address is not None:
+    reloc_addrs: list[str] = []
+
+    addresses_info: list[tuple[Optional[rominfo.EntryAddressInfo], str]] = [
+        (rom.entrypoint_info.main_address, "main"),
+        (rom.entrypoint_info.bss_start_address, "main_BSS_START"),
+        (rom.entrypoint_info.bss_size, "main_BSS_SIZE"),
+        (rom.entrypoint_info.bss_end_address, "main_BSS_END"),
+    ]
+
+    for addr_info, sym_name in addresses_info:
+        if addr_info is None:
+            continue
+        if addr_info.ori:
+            # Avoid emitting relocations for `ori`s since `%lo` doesn't support it.
+            continue
+        if addr_info.rom_hi == addr_info.rom_lo:
+            # hi and lo may be the same for the "main" address, i.e. a direct jal.
+            continue
+
         reloc_addrs.append(
-            f"rom:0x{rom.entrypoint_info.bss_end_address.rom_hi:06X} reloc:MIPS_HI16 symbol:main_BSS_END"
+            f"rom:0x{addr_info.rom_hi:06X} reloc:MIPS_HI16 symbol:{sym_name}"
         )
         reloc_addrs.append(
-            f"rom:0x{rom.entrypoint_info.bss_end_address.rom_lo:06X} reloc:MIPS_LO16 symbol:main_BSS_END"
+            f"rom:0x{addr_info.rom_lo:06X} reloc:MIPS_LO16 symbol:{sym_name}"
         )
         reloc_addrs.append("")
-    if rom.entrypoint_info.stack_top is not None:
+
+    if (
+        rom.entrypoint_info.stack_top is not None
+        and not rom.entrypoint_info.stack_top.ori
+    ):
         reloc_addrs.append(
             '// This entry corresponds to the "stack top", which is the end of the array used as the stack for the main segment.'
         )

{splat64-0.37.1 → splat64-0.37.3}/src/splat/scripts/split.py

@@ -6,6 +6,8 @@ import importlib
 from typing import Any, Dict, List, Optional, Set, Tuple, Union
 from pathlib import Path
 
+from collections import defaultdict, deque
+
 from .. import __package_name__, __version__
 from ..disassembler import disassembler_instance
 from ..util import cache_handler, progress_bar, vram_classes, statistics, file_presets
@@ -222,6 +224,47 @@ def calc_segment_dependences(
     return vram_class_to_follows_segments
 
 
+def sort_segments_by_vram_class_dependency(
+    all_segments: List[Segment],
+) -> List[Segment]:
+    # map all "_VRAM_END" strings to segments
+    end_sym_to_seg: Dict[str, Segment] = {}
+    for seg in all_segments:
+        end_sym_to_seg[get_segment_vram_end_symbol_name(seg)] = seg
+
+    # build dependency graph: A -> B means "A must come before B"
+    graph: Dict[Segment, List[Segment]] = defaultdict(list)
+    indeg: Dict[Segment, int] = {seg: 0 for seg in all_segments}
+
+    for seg in all_segments:
+        sym = seg.vram_symbol
+        if sym is None:
+            continue
+        dep = end_sym_to_seg.get(sym)
+        if dep is None or dep is seg:
+            continue
+        graph[dep].append(seg)
+        indeg[seg] += 1
+
+    # stable topo sort with queue seeded in original order
+    q = deque([seg for seg in all_segments if indeg[seg] == 0])
+    out: List[Segment] = []
+
+    while q:
+        n = q.popleft()
+        out.append(n)
+        for m in graph.get(n, []):
+            indeg[m] -= 1
+            if indeg[m] == 0:
+                q.append(m)
+
+    assert len(out) == len(all_segments), (
+        "Encountered cyclic dependency when reordering segments by vram class."
+    )
+
+    return out
+
+
 def read_target_binary() -> bytes:
     rom_bytes = options.opts.target_path.read_bytes()
 
@@ -317,6 +360,9 @@ def do_split(
 
 
 def write_linker_script(all_segments: List[Segment]) -> LinkerWriter:
+    if options.opts.ld_sort_segments_by_vram_class_dependency:
+        all_segments = sort_segments_by_vram_class_dependency(all_segments)
+
     vram_class_dependencies = calc_segment_dependences(all_segments)
     vram_classes_to_search = set(vram_class_dependencies.keys())
 

{splat64-0.37.1 → splat64-0.37.3}/src/splat/segtypes/common/c.py

@@ -51,7 +51,7 @@ class CommonSegC(CommonSegCodeSubsegment):
 
     @staticmethod
     def get_funcs_defined_in_c(c_file: Path) -> Set[str]:
-        with open(c_file, "r") as f:
+        with open(c_file, "r", encoding="utf-8") as f:
             text = CommonSegC.strip_c_comments(f.read())
 
         return set(m.group(1) for m in C_FUNC_RE.finditer(text))
@@ -105,7 +105,7 @@ class CommonSegC(CommonSegCodeSubsegment):
 
     @staticmethod
     def get_global_asm_funcs(c_file: Path) -> Set[str]:
-        with c_file.open() as f:
+        with c_file.open(encoding="utf-8") as f:
             text = CommonSegC.strip_c_comments(f.read())
         if options.opts.compiler == IDO:
             return set(m.group(2) for m in C_GLOBAL_ASM_IDO_RE.finditer(text))
@@ -114,7 +114,7 @@ class CommonSegC(CommonSegCodeSubsegment):
 
     @staticmethod
     def get_global_asm_rodata_syms(c_file: Path) -> Set[str]:
-        with c_file.open() as f:
+        with c_file.open(encoding="utf-8") as f:
             text = CommonSegC.strip_c_comments(f.read())
         if options.opts.compiler == IDO:
             return set(m.group(2) for m in C_GLOBAL_ASM_IDO_RE.finditer(text))

{splat64-0.37.1 → splat64-0.37.3}/src/splat/util/n64/rominfo.py

@@ -74,13 +74,14 @@ class EntryAddressInfo:
     value: int
     rom_hi: int
     rom_lo: int
+    ori: bool
 
     @staticmethod
     def new(
-        value: Optional[int], hi: Optional[int], lo: Optional[int]
+        value: Optional[int], hi: Optional[int], lo: Optional[int], ori: Optional[int]
     ) -> Optional["EntryAddressInfo"]:
         if value is not None and hi is not None and lo is not None:
-            return EntryAddressInfo(value, hi, lo)
+            return EntryAddressInfo(value, hi, lo, ori == lo)
         return None
 
 
@@ -94,6 +95,7 @@ class N64EntrypointInfo:
     main_address: Optional[EntryAddressInfo]
     stack_top: Optional[EntryAddressInfo]
     traditional_entrypoint: bool
+    ori_entrypoint: bool
 
     def segment_size(self) -> int:
         if self.data_size is not None:
@@ -120,6 +122,10 @@ class N64EntrypointInfo:
         completed_pair = [False for _ in range(32)]
         hi_assignments: List[Optional[int]] = [None for _ in range(32)]
         lo_assignments: List[Optional[int]] = [None for _ in range(32)]
+        # We need to track if something was paired using an ori instead of an
+        # addiu or similar, because if that's the case we can't emit normal
+        # relocations in the generated symbol_addrs file for it.
+        ori_assignments: List[Optional[int]] = [None for _ in range(32)]
 
         register_bss_address: Optional[int] = None
         register_bss_size: Optional[int] = None
@@ -130,6 +136,7 @@ class N64EntrypointInfo:
         bss_end_address: Optional[EntryAddressInfo] = None
 
         traditional_entrypoint = True
+        ori_entrypoint = False
         decrementing_bss_routine = True
         data_size: Optional[int] = None
         func_call_target: Optional[EntryAddressInfo] = None
@@ -162,8 +169,10 @@ class N64EntrypointInfo:
                         register_values[insn.rs.value] + insn.getProcessedImmediate()
                     )
                     completed_pair[insn.rt.value] = True
-                    if not insn.isUnsigned():
-                        lo_assignments[insn.rt.value] = current_rom
+                    lo_assignments[insn.rt.value] = current_rom
+                    if insn.isUnsigned():
+                        ori_assignments[insn.rt.value] = current_rom
+                        ori_entrypoint = True
                 elif insn.doesStore():
                     if insn.rt == rabbitizer.RegGprO32.zero:
                         # Try to detect the zero-ing bss algorithm
@@ -209,11 +218,13 @@ class N64EntrypointInfo:
                         register_values[insn.rs.value],
                         hi_assignments[insn.rs.value],
                         lo_assignments[insn.rs.value],
+                        ori_assignments[insn.rs.value],
                     )
                     bss_end_address = EntryAddressInfo.new(
                         register_values[insn.rt.value],
                         hi_assignments[insn.rt.value],
                         lo_assignments[insn.rt.value],
+                        ori_assignments[insn.rt.value],
                     )
 
             elif insn.isFunctionCall():
@@ -222,7 +233,7 @@ class N64EntrypointInfo:
                 # entrypoint to actual code.
                 traditional_entrypoint = False
                 func_call_target = EntryAddressInfo(
-                    insn.getInstrIndexAsVram(), current_rom, current_rom
+                    insn.getInstrIndexAsVram(), current_rom, current_rom, False
                 )
 
             elif insn.uniqueId == rabbitizer.InstrId.cpu_break:
@@ -255,12 +266,14 @@ class N64EntrypointInfo:
                     register_values[register_bss_address],
                     hi_assignments[register_bss_address],
                     lo_assignments[register_bss_address],
+                    ori_assignments[register_bss_address],
                 )
             if register_bss_size is not None:
                 bss_size = EntryAddressInfo.new(
                     register_values[register_bss_size],
                     hi_assignments[register_bss_size],
                     lo_assignments[register_bss_size],
+                    ori_assignments[register_bss_size],
                 )
 
         if register_main_address is not None:
@@ -268,6 +281,7 @@ class N64EntrypointInfo:
                 register_values[register_main_address],
                 hi_assignments[register_main_address],
                 lo_assignments[register_main_address],
+                ori_assignments[register_main_address],
             )
         else:
             main_address = None
@@ -276,6 +290,7 @@ class N64EntrypointInfo:
             register_values[rabbitizer.RegGprO32.sp.value],
             hi_assignments[rabbitizer.RegGprO32.sp.value],
             lo_assignments[rabbitizer.RegGprO32.sp.value],
+            ori_assignments[rabbitizer.RegGprO32.sp.value],
         )
 
         if not traditional_entrypoint:
@@ -300,6 +315,7 @@ class N64EntrypointInfo:
             main_address,
             stack_top,
             traditional_entrypoint,
+            ori_entrypoint,
         )
 
 

{splat64-0.37.1 → splat64-0.37.3}/src/splat/util/options.py

@@ -127,6 +127,8 @@ class SplatOpts:
     # `vram_symbol` / `follows_classes` (vram_class options) to calculate vram addresses in the linker script.
     # If disabled, this uses the plain integer values for vram addresses defined in the yaml.
     ld_use_symbolic_vram_addresses: bool
+    # Ensures segments of vram classes with dependencies (`vram_symbol` / `follows_classes`) are written to the linker script AFTER the segments they depend on.
+    ld_sort_segments_by_vram_class_dependency: bool
     # Change linker script generation to allow partially linking segments. Requires both `ld_partial_scripts_path` and `ld_partial_build_segments_path` to be set.
     ld_partial_linking: bool
     # Folder were each intermediary linker script will be written to.
@@ -512,6 +514,9 @@ def _parse_yaml(
         ld_sections_allowlist=p.parse_opt("ld_sections_allowlist", list, []),
         ld_sections_denylist=p.parse_opt("ld_sections_denylist", list, []),
         ld_wildcard_sections=p.parse_opt("ld_wildcard_sections", bool, False),
+        ld_sort_segments_by_vram_class_dependency=p.parse_opt(
+            "ld_sort_segments_by_vram_class_dependency", bool, False
+        ),
         ld_use_symbolic_vram_addresses=p.parse_opt(
             "ld_use_symbolic_vram_addresses", bool, True
         ),