splat64 0.32.3__tar.gz → 0.33.0__tar.gz

splat64-0.33.0/.gitattributes

@@ -0,0 +1,2 @@
+# Force LF on everyfile
+* text eol=lf

{splat64-0.32.3 → splat64-0.33.0}/.github/workflows/mypy.yml

@@ -22,6 +22,7 @@ jobs:
         pip install mypy
         pip install -r requirements.txt
         pip install types-PyYAML
+        pip install -e .
 
     - name: mypy splat lib
       run: mypy --show-column-numbers --hide-error-context src/splat

splat64-0.33.0/.github/workflows/unit_tests.yml

@@ -0,0 +1,60 @@
+name: unit_tests
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  unit_tests:
+    name: unit_tests on ${{ matrix.os.name }}
+    runs-on: ${{ matrix.os.runner }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [
+          {
+            name: linux,
+            runner: ubuntu-latest,
+            python_venv: .venv/bin/python3,
+          },
+          {
+            name: macos,
+            runner: macos-latest,
+            python_venv: .venv/bin/python3,
+          },
+          {
+            name: windows,
+            runner: windows-latest,
+            python_venv: .venv/Scripts/python3,
+          },
+        ]
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install dependencies
+        if: matrix.os.name == 'linux'
+        run: sudo apt-get install -y build-essential make binutils-mips-linux-gnu python3 python3-pip python3-venv wget
+
+      - name: Setup Python venv
+        run: |
+          python3 -m venv .venv
+
+      - name: Install Python dependencies
+        run: |
+          ${{ matrix.os.python_venv }} -m pip install -U -r requirements.txt
+          ${{ matrix.os.python_venv }} -m pip install -e .
+
+      - name: Build `basic_app` on ${{ matrix.os.name }}
+        if: matrix.os.name == 'linux'
+        # Linux CI checks if any of the test C code has changed without updating the generated binary
+        run: |
+          make -C test/basic_app clean
+          make -C test/basic_app download_kmc
+          make -C test/basic_app all
+          git diff --exit-code test/basic_app/build/basic_app.bin
+
+      - name: Run the test
+        run: |
+          ${{ matrix.os.python_venv }} test.py

{splat64-0.32.3 → splat64-0.33.0}/CHANGELOG.md

@@ -1,9 +1,22 @@
 # splat Release Notes
 
+### 0.33.0
+
+* BREAKING: Move splat config loading and parsing into "splat.util.conf" to modularize this part of the code and allow projects to interface with segment loading more gracefully.
+  * `initialize_config` is now `splat.util.conf.load`. It takes `Path`s instead of `str`s for YAML paths. It will no longer `exit` if config merging fails, but will throw a `TypeError` which can be handled as desired by callers.
+* Fix splat on Windows not using forward slashes on generated paths.
+* Setup CI to be run on Windows and MacOS too.
+* Fix output of incbin segments for SN64 projects
+* New yaml option: `make_full_disasm_for_code`
+  * Emit a full `.s` file for each `c`/`cpp` segment besides the generated `nonmatchings` individual functions.
+  * Can be used to generate "target" or "expected" objects for asm diffing.
+  * Also available as a cli parameter: `--make-full-disasm-for-code`
+* `spimdisasm` 1.33.0 or above is now required.
+
 ### 0.32.3
 
-- Fix "unrecognized YAML option" error if disassemble_all is provided via CLI and as a YAML option.
-- Slightly speed up `bss` disassembly for projects over thousands of subsegments in the same top-level segment.
+* Fix "unrecognized YAML option" error if disassemble_all is provided via CLI and as a YAML option.
+* Slightly speed up `bss` disassembly for projects over thousands of subsegments in the same top-level segment.
 
 ### 0.32.2
 

{splat64-0.32.3 → splat64-0.33.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: splat64
-Version: 0.32.3
+Version: 0.33.0
 Summary: A binary splitting tool to assist with decompilation and modding projects
 Project-URL: Repository, https://github.com/ethteck/splat
 Project-URL: Issues, https://github.com/ethteck/splat/issues
@@ -76,7 +76,7 @@ The brackets corresponds to the optional dependencies to install while installin
 If you use a `requirements.txt` file in your repository, then you can add this library with the following line:
 
 ```txt
-splat64[mips]>=0.32.3,<1.0.0
+splat64[mips]>=0.33.0,<1.0.0
 ```
 
 ### Optional dependencies

{splat64-0.32.3 → splat64-0.33.0}/README.md

@@ -21,7 +21,7 @@ The brackets corresponds to the optional dependencies to install while installin
 If you use a `requirements.txt` file in your repository, then you can add this library with the following line:
 
 ```txt
-splat64[mips]>=0.32.3,<1.0.0
+splat64[mips]>=0.33.0,<1.0.0
 ```
 
 ### Optional dependencies

{splat64-0.32.3 → splat64-0.33.0}/docs/Configuration.md

@@ -403,20 +403,84 @@ ld_symbol_header_path: path/to/linker_symbol_header
 
 ### ld_discard_section
 
-Determines whether to add a discard section to the linker script
+Determines whether to add a wildcard discard section to the linker script.
+
+This tells the linker that every section not explicitly listed on the linker script will be discarded.
+
+### ld_sections_allowlist
+
+A list of sections to preserve during link time. It can be useful to preserve debugging sections.
+
+#### Usage
+
+```yaml
+  ld_sections_allowlist:
+    - .shstrtab
+    - .mdebug
+    - .mdebug.abi32
+```
+
+Generates entries at the bottom of the linker script like:
+
+```text
+    .shstrtab 0 :
+    {
+        *(.shstrtab);
+    }
+    .mdebug 0 :
+    {
+        *(.mdebug);
+    }
+    .mdebug.abi32 0 :
+    {
+        *(.mdebug.abi32);
+    }
+```
+
+### ld_sections_denylist
+
+A list of sections to discard during link time. It can be useful to avoid using the [wildcard discard](#ld_discard_section).
+
+Note this option does not turn off [`ld_discard_section`](#ld_discard_section), neither checks if the listed sections overlap with it.
+
+#### Usage
+
+```yaml
+  ld_sections_denylist:
+    - .reginfo
+    - .MIPS.abiflags
+    - .MIPS.options
+    - .note.gnu.build-id
+    - .interp
+    - .eh_frame
+```
+
+Generates a discard section like this:
+
+```text
+    /DISCARD/ :
+    {
+        *(.reginfo);
+        *(.MIPS.abiflags);
+        *(.MIPS.options);
+        *(.note.gnu.build-id);
+        *(.interp);
+        *(.eh_frame);
+    }
+```
 
 ### ld_wildcard_sections
 
 Determines whether to add wildcards for section linking in the linker script (.rodata* for example)
 
-### ld_use_symbolic_vram_addreses
+### ld_use_symbolic_vram_addresses
 
 Determines whether to use `follows_vram` (segment option) and `vram_symbol` / `follows_classes` (vram_class options) to calculate vram addresses in the linker script.
 Enabled by default. If disabled, this uses the plain integer values for vram addresses defined in the yaml.
 
 ### ld_partial_linking
 
-Change linker script generation to allow partially linking segments. Requires both `ld_partial_scripts_path` and `ld_partial_build_segments_path` to be set.
+Change linker script generation to allow partially linking segments. Requires both [`ld_partial_scripts_path`](#ld_partial_scripts_path) and [`ld_partial_build_segments_path`](#ld_partial_build_segments_path) to be set.
 
 ### ld_partial_scripts_path
 
@@ -542,7 +606,7 @@ Determine the format that symbols should be named by default
 
 ### symbol_name_format_no_rom
 
-Same as `symbol_name_format` but for symbols with no rom address
+Same as [`symbol_name_format`](#symbol_name_format) but for symbols with no rom address
 
 ### find_file_boundaries
 
@@ -663,6 +727,12 @@ Tries to detect redundant and unreferenced functions ends and merge them togethe
 
 Don't skip disassembling already matched functions and migrated sections
 
+### make_full_disasm_for_code
+
+Emit a full `.s` file for each `c`/`cpp` segment besides the generated `nonmatchings` individual functions.
+
+Can be used to generate "target" or "expected" objects for asm diffing.
+
 ### global_vram_start and global_vram_end
 
 Allow specifying that the global memory range may be larger than what was automatically detected.

{splat64-0.32.3 → splat64-0.33.0}/pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "splat64"
 # Should be synced with src/splat/__init__.py
-version = "0.32.3"
+version = "0.33.0"
 description = "A binary splitting tool to assist with decompilation and modding projects"
 readme = "README.md"
 license = {file = "LICENSE"}

{splat64-0.32.3 → splat64-0.33.0}/requirements.txt

@@ -4,7 +4,7 @@ tqdm
 intervaltree
 colorama
 # This value should be keep in sync with the version listed on disassembler/spimdisasm_disassembler.py and pyproject.toml
-spimdisasm>=1.32.0
+spimdisasm>=1.33.0
 rabbitizer>=1.10.0
 pygfxd
 n64img>=0.1.4

{splat64-0.32.3 → splat64-0.33.0}/src/splat/__init__.py

@@ -1,7 +1,7 @@
 __package_name__ = __name__
 
 # Should be synced with pyproject.toml
-__version__ = "0.32.3"
+__version__ = "0.33.0"
 __author__ = "ethteck"
 
 from . import util as util

{splat64-0.32.3 → splat64-0.33.0}/src/splat/disassembler/spimdisasm_disassembler.py

@@ -7,7 +7,7 @@ from typing import Set
 
 class SpimdisasmDisassembler(disassembler.Disassembler):
     # This value should be kept in sync with the version listed on requirements.txt and pyproject.toml
-    SPIMDISASM_MIN = (1, 32, 0)
+    SPIMDISASM_MIN = (1, 33, 0)
 
     def configure(self):
         # Configure spimdisasm

{splat64-0.32.3 → splat64-0.33.0}/src/splat/scripts/split.py

@@ -10,10 +10,6 @@ from .. import __package_name__, __version__
 from ..disassembler import disassembler_instance
 from ..util import cache_handler, progress_bar, vram_classes, statistics
 
-# This unused import makes the yaml library faster. don't remove
-import pylibyaml  # pyright: ignore
-import yaml
-
 from colorama import Fore, Style
 from intervaltree import Interval, IntervalTree
 import sys
@@ -23,7 +19,7 @@ from ..segtypes.linker_entry import (
     get_segment_vram_end_symbol_name,
 )
 from ..segtypes.segment import Segment
-from ..util import log, options, palettes, symbols, relocs
+from ..util import conf, log, options, palettes, symbols, relocs
 
 linker_writer: LinkerWriter
 config: Dict[str, Any]
@@ -142,34 +138,6 @@ def assign_symbols_to_segments():
                     seg.add_symbol(symbol)
 
 
-def merge_configs(main_config, additional_config):
-    # Merge rules are simple
-    # For each key in the dictionary
-    # - If list then append to list
-    # - If a dictionary then repeat merge on sub dictionary entries
-    # - Else assume string or number and replace entry
-
-    for curkey in additional_config:
-        if curkey not in main_config:
-            main_config[curkey] = additional_config[curkey]
-        elif type(main_config[curkey]) != type(additional_config[curkey]):
-            log.error(f"Type for key {curkey} in configs does not match")
-        else:
-            # keys exist and match, see if a list to append
-            if type(main_config[curkey]) == list:
-                main_config[curkey] += additional_config[curkey]
-            elif type(main_config[curkey]) == dict:
-                # need to merge sub areas
-                main_config[curkey] = merge_configs(
-                    main_config[curkey], additional_config[curkey]
-                )
-            else:
-                # not a list or dictionary, must be a number or string, overwrite
-                main_config[curkey] = additional_config[curkey]
-
-    return main_config
-
-
 def brief_seg_name(seg: Segment, limit: int, ellipsis="…") -> str:
     s = seg.name.strip()
     if len(s) > limit:
@@ -203,25 +171,6 @@ def calc_segment_dependences(
     return vram_class_to_follows_segments
 
 
-def initialize_config(
-    config_path: List[str],
-    modes: Optional[List[str]],
-    verbose: bool,
-    disassemble_all: bool = False,
-) -> Dict[str, Any]:
-    config: Dict[str, Any] = {}
-    for entry in config_path:
-        with open(entry) as f:
-            additional_config = yaml.load(f.read(), Loader=yaml.SafeLoader)
-        config = merge_configs(config, additional_config)
-
-    vram_classes.initialize(config.get("vram_classes"))
-
-    options.initialize(config, config_path, modes, verbose, disassemble_all)
-
-    return config
-
-
 def read_target_binary() -> bytes:
     rom_bytes = options.opts.target_path.read_bytes()
 
@@ -493,20 +442,27 @@ def dump_symbols() -> None:
 
 
 def main(
-    config_path: List[str],
+    config_path: List[Path],
     modes: Optional[List[str]],
     verbose: bool,
     use_cache: bool = True,
     skip_version_check: bool = False,
     stdout_only: bool = False,
     disassemble_all: bool = False,
+    make_full_disasm_for_code=False,
 ):
     if stdout_only:
         progress_bar.out_file = sys.stdout
 
     # Load config
     global config
-    config = initialize_config(config_path, modes, verbose, disassemble_all)
+    config = conf.load(
+        config_path,
+        modes,
+        verbose,
+        disassemble_all,
+        make_full_disasm_for_code,
+    )
 
     disassembler_instance.create_disassembler_instance(skip_version_check, __version__)
 
@@ -565,7 +521,10 @@ def main(
 
 def add_arguments_to_parser(parser: argparse.ArgumentParser):
     parser.add_argument(
-        "config", help="path to a compatible config .yaml file", nargs="+"
+        "config",
+        help="path to a compatible config .yaml file",
+        nargs="+",
+        type=Path,
     )
     parser.add_argument("--modes", nargs="+", default="all")
     parser.add_argument("--verbose", action="store_true", help="Enable debug logging")
@@ -585,6 +544,11 @@ def add_arguments_to_parser(parser: argparse.ArgumentParser):
         help="Disasemble matched functions and migrated data",
         action="store_true",
     )
+    parser.add_argument(
+        "--make-full-disasm-for-code",
+        help="Emit a full `.s` file for each `c`/`cpp` segment besides the generated `nonmatchings` individual functions",
+        action="store_true",
+    )
 
 
 def process_arguments(args: argparse.Namespace):
@@ -596,6 +560,7 @@ def process_arguments(args: argparse.Namespace):
         args.skip_version_check,
         args.stdout_only,
         args.disassemble_all,
+        args.make_full_disasm_for_code,
     )
 
 

splat64-0.33.0/src/splat/segtypes/common/asm.py

@@ -0,0 +1,29 @@
+from pathlib import Path
+from typing import Optional, List
+
+from ...util import options
+
+from .codesubsegment import CommonSegCodeSubsegment
+
+
+class CommonSegAsm(CommonSegCodeSubsegment):
+    @staticmethod
+    def is_text() -> bool:
+        return True
+
+    def get_section_flags(self) -> Optional[str]:
+        return "ax"
+
+    def scan(self, rom_bytes: bytes):
+        if (
+            self.rom_start is not None
+            and self.rom_end is not None
+            and self.rom_start != self.rom_end
+        ):
+            self.scan_code(rom_bytes)
+
+    def split(self, rom_bytes: bytes):
+        if self.rom_start == self.rom_end:
+            return
+
+        self.split_as_asm_file(self.out_path())

{splat64-0.32.3 → splat64-0.33.0}/src/splat/segtypes/common/asmtu.py

@@ -24,7 +24,7 @@ class CommonSegAsmtu(CommonSegAsm):
 
         with open(out_path, "w", newline="\n") as f:
             # Write `.text` contents
-            for line in self.get_file_header():
+            for line in self.get_asm_file_header():
                 f.write(line + "\n")
             f.write(self.spim_section.disassemble())
 
@@ -40,7 +40,7 @@ class CommonSegAsmtu(CommonSegAsm):
                 if (
                     isinstance(sibling, CommonSegCodeSubsegment)
                     and sibling.spim_section is not None
-                    and not sibling.should_split()
+                    and not sibling.should_self_split()
                 ):
                     f.write("\n")
                     f.write(f"{sibling.get_section_asm_line()}\n\n")
@@ -58,7 +58,7 @@ class CommonSegAsmtu(CommonSegAsm):
                 if (
                     isinstance(sibling, CommonSegCodeSubsegment)
                     and sibling.spim_section is not None
-                    and not sibling.should_split()
+                    and not sibling.should_self_split()
                 ):
                     f.write("\n")
                     f.write(f"{sibling.get_section_asm_line()}\n\n")

{splat64-0.32.3 → splat64-0.33.0}/src/splat/segtypes/common/c.py

@@ -276,6 +276,14 @@ class CommonSegC(CommonSegCodeSubsegment):
                                 spim_rodata_sym, asm_out_dir, rodata_sym
                             )
 
+            if options.opts.make_full_disasm_for_code:
+                # Disable gpRelHack since this file is expected to be built with modern gas
+                section = self.spim_section.get_section()
+                old_value = section.getGpRelHack()
+                section.setGpRelHack(False)
+                self.split_as_asm_file(self.asm_out_path())
+                section.setGpRelHack(old_value)
+
     def get_c_preamble(self):
         ret = []
 
@@ -380,13 +388,16 @@ class CommonSegC(CommonSegCodeSubsegment):
             # IDO uses the asm processor to embeed assembly, and it doesn't require a special directive to include symbols
             asm_outpath = asm_out_dir / self.name / f"{sym.filename}.s"
             rel_asm_outpath = os.path.relpath(asm_outpath, options.opts.base_path)
-            return f'#pragma GLOBAL_ASM("{rel_asm_outpath}")'
+            final_path = Path(rel_asm_outpath).as_posix()
+            return f'#pragma GLOBAL_ASM("{final_path}")'
 
         if options.opts.use_legacy_include_asm:
             rel_asm_out_dir = asm_out_dir.relative_to(options.opts.nonmatchings_path)
-            return f'{macro_name}(const s32, "{rel_asm_out_dir / self.name}", {sym.filename});'
+            final_path = (rel_asm_out_dir / self.name).as_posix()
+            return f'{macro_name}(const s32, "{final_path}", {sym.filename});'
 
-        return f'{macro_name}("{asm_out_dir / self.name}", {sym.filename});'
+        final_path = (asm_out_dir / self.name).as_posix()
+        return f'{macro_name}("{final_path}", {sym.filename});'
 
     def get_c_lines_for_function(
         self,
@@ -448,7 +459,7 @@ class CommonSegC(CommonSegCodeSubsegment):
                     c_lines += self.get_c_lines_for_rodata_sym(rodata_sym, asm_out_dir)
 
         c_path.parent.mkdir(parents=True, exist_ok=True)
-        with c_path.open("w") as f:
+        with c_path.open("w", newline=options.opts.c_newline) as f:
             f.write("\n".join(c_lines))
         log.write(f"Wrote {self.name} to {c_path}")
 
@@ -472,12 +483,12 @@ class CommonSegC(CommonSegCodeSubsegment):
 
         dep_path = build_path / c_path.with_suffix(".asmproc.d")
         dep_path.parent.mkdir(parents=True, exist_ok=True)
-        with dep_path.open("w") as f:
+        with dep_path.open("w", newline=options.opts.c_newline) as f:
             if options.opts.use_o_as_suffix:
                 o_path = build_path / c_path.with_suffix(".o")
             else:
                 o_path = build_path / c_path.with_suffix(c_path.suffix + ".o")
-            f.write(f"{o_path}:")
+            f.write(f"{o_path.as_posix()}:")
             depend_list = []
             for entry in symbols_entries:
                 if entry.function is not None:
@@ -488,7 +499,7 @@ class CommonSegC(CommonSegCodeSubsegment):
                         outpath.parent.mkdir(parents=True, exist_ok=True)
 
                         depend_list.append(outpath)
-                        f.write(f" \\\n    {outpath}")
+                        f.write(f" \\\n    {outpath.as_posix()}")
                 else:
                     for rodata_sym in entry.rodataSyms:
                         rodata_name = rodata_sym.getName()
@@ -498,9 +509,9 @@ class CommonSegC(CommonSegCodeSubsegment):
                             outpath.parent.mkdir(parents=True, exist_ok=True)
 
                             depend_list.append(outpath)
-                            f.write(f" \\\n    {outpath}")
+                            f.write(f" \\\n    {outpath.as_posix()}")
 
             f.write("\n")
 
             for depend_file in depend_list:
-                f.write(f"{depend_file}:\n")
+                f.write(f"{depend_file.as_posix()}:\n")

{splat64-0.32.3 → splat64-0.33.0}/src/splat/segtypes/common/codesubsegment.py

@@ -1,4 +1,5 @@
-from typing import Optional
+from pathlib import Path
+from typing import Optional, List
 
 import spimdisasm
 import rabbitizer
@@ -60,7 +61,7 @@ class CommonSegCodeSubsegment(Segment):
         section.isHandwritten = self.is_hasm
         section.instrCat = self.instr_category
         section.detectRedundantFunctionEnd = self.detect_redundant_function_end
-        section.gpRelHack = not self.use_gp_rel_macro
+        section.setGpRelHack(not self.use_gp_rel_macro)
 
     def scan_code(self, rom_bytes, is_hasm=False):
         self.is_hasm = is_hasm
@@ -194,3 +195,55 @@ class CommonSegCodeSubsegment(Segment):
         return (
             self.extract and options.opts.is_mode_active("code") and self.should_scan()
         )  # only split if the segment was scanned first
+
+    def should_self_split(self) -> bool:
+        return self.should_split()
+
+    def get_asm_file_header(self) -> List[str]:
+        ret = []
+
+        ret.append('.include "macro.inc"')
+        ret.append("")
+
+        ret.extend(self.get_asm_file_extra_directives())
+
+        preamble = options.opts.generated_s_preamble
+        if preamble:
+            ret.append(preamble)
+            ret.append("")
+
+        ret.append(self.get_section_asm_line())
+        ret.append("")
+
+        return ret
+
+    def get_asm_file_extra_directives(self) -> List[str]:
+        ret = []
+
+        ret.append(".set noat")  # allow manual use of $at
+        ret.append(".set noreorder")  # don't insert nops after branches
+        if options.opts.add_set_gp_64:
+            ret.append(".set gp=64")  # allow use of 64-bit general purpose registers
+        ret.append("")
+
+        return ret
+
+    def asm_out_path(self) -> Path:
+        return options.opts.asm_path / self.dir / f"{self.name}.s"
+
+    def out_path(self) -> Optional[Path]:
+        return self.asm_out_path()
+
+    def split_as_asm_file(self, out_path: Optional[Path]):
+        if self.spim_section is None:
+            return
+
+        if out_path:
+            out_path.parent.mkdir(parents=True, exist_ok=True)
+
+            self.print_file_boundaries()
+
+            with open(out_path, "w", newline="\n") as f:
+                for line in self.get_asm_file_header():
+                    f.write(line + "\n")
+                f.write(self.spim_section.disassemble())

{splat64-0.32.3 → splat64-0.33.0}/src/splat/segtypes/common/data.py

@@ -1,5 +1,5 @@
 from pathlib import Path
-from typing import Optional
+from typing import Optional, List
 from ...util import options, symbols, log
 
 from .codesubsegment import CommonSegCodeSubsegment
@@ -38,33 +38,21 @@ class CommonSegData(CommonSegCodeSubsegment, CommonSegGroup):
         if self.rom_start is not None and self.rom_end is not None:
             self.disassemble_data(rom_bytes)
 
+    def get_asm_file_extra_directives(self) -> List[str]:
+        return []
+
     def split(self, rom_bytes: bytes):
         super().split(rom_bytes)
 
-        if self.type.startswith(".") and not options.opts.disassemble_all:
-            return
-
         if self.spim_section is None or not self.should_self_split():
             return
 
-        path = self.asm_out_path()
-
-        path.parent.mkdir(parents=True, exist_ok=True)
-
-        self.print_file_boundaries()
-
-        with path.open("w", newline="\n") as f:
-            f.write('.include "macro.inc"\n\n')
-            preamble = options.opts.generated_s_preamble
-            if preamble:
-                f.write(preamble + "\n")
-
-            f.write(f"{self.get_section_asm_line()}\n\n")
-
-            f.write(self.spim_section.disassemble())
+        self.split_as_asm_file(self.asm_out_path())
 
     def should_self_split(self) -> bool:
-        return options.opts.is_mode_active("data")
+        return options.opts.is_mode_active("data") and (
+            not self.type.startswith(".") or options.opts.disassemble_all
+        )
 
     def should_scan(self) -> bool:
         return True