splat64 0.36.3__tar.gz → 0.37.0__tar.gz

{splat64-0.36.3 → splat64-0.37.0}/.github/workflows/unit_tests.yml

@@ -6,7 +6,7 @@ on:
 
 jobs:
   unit_tests:
-    name: unit_tests on ${{ matrix.os.name }}
+    name: unit_tests on '${{ matrix.os.name }}', Python ${{ matrix.py_version }}
     runs-on: ${{ matrix.os.runner }}
     strategy:
       fail-fast: false
@@ -15,19 +15,27 @@ jobs:
           {
             name: linux,
             runner: ubuntu-latest,
-            python_venv: .venv/bin/python3,
+            python_venv_activate: source .venv/bin/activate,
           },
           {
             name: macos,
             runner: macos-latest,
-            python_venv: .venv/bin/python3,
+            python_venv_activate: source .venv/bin/activate,
           },
           {
             name: windows,
             runner: windows-latest,
-            python_venv: .venv/Scripts/python3,
+            python_venv_activate: .venv/Scripts/Activate.ps1,
           },
         ]
+        py_version:
+          - '3.9'
+          - '3.10'
+          - '3.11'
+          - '3.12'
+          - '3.13'
+          - '3.14'
+          - '3.x' # Explicit latest
 
     steps:
       - name: Checkout
@@ -35,16 +43,29 @@ jobs:
 
       - name: Install dependencies
         if: matrix.os.name == 'linux'
-        run: sudo apt-get install -y build-essential make binutils-mips-linux-gnu python3 python3-pip python3-venv wget
+        run: sudo apt-get install -y build-essential make binutils-mips-linux-gnu wget
+
+      - name: Print Python version
+        run: |
+          python3 --version
+
+      - name: Set up Python ${{ matrix.py_version }}
+        uses: actions/setup-python@v6
+        with:
+          python-version: ${{ matrix.py_version }}
 
       - name: Setup Python venv
         run: |
+          python3 --version
           python3 -m venv .venv
+          ${{ matrix.os.python_venv_activate }}
+          python3 --version
 
       - name: Install Python dependencies
         run: |
-          ${{ matrix.os.python_venv }} -m pip install -U -r requirements.txt
-          ${{ matrix.os.python_venv }} -m pip install -e .
+          ${{ matrix.os.python_venv_activate }}
+          python3 -m pip install -U -r requirements.txt
+          python3 -m pip install -e .
 
       - name: Build `basic_app` on ${{ matrix.os.name }}
         if: matrix.os.name == 'linux'
@@ -57,4 +78,5 @@ jobs:
 
       - name: Run the test
         run: |
-          ${{ matrix.os.python_venv }} test.py
+          ${{ matrix.os.python_venv_activate }}
+          python3 test.py

{splat64-0.36.3 → splat64-0.37.0}/.gitignore

@@ -8,3 +8,4 @@ __pycache__/
 *.n64
 *.yaml
 *.z64
+uv.lock

{splat64-0.36.3 → splat64-0.37.0}/CHANGELOG.md

@@ -1,5 +1,26 @@
 # splat Release Notes
 
+### 0.37.0
+
+* Add define check to allow using `macro.inc` instead of `labels.inc` in `include_asm.h`.
+  * The macro must be `INCLUDE_ASM_USE_MACRO_INC` and have a non zero value.
+* New option: `generated_asm_macros_directory`
+  * Allow changing the directory for the generated assembly macros files.
+* Add workaround for the short loop bug that happens in SN PS2 compilers.
+  * This workaround is only used when the selected compiler is `EEGCC`.
+    * This can be manually overriden with the option `align_on_branch_labels`.
+  * Emit an `.align` directive in branch labels to avoid triggering the bug.
+  * The bug in question adds a non-zero number of nops as a workaround for a hardware defect, producing non-matching builds.
+
+### 0.36.4
+
+* Fix missing dollar signs (`$`) on specific ps2 registers on the extra assembly files generated by `make_full_disasm_for_code`.
+* Pin versions of dependencies to avoid breaking because they broke.
+* Add more checks for texture and palette segments to have a correct size in their yaml entries, ensuring their `width`/`height` or `size` attributes match the sizes relative to other segments.
+* Fix `gfx` segments sometimes not picking up properly the corresponding typed reference.
+* Fix sometimes picking a name symbol from a different segment instead of the owned segment when both symbols have the same address but are from completely different segments.
+* Add PS2 compatibility to `create_config`.
+
 ### 0.36.3
 
 * Fix not generating nonmatching `.s` files for quoted symbols.

{splat64-0.36.3 → splat64-0.37.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: splat64
-Version: 0.36.3
+Version: 0.37.0
 Summary: A binary splitting tool to assist with decompilation and modding projects
 Project-URL: Repository, https://github.com/ethteck/splat
 Project-URL: Issues, https://github.com/ethteck/splat/issues
@@ -30,27 +30,27 @@ License-File: LICENSE
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3
 Requires-Python: >=3.9
-Requires-Dist: colorama
-Requires-Dist: intervaltree
-Requires-Dist: pylibyaml
-Requires-Dist: pyyaml
-Requires-Dist: tqdm
+Requires-Dist: colorama==0.4.6
+Requires-Dist: intervaltree==3.1.0
+Requires-Dist: pylibyaml==0.1.0
+Requires-Dist: pyyaml==6.0.3
+Requires-Dist: tqdm==4.67.1
 Provides-Extra: dev
 Requires-Dist: crunch64<1.0.0,>=0.5.1; extra == 'dev'
 Requires-Dist: mypy; extra == 'dev'
 Requires-Dist: n64img>=0.3.3; extra == 'dev'
-Requires-Dist: pygfxd; extra == 'dev'
+Requires-Dist: pygfxd>=1.0.5; extra == 'dev'
 Requires-Dist: rabbitizer<2.0.0,>=1.12.0; extra == 'dev'
 Requires-Dist: ruff; extra == 'dev'
-Requires-Dist: spimdisasm<2.0.0,>=1.38.0; extra == 'dev'
+Requires-Dist: spimdisasm<2.0.0,>=1.39.0; extra == 'dev'
 Requires-Dist: types-colorama; extra == 'dev'
 Requires-Dist: types-pyyaml; extra == 'dev'
 Provides-Extra: mips
 Requires-Dist: crunch64<1.0.0,>=0.5.1; extra == 'mips'
 Requires-Dist: n64img>=0.3.3; extra == 'mips'
-Requires-Dist: pygfxd; extra == 'mips'
+Requires-Dist: pygfxd>=1.0.5; extra == 'mips'
 Requires-Dist: rabbitizer<2.0.0,>=1.12.0; extra == 'mips'
-Requires-Dist: spimdisasm<2.0.0,>=1.38.0; extra == 'mips'
+Requires-Dist: spimdisasm<2.0.0,>=1.39.0; extra == 'mips'
 Description-Content-Type: text/markdown
 
 # splat
@@ -76,7 +76,7 @@ The brackets corresponds to the optional dependencies to install while installin
 If you use a `requirements.txt` file in your repository, then you can add this library with the following line:
 
 ```txt
-splat64[mips]>=0.36.3,<1.0.0
+splat64[mips]>=0.37.0,<1.0.0
 ```
 
 ### Optional dependencies

{splat64-0.36.3 → splat64-0.37.0}/README.md

@@ -21,7 +21,7 @@ The brackets corresponds to the optional dependencies to install while installin
 If you use a `requirements.txt` file in your repository, then you can add this library with the following line:
 
 ```txt
-splat64[mips]>=0.36.3,<1.0.0
+splat64[mips]>=0.37.0,<1.0.0
 ```
 
 ### Optional dependencies

{splat64-0.36.3 → splat64-0.37.0}/create_config.py

@@ -1,9 +1,7 @@
 #! /usr/bin/env python3
 
-from pathlib import Path
-
 import src.splat as splat
 
 if __name__ == "__main__":
     args = splat.scripts.create_config.parser.parse_args()
-    splat.scripts.create_config.main(Path(args.file))
+    splat.scripts.create_config.process_arguments(args)

{splat64-0.36.3 → splat64-0.37.0}/docs/Configuration.md

@@ -147,6 +147,12 @@ Allows configuring the style used by the generated `INCLUDE_ASM` macro. It curre
 
 Defaults to `default`.
 
+### generated_asm_macros_directory
+
+Change the directory where the assembly macros files are written to.
+
+Defaults to `include`.
+
 ### o_as_suffix
 
 Used to determine the file extension of the built files that will be listed on the linker script.
@@ -932,3 +938,21 @@ use_legacy_include_asm: True
 
 #### Default
 `False`
+
+## align_on_branch_labels
+
+If enabled emit no-op alignment directives on all branch labels.
+
+This is useful as a workaround for the "short loop bug" present in SN PS2 compilers, in which the compiler may decide to insert extra `nop` instructions due to a hardware defect, producing non-matching builds.
+
+This option is enabled automatically if the selected compiler is known to have this issue, but this option allows to override the default selection in case this setting ends up producing problems.
+
+#### Usage
+
+```yaml
+align_on_branch_labels: False
+```
+
+#### Default
+
+`True` if [`compiler`](#compiler) is set to `EEGCC`, `False` otherwise

{splat64-0.36.3 → splat64-0.37.0}/docs/Home.md

@@ -22,4 +22,4 @@ After scanning and splitting, **splat** will output a linker script that can be
 
 ### Sounds great, how do I get started?
 
-Have a look at the [Quickstart](https://github.com/ethteck/splat/wiki/Quickstart), or check out the [Examples](https://github.com/ethteck/splat/wiki/Examples) page to see projects that are using **splat**.
+Have a look at the [Quickstart](https://github.com/ethteck/splat/wiki/Quickstart), [Quickstart for PS2 ELFs](https://github.com/ethteck/splat/wiki/Quickstart-Elf), or check out the [Examples](https://github.com/ethteck/splat/wiki/Examples) page to see projects that are using **splat**.

splat64-0.37.0/docs/Quickstart-Elf.md

@@ -0,0 +1,199 @@
+# Quickstart Elf
+
+> [!NOTE]
+> This quickstart is written with PS2 ELFs in mind, relocatable ELFs are not supported. It is also assumed that you are using Ubuntu 22.04 either natively, via WSL2 or via Docker.
+
+For the purposes of this quickstart, we will assume that we are going to split a game called `mygame` and we have the ELF from the iso named `SLUS_XXX.YY`.
+
+Create a directory for `~/mygame` and `cd` into it:
+
+```sh
+mkdir -p ${HOME}/mygame && cd ${HOME}/mygame
+```
+
+Copy the `SLUS_XXX.YY` file into the `mygame` directory inside your home directory.
+
+## System packages
+
+### Python 3.9
+
+Ensure you are have **Python 3.9** or higher installed:
+
+```sh
+$ python3 --version
+Python 3.9.10
+```
+
+If you get `bash: python3: command not found` install it with the following command:
+
+```sh
+sudo apt update && sudo apt install -y python3 python3-pip
+```
+
+### MIPS binutils
+
+Ensure you have a MIPS binutils installed in your PC. Specifically we'll need a MIPS `objcopy`.
+
+```sh
+$ mips-linux-gnu-objcopy --version
+GNU objcopy (GNU Binutils for Ubuntu) 2.38
+```
+
+If you get an error then install it with the following command:
+
+```sh
+sudo apt install binutils-mips-linux-gnu
+```
+
+## Install splat
+
+We'll install `splat` using `pip` and enable its `mips` dependencies:
+
+```sh
+python3 -m pip install -U splat64[mips]
+```
+
+## Create a config file from your ELF
+
+`splat` has a script that will generate a `yaml` file based in the ELF file.
+
+```sh
+python3 -m splat create_config SLUS_XXX.YY
+```
+
+This command generates a few files, from which the most important ones are the `SLUS_XXX.YY.rom` and the `SLUS_XXX.YY.yaml`.
+
+### The generated ROM
+
+The original ELF file contains the game code and a lot of extra metadata which we don't care about.
+
+To get rid of the extra metadata the `create_config` script generated a ROM that only contains the game code and data, and the generated `yaml` is used to split this ROM instead of splitting the original ELF, making the splitting and build process a lot more cleaner.
+
+To generate this ROM the `create_config` script uses `objcopy`. You can (and should) integrate this ROM generation step into your setup script / configure script. `create_config` tells you exactly what command it used to generate the ROM, which should look similar to the following:
+
+```sh
+mips-linux-gnu-objcopy -O binary --gap-fill=0x00 SLUS_XXX.YY SLUS_XXX.YY.rom
+```
+
+This approach has a few pros and cons. The biggest pro is being able to generate an ELF with proper metadata as part of your build system, making a lot easier to achieve shiftability eventually. Note only this is not enough for shiftability, other factors must be worked out, like symbol alignment, fake symbols, etc.
+
+### The generated `yaml`
+
+`create_config` generates a `yaml` file which describes how the ROM is structured, what parts are code, data, etc. It also includes configuration values for splat.
+
+Below is an example of what a generated yaml may look like:
+
+```yaml
+# name: Your game name here!
+sha1: 6da2d0a02aafe3bfba71ca8ba859174756ba3f5e
+options:
+  basename: SLUS_206.24
+  target_path: SLUS_206.24.rom
+  elf_path: build/SLUS_206.24.elf
+  base_path: .
+  platform: ps2
+  compiler: EEGCC
+
+  gp_value: 0x003A0EF0
+  ld_gp_expression: cod_SBSS_START + 0x7FF0
+
+  # asm_path: asm
+  # src_path: src
+  # build_path: build
+
+  ld_script_path: SLUS_206.24.ld
+  ld_dependencies: True
+  ld_wildcard_sections: True
+  ld_bss_contains_common: True
+
+  create_asm_dependencies: True
+
+  find_file_boundaries: False
+
+  o_as_suffix: True
+
+  symbol_addrs_path:
+    - symbol_addrs.txt
+  reloc_addrs_path:
+    - reloc_addrs.txt
+
+  # undefined_funcs_auto_path: undefined_funcs_auto.txt
+  # undefined_syms_auto_path: undefined_syms_auto.txt
+
+  extensions_path: tools/splat_ext
+
+  string_encoding: ASCII
+  data_string_encoding: ASCII
+  rodata_string_guesser_level: 2
+  data_string_guesser_level: 2
+
+  named_regs_for_c_funcs: False
+
+  section_order:
+    - .text
+    # - .vutext
+    - .data
+    - .rodata
+    - .gcc_except_table
+    - .sbss
+    - .bss
+
+  auto_link_sections:
+    - .data
+    - .rodata
+    - .gcc_except_table
+    - .sbss
+    - .bss
+
+segments:
+  - name: cod
+    type: code
+    start: 0x000000
+    vram: 0x00100000
+    bss_size: 0x6907C
+    subalign: null
+    subsegments:
+      - [0x000000, asm, cod/000000] # .text
+      - [0x3209C0, textbin, cod/3209C0] # .vutext
+      - [0x324680, data, cod/324680] # .data
+      - [0x350100, rodata, cod/350100] # .rodata
+      - [0x3B0780, gcc_except_table, cod/3B0780] # .gcc_except_table
+      - { type: sbss, vram: 0x004B0880, name: cod/004B0880 } # .sbss
+      - { type: bss, vram: 0x004B0F80, name: cod/004B0F80 } # .bss
+  - [0x3B0864]
+```
+
+This is a bare-bones configuration and there is a lot of work required to map out the different sections of the ROM.
+
+## Run splat with your configuration
+
+```sh
+python3 -m splat split SLUS_XXX.YY.yaml
+```
+
+The output will look something like this:
+
+```plain_text
+splat 0.36.2 (powered by spimdisasm 1.38.0)
+Loading symbols (symbol_addrs): 100%|█████████████████████████| 2/2 [00:00<00:00, 9868.95it/s]
+Scanning cod:   0%|                                                     | 0/1 [00:00<?, ?it/s]
+Rodata segment 'cod/350100' may belong to the text segment 'cod/000000'
+    Based on the usage from the function func_00102E48 to the symbol D_00450308
+Scanning cod: 100%|█████████████████████████████████████████████| 1/1 [00:26<00:00, 26.62s/it]
+Splitting cod: 100%|████████████████████████████████████████████| 1/1 [00:14<00:00, 14.77s/it]
+Linker script cod: 100%|███████████████████████████████████████| 1/1 [00:00<00:00, 447.54it/s]
+Split 3 MB (85.17%) in defined segments
+                 asm:     3 MB (84.76%) 1 split, 0 cached
+             textbin:    15 KB (0.40%) 1 split, 0 cached
+             unknown:      0 B (0.00%) from unknown bin files
+```
+
+It's up to you to figure out the layout of the ROM, finding proper file splits.
+
+## Next Steps
+
+The reassembly of the ROM is currently out of scope of this quickstart, as is switching out the `asm` segments for `c` or `cpp`.
+
+You can find a general workflow for using `splat` at [General Workflow](https://github.com/ethteck/splat/wiki/General-Workflow)
+
+Please feel free to improve this guide!

{splat64-0.36.3 → splat64-0.37.0}/docs/Quickstart.md

@@ -1,6 +1,7 @@
 # Quickstart
 
-> **Note**: This quickstart is written with N64 ROMs in mind, and the assumption that you are using Ubuntu 20.04 either natively, via WSL2 or via Docker.
+> [!NOTE]
+> This quickstart is written with N64 ROMs in mind, and the assumption that you are using Ubuntu 20.04 either natively, via WSL2 or via Docker.
 
 For the purposes of this quickstart, we will assume that we are going to split a game called `mygame` and we have the ROM in `.z64` format named `baserom.z64`.
 
@@ -19,7 +20,7 @@ Copy the `baserom.z64` file into the `mygame` directory inside your home directo
 Ensure you are have **Python 3.9** or higher installed:
 
 ```sh
-python3 --version
+$ python3 --version
 Python 3.9.10
 ```
 

{splat64-0.36.3 → splat64-0.37.0}/pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "splat64"
 # Should be synced with src/splat/__init__.py
-version = "0.36.3"
+version = "0.37.0"
 description = "A binary splitting tool to assist with decompilation and modding projects"
 readme = "README.md"
 license = {file = "LICENSE"}
@@ -11,18 +11,18 @@ classifiers = [
     "License :: OSI Approved :: MIT License",
 ]
 dependencies = [
-    "PyYAML",
-    "pylibyaml",
-    "tqdm",
-    "intervaltree",
-    "colorama",
+    "PyYAML==6.0.3",
+    "pylibyaml==0.1.0",
+    "tqdm==4.67.1",
+    "intervaltree==3.1.0",
+    "colorama==0.4.6",
 ]
 
 [project.optional-dependencies]
 mips = [
-    "spimdisasm>=1.38.0,<2.0.0", # This value should be keep in sync with the version listed on disassembler/spimdisasm_disassembler.py
+    "spimdisasm>=1.39.0,<2.0.0", # This value should be keep in sync with the version listed on disassembler/spimdisasm_disassembler.py
     "rabbitizer>=1.12.0,<2.0.0",
-    "pygfxd",
+    "pygfxd>=1.0.5",
     "n64img>=0.3.3",
     "crunch64>=0.5.1,<1.0.0",
 ]

{splat64-0.36.3 → splat64-0.37.0}/requirements.txt

@@ -1,11 +1,11 @@
-PyYAML
-pylibyaml
-tqdm
-intervaltree
-colorama
+PyYAML==6.0.3
+pylibyaml==0.1.0
+tqdm==4.67.1
+intervaltree==3.1.0
+colorama==0.4.6
 # This value should be keep in sync with the version listed on disassembler/spimdisasm_disassembler.py and pyproject.toml
-spimdisasm>=1.38.0
+spimdisasm>=1.39.0
 rabbitizer>=1.10.0
-pygfxd
+pygfxd>=1.0.5
 n64img>=0.1.4
 crunch64>=0.2.0

{splat64-0.36.3 → splat64-0.37.0}/src/splat/__init__.py

@@ -1,7 +1,7 @@
 __package_name__ = __name__
 
 # Should be synced with pyproject.toml
-__version__ = "0.36.3"
+__version__ = "0.37.0"
 __author__ = "ethteck"
 
 from . import util as util

{splat64-0.36.3 → splat64-0.37.0}/src/splat/disassembler/spimdisasm_disassembler.py

@@ -7,7 +7,7 @@ from typing import Set
 
 class SpimdisasmDisassembler(disassembler.Disassembler):
     # This value should be kept in sync with the version listed on requirements.txt and pyproject.toml
-    SPIMDISASM_MIN = (1, 38, 0)
+    SPIMDISASM_MIN = (1, 39, 0)
 
     def configure(self):
         # Configure spimdisasm
@@ -74,6 +74,8 @@ class SpimdisasmDisassembler(disassembler.Disassembler):
             spimdisasm.common.GlobalConfig.SYMBOL_FINDER_FILTERED_ADDRESSES_AS_HILO = (
                 False
             )
+        if options.opts.align_on_branch_labels:
+            spimdisasm.common.GlobalConfig.ASM_EMIT_ALIGN_BRANCH_LABELS = True
         rabbitizer.config.toolchainTweaks_treatJAsUnconditionalBranch = (
             selected_compiler.j_as_branch
         )