splat64 0.36.2__py3-none-any.whl → 0.36.4__py3-none-any.whl

splat/segtypes/n64/img.py

@@ -64,6 +64,10 @@ class N64SegImg(Segment):
             log.error(
                 f"Error: {self.name} should end at 0x{self.rom_start + expected_len:X}, but it ends at 0x{self.rom_end:X}\n(hint: add a 'bin' segment after it)"
             )
+        elif actual_len < expected_len:
+            log.error(
+                f"Error: {self.name} should end at 0x{self.rom_start + expected_len:X}, but it ends at 0x{self.rom_end:X}"
+            )
 
     def out_path(self) -> Path:
         type_extension = f".{self.type}" if self.image_type_in_extension else ""

splat/segtypes/n64/palette.py

@@ -23,7 +23,21 @@ class N64SegPalette(Segment):
                     f"segment {self.name} needs to know where it ends; add a position marker [0xDEADBEEF] after it"
                 )
 
-            if not isinstance(self.yaml, dict) or "size" not in self.yaml:
+            if isinstance(self.yaml, dict) and "size" in self.yaml:
+                yaml_size: int = self.yaml["size"]
+                if isinstance(self.rom_start, int) and isinstance(self.rom_end, int):
+                    rom_len = self.rom_end - self.rom_start
+                    if rom_len < yaml_size:
+                        log.error(
+                            f"Error: {self.name} has a `size` value of 0x{yaml_size:X}, but this is smaller than the actual rom size of the palette (0x{rom_len:X}, start: 0x{self.rom_start:X}, end: 0x{self.rom_end:X})"
+                        )
+                    elif rom_len > yaml_size:
+                        log.error(
+                            f"Warning: {self.name} has a `size` value of 0x{yaml_size:X}, but this is larger than the end of the palette (0x{rom_len:X}, start: 0x{self.rom_start:X}, end: 0x{self.rom_end:X})\n(hint add a 'bin' segment after this palette with address 0x{self.rom_end:X})",
+                        )
+
+                size = yaml_size
+            else:
                 assert self.rom_end is not None
                 assert self.rom_start is not None
                 actual_len = self.rom_end - self.rom_start
@@ -39,7 +53,11 @@ class N64SegPalette(Segment):
                     log.error(
                         f"Error: {self.name} (0x{actual_len:X} bytes) is not a valid palette size ({', '.join(hex(s) for s in VALID_SIZES)})\n{hint_msg}"
                     )
+                size = actual_len
+        else:
+            size = 0
 
+        self.palette_size: int = size
         self.global_id: Optional[str] = (
             self.yaml.get("global_id") if isinstance(self.yaml, dict) else None
         )
@@ -61,7 +79,8 @@ class N64SegPalette(Segment):
         return palette
 
     def parse_palette(self, rom_bytes) -> List[Tuple[int, int, int, int]]:
-        data = rom_bytes[self.rom_start : self.rom_end]
+        assert self.rom_start is not None
+        data = rom_bytes[self.rom_start : self.rom_start + self.palette_size]
 
         return N64SegPalette.parse_palette_bytes(data)
 

splat/segtypes/segment.py

@@ -101,15 +101,15 @@ class Segment:
 
         if options.opts.allow_segment_overrides:
             segment_class = Segment.get_extension_segment_class(seg_type)
-            if segment_class == None:
+            if segment_class is None:
                 segment_class = Segment.get_base_segment_class(seg_type)
         else:
             segment_class = Segment.get_base_segment_class(seg_type)
-            if segment_class == None:
+            if segment_class is None:
                 # Look in extensions
                 segment_class = Segment.get_extension_segment_class(seg_type)
 
-        if segment_class == None:
+        if segment_class is None:
             log.error(
                 f"could not load segment type '{seg_type}'\n(hint: confirm your extension directory is configured correctly)"
             )
@@ -538,9 +538,9 @@ class Segment:
 
     @property
     def subalign(self) -> Optional[int]:
-        assert (
-            self.parent is None
-        ), f"subalign is not valid for non-top-level segments. ({self})"
+        assert self.parent is None, (
+            f"subalign is not valid for non-top-level segments. ({self})"
+        )
         return self.given_subalign
 
     @property
@@ -811,8 +811,7 @@ class Segment:
         items = [
             i
             for i in items
-            if i.segment is None
-            or Segment.visible_ram(self, i.segment)
+            if (i.segment is None or Segment.visible_ram(self, i.segment))
             and (type == i.type)
         ]
 

splat/util/__init__.py

@@ -7,6 +7,7 @@ from . import n64 as n64
 from . import options as options
 from . import palettes as palettes
 from . import progress_bar as progress_bar
+from . import ps2 as ps2
 from . import psx as psx
 from . import relocs as relocs
 from . import statistics as statistics

splat/util/cache_handler.py

@@ -20,7 +20,7 @@ class Cache:
                     log.write(f"Loaded cache ({len(self.cache.keys())} items)")
             except Exception:
                 log.write(
-                    f"Not able to load cache file. Discarding old cache", status="warn"
+                    "Not able to load cache file. Discarding old cache", status="warn"
                 )
 
         # invalidate entire cache if options change

splat/util/conf.py

@@ -10,10 +10,10 @@ from typing import Any, Dict, List, Optional
 from pathlib import Path
 
 # This unused import makes the yaml library faster. don't remove
-import pylibyaml  # pyright: ignore
+import pylibyaml  # noqa: F401
 import yaml
 
-from . import log, options, vram_classes
+from . import options, vram_classes
 
 
 def _merge_configs(main_config, additional_config, additional_config_path):
@@ -26,15 +26,15 @@ def _merge_configs(main_config, additional_config, additional_config_path):
     for curkey in additional_config:
         if curkey not in main_config:
             main_config[curkey] = additional_config[curkey]
-        elif type(main_config[curkey]) != type(additional_config[curkey]):
+        elif type(main_config[curkey]) is not type(additional_config[curkey]):
             raise TypeError(
                 f"Could not merge {str(additional_config_path)}: type for key '{curkey}' in configs does not match"
             )
         else:
             # keys exist and match, see if a list to append
-            if type(main_config[curkey]) == list:
+            if type(main_config[curkey]) is list:
                 main_config[curkey] += additional_config[curkey]
-            elif type(main_config[curkey]) == dict:
+            elif type(main_config[curkey]) is dict:
                 # need to merge sub areas
                 main_config[curkey] = _merge_configs(
                     main_config[curkey],

splat/util/n64/rominfo.py

@@ -437,7 +437,7 @@ def get_info_bytes(rom_bytes: bytes, header_encoding: str) -> N64Rom:
 
     try:
         name = rom_bytes[0x20:0x34].decode(header_encoding).rstrip(" \0") or "empty"
-    except:
+    except UnicodeError:
         sys.exit(
             "splat could not decode the game name;"
             " try using a different encoding by passing the --header-encoding argument"

splat/util/ps2/__init__.py

@@ -0,0 +1 @@
+from . import ps2elfinfo as ps2elfinfo

splat/util/ps2/ps2elfinfo.py

@@ -0,0 +1,249 @@
+#! /usr/bin/env python3
+
+from __future__ import annotations
+
+import dataclasses
+from pathlib import Path
+import spimdisasm
+from spimdisasm.elf32 import (
+    Elf32File,
+    Elf32Constants,
+    Elf32SectionHeaderFlag,
+    Elf32ObjectFileType,
+)
+from typing import Optional
+
+from .. import log
+
+
+ELF_SECTION_MAPPING: dict[str, str] = {
+    ".text": "asm",
+    ".data": "data",
+    ".rodata": "rodata",
+    ".bss": "bss",
+    ".sdata": "sdata",
+    ".sbss": "sbss",
+    ".gcc_except_table": "gcc_except_table",
+    ".lit4": "lit4",
+    ".lit8": "lit8",
+    ".ctor": "ctor",
+    ".vtables": "vtables",
+    ".vutext": "textbin",  # No "proper" support yet
+    ".vudata": "databin",  # No "proper" support yet
+}
+
+# Section to not put into the elf_section_names list, because splat doesn't
+# know have support for them yet.
+ELF_SECTIONS_IGNORE: set[str] = {
+    ".vutext",
+    ".vudata",
+    ".vubss",
+}
+
+ELF_SMALL_SECTIONS: set[str] = {
+    ".lit4",
+    ".lit8",
+    ".sdata",
+    ".srdata",
+    ".sbss",
+}
+
+
+@dataclasses.dataclass
+class Ps2Elf:
+    entrypoint: int
+    segs: list[FakeSegment]
+    size: int
+    compiler: str
+    elf_section_names: list[tuple[str, bool]]
+    gp: Optional[int]
+    ld_gp_expression: Optional[str]
+
+    @staticmethod
+    def get_info(elf_path: Path, elf_bytes: bytes) -> Optional[Ps2Elf]:
+        # Avoid spimdisasm from complaining about unknown sections.
+        spimdisasm.common.GlobalConfig.QUIET = True
+
+        elf = Elf32File(elf_bytes)
+        if elf.header.type != Elf32ObjectFileType.EXEC.value:
+            log.write("Elf file is not an EXEC type.", status="warn")
+            return None
+        if elf.header.machine != 8:
+            # 8 corresponds to EM_MIPS
+            # We only care about mips binaries.
+            log.write("Elf file is not a MIPS binary.", status="warn")
+            return None
+        if Elf32Constants.Elf32HeaderFlag._5900 not in elf.elfFlags:
+            log.write("Missing 5900 flag", status="warn")
+            return None
+
+        if elf.reginfo is not None and elf.reginfo.gpValue != 0:
+            gp = elf.reginfo.gpValue
+        else:
+            gp = None
+        first_small_section_info: Optional[tuple[str, int]] = None
+
+        first_segment_name = "cod"
+        segs = [FakeSegment(first_segment_name, 0, 0, [])]
+
+        # TODO: check `.comment` section for any compiler info
+        compiler = "EEGCC"
+
+        elf_section_names: list[tuple[str, bool]] = []
+
+        first_offset: Optional[int] = None
+        rom_size = 0
+
+        previous_type = Elf32Constants.Elf32SectionHeaderType.PROGBITS
+        do_new_segs = False
+
+        # Loop over all the sections.
+        # Treat every normal elf section as part as the "main" segment.
+        # If we see a PROGBITS after a NOBITS then we shift the behavior into
+        # putting every new elf section into their own segment, this can happen
+        # when the elf contains a few special sections like `.mfifo`.
+        section_headers = sorted(elf.sectionHeaders, key=lambda x: x.offset)
+        for section in section_headers:
+            if section.size == 0:
+                # Skip over empty sections
+                continue
+
+            name = elf.shstrtab[section.name]
+            if name == ".mwcats":
+                compiler = "MWCCPS2"
+                continue
+
+            flags, _unknown_flags = Elf32SectionHeaderFlag.parseFlags(section.flags)
+            # if _unknown_flags != 0:
+            #     print(name, f"0x{_unknown_flags:08X}")
+            if Elf32SectionHeaderFlag.ALLOC not in flags:
+                # We don't care about non-alloc sections
+                continue
+
+            # We want PROGBITS (actual data) and NOBITS (bss and similar)
+            typ = Elf32Constants.Elf32SectionHeaderType.fromValue(section.type)
+            is_nobits = typ == Elf32Constants.Elf32SectionHeaderType.NOBITS
+            if typ == Elf32Constants.Elf32SectionHeaderType.PROGBITS:
+                if previous_type == Elf32Constants.Elf32SectionHeaderType.NOBITS:
+                    do_new_segs = True
+                pass
+            elif typ == Elf32Constants.Elf32SectionHeaderType.NOBITS:
+                pass
+            elif typ == Elf32Constants.Elf32SectionHeaderType.MIPS_REGINFO:
+                continue
+            else:
+                log.write(
+                    f"Unknown section type '{typ}' ({name}) found in the elf",
+                    status="warn",
+                )
+                return None
+
+            if first_offset is None:
+                first_offset = section.offset
+
+            if first_small_section_info is None and name in ELF_SMALL_SECTIONS:
+                first_small_section_info = (name, section.addr)
+
+            start = section.offset - first_offset
+            size = section.size
+
+            if do_new_segs:
+                segs.append(FakeSegment(name.lstrip("."), 0, start, []))
+
+            # Try to map this section to something splat can understand.
+            splat_segment_type = ELF_SECTION_MAPPING.get(name)
+            if splat_segment_type is None:
+                # Let's infer based on the section's flags
+                if is_nobits:
+                    splat_segment_type = "bss"
+                elif Elf32SectionHeaderFlag.EXECINSTR in flags:
+                    splat_segment_type = "asm"
+                elif Elf32SectionHeaderFlag.WRITE in flags:
+                    splat_segment_type = "data"
+                else:
+                    # Whatever...
+                    splat_segment_type = "rodata"
+
+            if name.startswith("."):
+                valid_for_splat = (
+                    name in ELF_SECTION_MAPPING
+                    and name not in ELF_SECTIONS_IGNORE
+                    and not do_new_segs
+                )
+                elf_section_names.append((name, valid_for_splat))
+
+            new_section = ElfSection(
+                name,
+                splat_segment_type,
+                section.addr,
+                start,
+                size,
+                is_nobits,
+            )
+            segs[-1].sections.append(new_section)
+            if is_nobits:
+                segs[-1].bss_size += size
+            else:
+                rom_size = start + size
+
+            previous_type = typ
+
+        # There are some games where they just squashed most sections into a
+        # single one, making the elf_section_names list pretty useless.
+        # We try to detect this and provide a default list if that's the case,
+        # hoping for the best.
+        if len(elf_section_names) < 4:
+            elf_section_names = [
+                (".text", True),
+                (".vutext", False),
+                (".data", True),
+                (".vudata", False),
+                (".rodata", True),
+                (".gcc_except_table", True),
+                (".lit8", True),
+                (".lit4", True),
+                (".sdata", True),
+                (".sbss", True),
+                (".bss", True),
+                (".vubss", False),
+            ]
+
+        # Fixup vram address of segments
+        for seg in segs:
+            seg.vram = seg.sections[0].vram
+
+        ld_gp_expression = None
+        if gp is not None and first_small_section_info is not None:
+            section_name, section_address = first_small_section_info
+            if gp > section_address:
+                diff = gp - section_address
+                ld_gp_expression = f"{first_segment_name}_{section_name.strip('.').upper()}_START + 0x{diff:04X}"
+
+        return Ps2Elf(
+            elf.header.entry,
+            segs,
+            rom_size,
+            compiler,
+            elf_section_names,
+            gp,
+            ld_gp_expression,
+        )
+
+
+@dataclasses.dataclass
+class FakeSegment:
+    name: str
+    vram: int
+    start: int
+    sections: list[ElfSection]
+    bss_size: int = 0
+
+
+@dataclasses.dataclass
+class ElfSection:
+    name: str
+    splat_segment_type: str
+    vram: int
+    start: int
+    size: int
+    is_nobits: bool

splat/util/psx/psxexeinfo.py

@@ -12,7 +12,6 @@ import dataclasses
 from pathlib import Path
 
 import rabbitizer
-import spimdisasm
 
 # PSX EXE has the following layout
 # header   ; 0x80 bytes
@@ -215,11 +214,11 @@ def main():
 
     print(f"Entrypoint: 0x{exe.entrypoint:08X}")
 
-    print(f"Initial GP: ", end="")
+    print("Initial GP: ", end="")
     if exe.initial_gp != 0:
         print(f"0x{exe.initial_gp:08X}")
     else:
-        print(f"No")
+        print("No")
 
     print()
     print(f"Destination VRAM: 0x{exe.destination_vram:08X}")

splat/util/relocs.py

@@ -94,13 +94,13 @@ def initialize():
 
             if rom_addr is None:
                 log.parsing_error_preamble(path, line_num, line)
-                log.error(f"Missing required 'rom' attribute for reloc")
+                log.error("Missing required 'rom' attribute for reloc")
             if reloc_type is None:
                 log.parsing_error_preamble(path, line_num, line)
-                log.error(f"Missing required 'reloc' attribute for reloc")
+                log.error("Missing required 'reloc' attribute for reloc")
             if symbol_name is None:
                 log.parsing_error_preamble(path, line_num, line)
-                log.error(f"Missing required 'symbol' attribute for reloc")
+                log.error("Missing required 'symbol' attribute for reloc")
 
             reloc = Reloc(rom_addr, reloc_type, symbol_name)
             if addend is not None:

splat/util/symbols.py

@@ -226,7 +226,9 @@ def handle_sym_addrs(
                         tf_val = (
                             True
                             if is_truey(attr_val)
-                            else False if is_falsey(attr_val) else None
+                            else False
+                            if is_falsey(attr_val)
+                            else None
                         )
                         if tf_val is None:
                             log.parsing_error_preamble(path, line_num, line)
@@ -432,9 +434,9 @@ def initialize_spim_context(all_segments: "List[Segment]") -> None:
         overlaps_found = False
         # Check the vram range of the global segment does not overlap with any overlay segment
         for ovl_segment in overlay_segments:
-            assert (
-                ovl_segment.vramStart <= ovl_segment.vramEnd
-            ), f"{ovl_segment.vramStart:08X} {ovl_segment.vramEnd:08X}"
+            assert ovl_segment.vramStart <= ovl_segment.vramEnd, (
+                f"{ovl_segment.vramStart:08X} {ovl_segment.vramEnd:08X}"
+            )
             if (
                 ovl_segment.vramEnd > global_vram_start
                 and global_vram_end > ovl_segment.vramStart
@@ -446,20 +448,20 @@ def initialize_spim_context(all_segments: "List[Segment]") -> None:
                 overlaps_found = True
         if overlaps_found:
             log.write(
-                f"Many overlaps between non-global and global segments were found.",
+                "Many overlaps between non-global and global segments were found.",
             )
             log.write(
-                f"This is usually caused by missing `exclusive_ram_id` tags on segments that have a higher vram address than other `exclusive_ram_id`-tagged segments"
+                "This is usually caused by missing `exclusive_ram_id` tags on segments that have a higher vram address than other `exclusive_ram_id`-tagged segments"
             )
             if len(global_segments_after_overlays) > 0:
                 log.write(
-                    f"These segments are the main suspects for missing a `exclusive_ram_id` tag:",
+                    "These segments are the main suspects for missing a `exclusive_ram_id` tag:",
                     status="warn",
                 )
                 for seg in global_segments_after_overlays:
                     log.write(f"    '{seg.name}', rom: 0x{seg.rom_start:06X}")
             else:
-                log.write(f"No suspected segments??", status="warn")
+                log.write("No suspected segments??", status="warn")
             log.error("Stopping due to the above errors")
 
     # pass the global symbols to spimdisasm
@@ -598,8 +600,13 @@ def add_symbol_to_spim_section(
     return context_sym
 
 
+# force_in_segment=True when the symbol belongs to this specific segment.
+# force_in_segment=False when this symbol is just a reference.
 def create_symbol_from_spim_symbol(
-    segment: "Segment", context_sym: spimdisasm.common.ContextSymbol
+    segment: "Segment",
+    context_sym: spimdisasm.common.ContextSymbol,
+    *,
+    force_in_segment: bool,
 ) -> "Symbol":
     in_segment = False
 
@@ -629,7 +636,7 @@ def create_symbol_from_spim_symbol(
                 in_segment = segment.contains_vram(context_sym.vram)
 
     sym = segment.create_symbol(
-        context_sym.vram, in_segment, type=sym_type, reference=True
+        context_sym.vram, force_in_segment or in_segment, type=sym_type, reference=True
     )
 
     if sym.given_name is None and context_sym.name is not None:

splat/util/utils.py

@@ -3,7 +3,7 @@ from typing import List, Optional, TypeVar
 T = TypeVar("T")
 
 
-def list_index(l: List[T], value: T) -> Optional[int]:
-    if value not in l:
+def list_index(the_list: List[T], value: T) -> Optional[int]:
+    if value not in the_list:
         return None
-    return l.index(value)
+    return the_list.index(value)

{splat64-0.36.2.dist-info → splat64-0.36.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: splat64
-Version: 0.36.2
+Version: 0.36.4
 Summary: A binary splitting tool to assist with decompilation and modding projects
 Project-URL: Repository, https://github.com/ethteck/splat
 Project-URL: Issues, https://github.com/ethteck/splat/issues
@@ -30,27 +30,27 @@ License-File: LICENSE
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3
 Requires-Python: >=3.9
-Requires-Dist: colorama
-Requires-Dist: intervaltree
-Requires-Dist: pylibyaml
-Requires-Dist: pyyaml
-Requires-Dist: tqdm
+Requires-Dist: colorama==0.4.6
+Requires-Dist: intervaltree==3.1.0
+Requires-Dist: pylibyaml==0.1.0
+Requires-Dist: pyyaml==6.0.3
+Requires-Dist: tqdm==4.67.1
 Provides-Extra: dev
-Requires-Dist: black; extra == 'dev'
 Requires-Dist: crunch64<1.0.0,>=0.5.1; extra == 'dev'
 Requires-Dist: mypy; extra == 'dev'
 Requires-Dist: n64img>=0.3.3; extra == 'dev'
-Requires-Dist: pygfxd; extra == 'dev'
+Requires-Dist: pygfxd>=1.0.5; extra == 'dev'
 Requires-Dist: rabbitizer<2.0.0,>=1.12.0; extra == 'dev'
-Requires-Dist: spimdisasm<2.0.0,>=1.36.1; extra == 'dev'
+Requires-Dist: ruff; extra == 'dev'
+Requires-Dist: spimdisasm<2.0.0,>=1.38.0; extra == 'dev'
 Requires-Dist: types-colorama; extra == 'dev'
 Requires-Dist: types-pyyaml; extra == 'dev'
 Provides-Extra: mips
 Requires-Dist: crunch64<1.0.0,>=0.5.1; extra == 'mips'
 Requires-Dist: n64img>=0.3.3; extra == 'mips'
-Requires-Dist: pygfxd; extra == 'mips'
+Requires-Dist: pygfxd>=1.0.5; extra == 'mips'
 Requires-Dist: rabbitizer<2.0.0,>=1.12.0; extra == 'mips'
-Requires-Dist: spimdisasm<2.0.0,>=1.36.1; extra == 'mips'
+Requires-Dist: spimdisasm<2.0.0,>=1.38.0; extra == 'mips'
 Description-Content-Type: text/markdown
 
 # splat
@@ -76,7 +76,7 @@ The brackets corresponds to the optional dependencies to install while installin
 If you use a `requirements.txt` file in your repository, then you can add this library with the following line:
 
 ```txt
-splat64[mips]>=0.36.2,<1.0.0
+splat64[mips]>=0.36.4,<1.0.0
 ```
 
 ### Optional dependencies