mapfile-parser 2.9.2__tar.gz → 2.9.4__tar.gz

{mapfile_parser-2.9.2 → mapfile_parser-2.9.4}/CHANGELOG.md

@@ -7,6 +7,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [2.9.4] - 2025-06-02
+
+### Changed
+
+- Implement parsing for `mwld` 2.7+ generated mapfiles.
+- Handle `*fill*`s in `mwld`.
+
+## [2.9.3] - 2025-06-01
+
+### Fixed
+
+- Try to infer the rom address of sections and segments even when the mapfile
+  does not have it explictly on GNU mapfiles.
+
 ## [2.9.2] - 2025-05-28
 
 ### Changed
@@ -606,7 +620,9 @@ Full changes: <https://github.com/Decompollaborate/mapfile_parser/compare/702a73
 - Initial release
 
 [unreleased]: https://github.com/Decompollaborate/mapfile_parser/compare/master...develop
-[2.9.2]: https://github.com/Decompollaborate/mapfile_parser/compare/2.9.0...2.9.2
+[2.9.4]: https://github.com/Decompollaborate/mapfile_parser/compare/2.9.3...2.9.4
+[2.9.3]: https://github.com/Decompollaborate/mapfile_parser/compare/2.9.2...2.9.3
+[2.9.2]: https://github.com/Decompollaborate/mapfile_parser/compare/2.9.1...2.9.2
 [2.9.1]: https://github.com/Decompollaborate/mapfile_parser/compare/2.9.0...2.9.1
 [2.9.0]: https://github.com/Decompollaborate/mapfile_parser/compare/2.8.1...2.9.0
 [2.8.1]: https://github.com/Decompollaborate/mapfile_parser/compare/2.8.0...2.8.1

{mapfile_parser-2.9.2 → mapfile_parser-2.9.4}/Cargo.lock

@@ -171,7 +171,7 @@ checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94"
 
 [[package]]
 name = "mapfile_parser"
-version = "2.9.2"
+version = "2.9.4"
 dependencies = [
  "lazy_static",
  "objdiff-core",

{mapfile_parser-2.9.2 → mapfile_parser-2.9.4}/Cargo.toml

@@ -3,7 +3,7 @@
 
 [package]
 name = "mapfile_parser"
-version = "2.9.2"
+version = "2.9.4"
 edition = "2021"
 rust-version = "1.74.0"
 authors = ["Anghelo Carvajal <angheloalf95@gmail.com>"]

{mapfile_parser-2.9.2 → mapfile_parser-2.9.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mapfile_parser
-Version: 2.9.2
+Version: 2.9.4
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy
@@ -55,7 +55,7 @@ If you use a `requirements.txt` file in your repository, then you can add
 this library with the following line:
 
 ```txt
-mapfile_parser>=2.9.2,<3.0.0
+mapfile_parser>=2.9.4,<3.0.0
 ```
 
 #### Development version
@@ -94,7 +94,7 @@ cargo add mapfile_parser
 Or add the following line manually to your `Cargo.toml` file:
 
 ```toml
-mapfile_parser = "2.9.2"
+mapfile_parser = "2.9.4"
 ```
 
 ## Versioning and changelog

{mapfile_parser-2.9.2 → mapfile_parser-2.9.4}/README.md

@@ -36,7 +36,7 @@ If you use a `requirements.txt` file in your repository, then you can add
 this library with the following line:
 
 ```txt
-mapfile_parser>=2.9.2,<3.0.0
+mapfile_parser>=2.9.4,<3.0.0
 ```
 
 #### Development version
@@ -75,7 +75,7 @@ cargo add mapfile_parser
 Or add the following line manually to your `Cargo.toml` file:
 
 ```toml
-mapfile_parser = "2.9.2"
+mapfile_parser = "2.9.4"
 ```
 
 ## Versioning and changelog

{mapfile_parser-2.9.2 → mapfile_parser-2.9.4}/pyproject.toml

@@ -3,7 +3,7 @@
 
 [project]
 name = "mapfile_parser"
-version = "2.9.2"
+version = "2.9.4"
 description = "Map file parser library focusing decompilation projects"
 readme = "README.md"
 requires-python = ">=3.9"

{mapfile_parser-2.9.2 → mapfile_parser-2.9.4}/src/mapfile_parser/__init__.py

@@ -5,7 +5,7 @@
 
 from __future__ import annotations
 
-__version_info__ = (2, 9, 2)
+__version_info__ = (2, 9, 4)
 __version__ = ".".join(map(str, __version_info__)) # + "-dev0"
 __author__ = "Decompollaborate"
 

{mapfile_parser-2.9.2 → mapfile_parser-2.9.4}/src/rs/lib.rs

@@ -63,6 +63,6 @@ mod tests {
 
     #[test]
     fn w0_000_map() {
-        let _ = MapFile::new_from_map_file(&PathBuf::from("tests/maps/w0_000.map"));
+        let _ = MapFile::new_from_map_file(&PathBuf::from("tests/maps/gnuld/misc/w0_000.map"));
     }
 }

{mapfile_parser-2.9.2 → mapfile_parser-2.9.4}/src/rs/parser.rs

@@ -232,7 +232,7 @@ impl MapFile {
                                     filepath,
                                     vram,
                                     size,
-                                    section_type,
+                                    section_type.into(),
                                 ));
                         } else if let Some(section_alone_match) =
                             regex_section_alone_entry.captures(prev_line)
@@ -250,7 +250,7 @@ impl MapFile {
                                     filepath,
                                     vram,
                                     size,
-                                    section_type,
+                                    section_type.into(),
                                 ));
                         }
                     }
@@ -319,7 +319,7 @@ impl MapFile {
                             filepath,
                             vram,
                             size,
-                            &section_type,
+                            section_type,
                         ));
                 }
             }
@@ -335,6 +335,9 @@ impl MapFile {
     ) -> Vec<segment::Segment> {
         let mut segments_list = Vec::with_capacity(temp_segment_list.len());
 
+        // We need to keep a calculated rom in case the segment doesn't specify it explicitly
+        let mut current_calculated_section_rom = 0;
+
         for (i, segment) in temp_segment_list.into_iter().enumerate() {
             if i == 0 && segment.is_placeholder() {
                 // skip the dummy segment if it has no size, sections or symbols
@@ -347,7 +350,13 @@ impl MapFile {
 
             let mut new_segment = segment.clone_no_sectionlist();
 
-            let mut vrom_offset = segment.vrom;
+            let mut vrom_offset = if let Some(vrom) = segment.vrom {
+                current_calculated_section_rom = vrom;
+                vrom
+            } else {
+                new_segment.vrom = Some(current_calculated_section_rom);
+                current_calculated_section_rom
+            };
             for mut section in segment.sections_list.into_iter() {
                 if section.is_placeholder() {
                     // drop placeholders
@@ -359,12 +368,12 @@ impl MapFile {
                 let symbols_count = section.symbols.len();
                 let is_noload_section = section.is_noload_section();
 
-                if section.vrom.is_some() {
-                    vrom_offset = section.vrom;
+                if let Some(vrom) = section.vrom {
+                    vrom_offset = vrom;
                 }
 
                 if !is_noload_section {
-                    section.vrom = vrom_offset;
+                    section.vrom = Some(vrom_offset);
                 }
 
                 if symbols_count > 0 {
@@ -377,7 +386,7 @@ impl MapFile {
                     // by the difference in vram address between the first symbol and the vram
                     // of the section.
                     if let Some(first_sym) = section.symbols.first() {
-                        sym_vrom = sym_vrom.map(|x| x + first_sym.vram - section.vram);
+                        sym_vrom = sym_vrom + first_sym.vram - section.vram;
 
                         // Aditionally, if the first symbol is missing then calculation of the size
                         // for the last symbol would be wrong, since we subtract the accumulated
@@ -397,8 +406,8 @@ impl MapFile {
 
                         if !is_noload_section {
                             // Only set vrom of non bss variables
-                            sym.vrom = sym_vrom;
-                            sym_vrom = sym_vrom.map(|x| x + sym_size);
+                            sym.vrom = Some(sym_vrom);
+                            sym_vrom += sym_size;
                         }
                     }
 
@@ -407,7 +416,7 @@ impl MapFile {
                     let sym_size = section.size - acummulated_size;
                     sym.size = sym_size;
                     if !is_noload_section {
-                        sym.vrom = sym_vrom;
+                        sym.vrom = Some(sym_vrom);
                         //sym_vrom += sym_size;
                     }
 
@@ -415,7 +424,8 @@ impl MapFile {
                 }
 
                 if !is_noload_section {
-                    vrom_offset = vrom_offset.map(|x| x + section.size);
+                    vrom_offset += section.size;
+                    current_calculated_section_rom += section.size;
                 }
 
                 new_segment.sections_list.push(section);
@@ -487,8 +497,8 @@ impl MapFile {
                     }
 
                     let mut new_section =
-                        section::Section::new_fill(filepath, vram, size, &section_type);
-                    if !utils::is_noload_section(&section_type) {
+                        section::Section::new_fill(filepath, vram, size, section_type);
+                    if !utils::is_noload_section(&new_section.section_type) {
                         new_section.vrom = vrom;
                     }
                     current_segment.sections_list.push(new_section);
@@ -500,9 +510,13 @@ impl MapFile {
                     if size > 0 {
                         let current_segment = temp_segment_list.last_mut().unwrap();
 
-                        let mut new_section =
-                            section::Section::new_default(filepath, vram, size, section_type);
-                        if !utils::is_noload_section(section_type) {
+                        let mut new_section = section::Section::new_default(
+                            filepath,
+                            vram,
+                            size,
+                            section_type.into(),
+                        );
+                        if !utils::is_noload_section(&new_section.section_type) {
                             new_section.vrom = vrom;
                         }
                         new_section.align = Some(align);
@@ -598,34 +612,41 @@ impl MapFile {
         let map_data = preprocess_map_data_mw(map_contents);
 
         let memory_map = parse_memory_map_mw(map_data);
-
-        // Almost every line starts with this information, so instead of duplicating it we put them on one single regex
-        let regex_row_entry = Regex::new(r"^\s*(?P<starting>[0-9a-fA-F]+)\s+(?P<size>[0-9a-fA-F]+)\s+(?P<vram>[0-9a-fA-F]+)\s+(?P<align>[0-9a-fA-F]+)\s+(?P<subline>.+)").unwrap();
-
-        let regex_segment_entry = Regex::new(r"^(?P<name>.+) section layout$").unwrap();
-        let regex_label_entry =
-            Regex::new(r"^(?P<label>lbl_[0-9A-F]{8})\s+(?P<filename>.+?)\s*$").unwrap();
-        let regex_symbol_entry =
-            Regex::new(r"^\s*(?P<name>[^ ]+)\s+(?P<filename>.+?)\s*$").unwrap();
+        let regex_entries = MwRegexEntries::new(map_data);
 
         let mut temp_segment_list = vec![segment::Segment::new_placeholder()];
 
         // Use a bunch of characters that shouldn't be valid in any os as a marker that we haven't found a file yet.
-        let mut current_filename = "invalid file <>:\"/\\|?*".to_string();
+        let invalid_file_name = "invalid file <>:\"/\\|?*";
+        let mut current_filename = invalid_file_name.to_string();
 
         for line in map_data.lines() {
-            // Check for regex_row_entry since it is more likely to match
-            if let Some(row_entry_match) = regex_row_entry.captures(line) {
+            // Check for regex_entries.common_row first since it is more likely to match
+            if let (Some(row_entry_match), false) = (
+                regex_entries.common_row.captures(line),
+                temp_segment_list.is_empty(),
+            ) {
                 let starting = utils::parse_hex(&row_entry_match["starting"]);
                 let size = utils::parse_hex(&row_entry_match["size"]);
                 let vram = utils::parse_hex(&row_entry_match["vram"]);
                 let align = utils::parse_hex(&row_entry_match["align"]);
 
+                let rom = row_entry_match.name("rom").map_or_else(
+                    || {
+                        temp_segment_list
+                            .last()
+                            .unwrap()
+                            .vrom
+                            .map(|segment_rom| segment_rom + starting)
+                    },
+                    |x| Some(utils::parse_hex(x.as_str())),
+                );
+
                 let subline = &row_entry_match["subline"];
 
-                if regex_label_entry.is_match(subline) {
+                if regex_entries.label.is_match(subline) {
                     // pass
-                } else if let Some(symbol_entry_match) = regex_symbol_entry.captures(subline) {
+                } else if let Some(symbol_entry_match) = regex_entries.symbol.captures(subline) {
                     let filename = &symbol_entry_match["filename"];
 
                     if filename == current_filename {
@@ -642,7 +663,7 @@ impl MapFile {
                                 new_symbol.size = size;
                             }
                             if !current_section.is_noload_section() {
-                                new_symbol.vrom = current_segment.vrom.map(|x| x + starting)
+                                new_symbol.vrom = rom
                             }
                             if align > 0 {
                                 new_symbol.align = Some(align);
@@ -652,27 +673,62 @@ impl MapFile {
                         }
                     } else {
                         // New file!
-                        current_filename = filename.to_string();
-
                         if size > 0 {
                             let section_type = &symbol_entry_match["name"];
                             let filepath = PathBuf::from(filename);
 
                             let current_segment = temp_segment_list.last_mut().unwrap();
 
-                            let mut new_section =
-                                section::Section::new_default(filepath, vram, size, section_type);
-                            if !utils::is_noload_section(section_type) {
-                                new_section.vrom = current_segment.vrom.map(|x| x + starting)
+                            let mut new_section = section::Section::new_default(
+                                filepath,
+                                vram,
+                                size,
+                                section_type.into(),
+                            );
+                            if !utils::is_noload_section(&new_section.section_type) {
+                                new_section.vrom = rom
                             }
 
                             current_segment.sections_list.push(new_section);
+
+                            // I'm not sure how to treat these cases.
+                            // I guess we can treat them as files without symbols for now...
+                            current_filename = if filename == "Linker Generated Symbol File" {
+                                invalid_file_name.to_string()
+                            } else {
+                                filename.to_string()
+                            };
                         }
                     }
+                } else if regex_entries.fill.is_match(subline) {
+                    // Make a dummy section to handle pads
+                    let current_segment = temp_segment_list.last_mut().unwrap();
+
+                    let mut filepath = PathBuf::new();
+                    let mut section_type = "".to_owned();
+
+                    if let Some(prev_section) = current_segment.sections_list.last() {
+                        let mut name = prev_section.filepath.file_name().unwrap().to_owned();
+
+                        name.push("__fill__");
+                        filepath = prev_section.filepath.with_file_name(name);
+                        section_type.clone_from(&prev_section.section_type);
+                    }
+
+                    let mut new_section =
+                        section::Section::new_fill(filepath, vram, size, section_type);
+                    new_section.align = Some(align);
+                    if !utils::is_noload_section(&new_section.section_type) {
+                        new_section.vrom = rom;
+                    }
+                    current_segment.sections_list.push(new_section);
+
+                    // Don't count this as a valid file.
+                    current_filename = invalid_file_name.to_string();
                 } else {
-                    println!("{}", subline);
+                    // println!("'{}'", subline);
                 }
-            } else if let Some(segment_entry_match) = regex_segment_entry.captures(line) {
+            } else if let Some(segment_entry_match) = regex_entries.segment.captures(line) {
                 let name = &segment_entry_match["name"];
 
                 let new_segment = if let Some(segment_entry) = memory_map.get(name) {
@@ -687,6 +743,10 @@ impl MapFile {
                 };
 
                 temp_segment_list.push(new_segment);
+
+                // Reset the tracked filename state.
+                // This avoid carrying the filename from one segment to the other
+                current_filename = invalid_file_name.to_string();
             }
         }
 
@@ -694,7 +754,6 @@ impl MapFile {
     }
 
     fn post_process_segments_mw(temp_segment_list: Vec<segment::Segment>) -> Vec<segment::Segment> {
-        // TODO: actually implement
         let mut segments_list = Vec::with_capacity(temp_segment_list.len());
 
         for (i, segment) in temp_segment_list.into_iter().enumerate() {
@@ -783,6 +842,40 @@ fn parse_memory_map_mw(map_data: &str) -> HashMap<String, MwMemoryMapEntry> {
     memory_map
 }
 
+struct MwRegexEntries {
+    common_row: Regex,
+    segment: Regex,
+    label: Regex,
+    symbol: Regex,
+    fill: Regex,
+}
+
+impl MwRegexEntries {
+    fn new(map_data: &str) -> Self {
+        // Almost every line starts with this information, so instead of duplicating it we put them on one single regex
+        let common_row = if map_data.contains("address  Size   address  offset") {
+            // mwld 2.7+
+            Regex::new(r"^\s*(?P<starting>[0-9a-fA-F]+)\s+(?P<size>[0-9a-fA-F]+)\s+(?P<vram>[0-9a-fA-F]+)\s+(?P<rom>[0-9a-fA-F]+)\s+(?P<align>[0-9a-fA-F]+)\s+(?P<subline>.+)").unwrap()
+        } else {
+            // mwld 1.3.2-
+            Regex::new(r"^\s*(?P<starting>[0-9a-fA-F]+)\s+(?P<size>[0-9a-fA-F]+)\s+(?P<vram>[0-9a-fA-F]+)\s+(?P<align>[0-9a-fA-F]+)\s+(?P<subline>.+)").unwrap()
+        };
+
+        let segment = Regex::new(r"^(?P<name>.+) section layout$").unwrap();
+        let label = Regex::new(r"^(?P<label>lbl_[0-9A-F]{8})\s+(?P<filename>.+?)\s*$").unwrap();
+        let symbol = Regex::new(r"^\s*(?P<name>[^ ]+)\s+(?P<filename>.+?)\s*$").unwrap();
+        let fill = Regex::new(r"^\s*\*fill\*\s*$").unwrap();
+
+        Self {
+            common_row,
+            segment,
+            label,
+            symbol,
+            fill,
+        }
+    }
+}
+
 impl MapFile {
     fn preprocess_map_data_gnu(map_data: &str) -> &str {
         // Skip the stuff we don't care about

{mapfile_parser-2.9.2 → mapfile_parser-2.9.4}/src/rs/section.rs

@@ -42,7 +42,7 @@ impl Section {
         filepath: PathBuf,
         vram: u64,
         size: u64,
-        section_type: &str,
+        section_type: String,
         vrom: Option<u64>,
         align: Option<u64>,
     ) -> Self {
@@ -53,7 +53,7 @@ impl Section {
         filepath: PathBuf,
         vram: u64,
         size: u64,
-        section_type: &str,
+        section_type: String,
         vrom: Option<u64>,
         align: Option<u64>,
         is_fill: bool,
@@ -62,7 +62,7 @@ impl Section {
             filepath,
             vram,
             size,
-            section_type: section_type.into(),
+            section_type,
             vrom,
             align,
             is_fill,
@@ -303,52 +303,20 @@ impl Section {
 
 impl Section {
     pub(crate) fn new_default(
-        filepath: std::path::PathBuf,
+        filepath: PathBuf,
         vram: u64,
         size: u64,
-        section_type: &str,
+        section_type: String,
     ) -> Self {
-        Section {
-            filepath,
-            vram,
-            size,
-            section_type: section_type.into(),
-            vrom: None,
-            align: None,
-            is_fill: false,
-            symbols: Vec::new(),
-        }
+        Self::new_impl(filepath, vram, size, section_type, None, None, false)
     }
 
     pub(crate) fn new_placeholder() -> Self {
-        Self {
-            filepath: "".into(),
-            vram: 0,
-            size: 0,
-            section_type: "".into(),
-            vrom: None,
-            align: None,
-            is_fill: false,
-            symbols: Vec::new(),
-        }
+        Self::new_impl("".into(), 0, 0, "".into(), None, None, false)
     }
 
-    pub(crate) fn new_fill(
-        filepath: std::path::PathBuf,
-        vram: u64,
-        size: u64,
-        section_type: &str,
-    ) -> Self {
-        Self {
-            filepath,
-            vram,
-            size,
-            section_type: section_type.into(),
-            vrom: None,
-            align: None,
-            is_fill: true,
-            symbols: Vec::new(),
-        }
+    pub(crate) fn new_fill(filepath: PathBuf, vram: u64, size: u64, section_type: String) -> Self {
+        Self::new_impl(filepath, vram, size, section_type, None, None, true)
     }
 
     pub fn is_placeholder(&self) -> bool {
@@ -420,8 +388,6 @@ impl Hash for Section {
 pub(crate) mod python_bindings {
     use pyo3::{intern, prelude::*, IntoPyObjectExt};
 
-    use std::path::PathBuf;
-
     // Required to call the `.hash` and `.finish` methods, which are defined on traits.
     use std::hash::{Hash, Hasher};
 
@@ -439,7 +405,7 @@ pub(crate) mod python_bindings {
             filepath: PathBuf,
             vram: u64,
             size: u64,
-            section_type: &str,
+            section_type: String,
             vrom: Option<u64>,
             align: Option<u64>,
             is_fill: bool,

{mapfile_parser-2.9.2 → mapfile_parser-2.9.4}/src/rs/segment.rs

@@ -195,7 +195,7 @@ impl Segment {
             let vram = first_section.vram;
             let mut size = 0;
             let vrom = first_section.vrom;
-            let section_type = &first_section.section_type;
+            let section_type = first_section.section_type.clone();
             let align = first_section.align;
 
             let mut symbols = Vec::new();