pyjmap 1.1.0__py3-none-any.whl

pyjmap/jmap.py

@@ -0,0 +1,997 @@
+"""
+High-level implementation of Nintendo's homemade BCSV / JMap container format that provides methods to construct,
+analyze, serialize, deserialize and manipulate JMapInfo data. As the name BCSV suggests, the data is laid out in a
+table-like structure. Each column (JMapField) has a name and binary storage information that describes how individual
+entries (JMapEntry) are to be interpreted. All integral data types are interpreted as signed data due to the lack of
+information about a field's signedness. The reverse-engineered specifications of this format can be accessed on the
+Luma's Workshop wiki: https://luma.aurumsmods.com/wiki/BCSV_(File_format)
+
+
+Different games use different hash algorithms, string encodings and endianness:
+
+Game                            Hash Algorithm  Endianness  Encoding
+--------------------------------------------------------------------------
+Luigi's Mansion (GameCube)      Old             big         shift_jisx0213
+Luigi's Mansion (3DS)           Old             little      utf-8(?)
+Super Mario Sunshine            Old             big         shift_jisx0213
+Donkey Kong Jungle Beat         New (JGadget)   big         shift_jisx0213
+Super Mario Galaxy (Wii)        New (JGadget)   big         shift_jisx0213
+Super Mario Galaxy (Switch)     New (JGadget)   little      utf-8
+Super Mario Galaxy 2            New (JGadget)   big         shift_jisx0213
+"""
+
+__all__ = [
+    "JMapException", "calc_old_hash", "calc_jgadget_hash", "JMapHashTable", "SuperMarioGalaxyHashTable",
+    "JungleBeatHashTable", "SuperMarioSunshineHashTable", "LuigisMansionHashTable", "JMapFieldType", "JMapField",
+    "JMapEntry", "JMapInfo", "from_buffer", "pack_buffer", "from_file", "write_file", "from_csv", "dump_csv"
+]
+
+import csv
+import enum
+import os
+import struct
+import warnings
+
+
+# ----------------------------------------------------------------------------------------------------------------------
+# Exception for JMap-related actions
+# ----------------------------------------------------------------------------------------------------------------------
+class JMapException(Exception):
+    """
+    Signals that an error occurred during any JMap-related action.
+    """
+    pass
+
+
+# ----------------------------------------------------------------------------------------------------------------------
+# Hash lookup table implementations
+# ----------------------------------------------------------------------------------------------------------------------
+def calc_old_hash(field_name: str) -> int:
+    """
+    The old hash function that is used in Luigi's Mansion. The resulting hash is a 32-bit value. The field name is
+    expected to be an ASCII-string.
+
+    :param field_name: the field name to be hashed.
+    :returns: the 32-bit hash value.
+    """
+    field_hash = 0
+
+    for ch in field_name.encode("ascii"):
+        ch |= ~0xFF if ch & 0x80 else 0  # Signed char
+        field_hash = (((field_hash << 8) & 0xFFFFFFFF) + ch) % 33554393
+
+    return field_hash
+
+
+def calc_jgadget_hash(field_name: str) -> int:
+    """
+    The JGadget hash function that is used in the Super Mario Galaxy games and Donkey Kong Jungle Beat. The resulting
+    hash is a 32-bit value. The field name is expected to be an ASCII-string.
+
+    :param field_name: the field name to be hashed.
+    :returns: the 32-bit hash value.
+    """
+    field_hash = 0
+
+    for ch in field_name.encode("ascii"):
+        ch |= ~0xFF if ch & 0x80 else 0  # Signed char
+        field_hash = (field_hash * 31) + ch
+
+    return field_hash & 0xFFFFFFFF
+
+
+class JMapHashTable:
+    """
+    A hash lookup table implementation for known field names. This stores a string for a given hash. The actual hashing
+    algorithm differs between the games and needs to be specified first. A file that lists known field names can be used
+    to initialize the hash table.
+    """
+
+    def __init__(self, hash_func, lookup_file_path):
+        """
+        Constructs a new hash table using the specified hash function and list file of known field names. Each line
+        corresponds to a field that should be included. Lines that start with a '#' will be ignored and are treated as
+        comments. This raises a FileNotFoundError if the lookup file does not exist.
+
+        :param hash_func: the hashing algorithm to be used.
+        :param lookup_file_path: the path to the file containing the known field names.
+        :raises FileNotFoundError: when the lookup file cannot be found.
+        """
+        self._hash_func_ = hash_func
+        self._lookup_ = dict()
+
+        if os.path.exists(lookup_file_path):
+            for field in open(lookup_file_path, "r", encoding="utf-8").readlines():
+                # Comment line?
+                if field.startswith("#"):
+                    continue
+
+                field = field.strip("\r\n")
+                self._lookup_[self.calc(field)] = field
+        else:
+            raise FileNotFoundError(f"Lookup names file \"{lookup_file_path}\" cannot be found!")
+
+    def calc(self, field_name: str) -> int:
+        """
+        Calculates the hash value over a given field name. The resulting hash is a 32-bit value.
+
+        :param field_name: the field name to be hashed.
+        :returns: the 32-bit hash value.
+        """
+        return self._hash_func_(field_name)
+
+    def find(self, field_hash: int) -> str:
+        """
+        Attempts to retrieve a valid field name for the specified hash value. If the hash is not found in the lookup
+        table, it returns a hexadecimal representation of the hash. For example, if the hash value 0xDEADBEEF could not
+        be found, this returns "[DEADBEEF]".
+
+        :param field_hash: the hash to find the field name for.
+        :returns: the field name if it exists, otherwise a hexadecimal string representation.
+        """
+        if field_hash in self._lookup_:
+            return self._lookup_[field_hash]
+        else:
+            return f"[{field_hash:08X}]"
+
+    def add(self, field_name: str) -> int:
+        """
+        Calculates the hash value for the specified field name tries to add the hash-name pair to the hash lookup table
+        if it does not exist already. The hash will be returned as well.
+
+        :param field_name: the string to calculate the hash for.
+        :returns: the 32-bit hash value.
+        """
+        field_hash = self.calc(field_name)
+
+        if field_hash not in self._lookup_:
+            self._lookup_[field_hash] = field_name
+
+        return field_hash
+
+
+class SuperMarioGalaxyHashTable(JMapHashTable):
+    """A hash table implementation for Super Mario Galaxy 1/2."""
+
+    def __init__(self):
+        """Constructs a new hash lookup table using the known field names from Super Mario Galaxy 1/2."""
+        super().__init__(calc_jgadget_hash, os.path.join(os.path.dirname(__file__), "lookup_supermariogalaxy.txt"))
+
+
+class JungleBeatHashTable(JMapHashTable):
+    """A hash table implementation for Donkey Kong Jungle Beat."""
+
+    def __init__(self):
+        """Constructs a new hash lookup table using the known field names from Donkey Kong Jungle Beat."""
+        super().__init__(calc_jgadget_hash, os.path.join(os.path.dirname(__file__), "lookup_donkeykongjunglebeat.txt"))
+
+
+class SuperMarioSunshineHashTable(JMapHashTable):
+    """A hash table implementation for Super Mario Sunshine."""
+
+    def __init__(self):
+        """Constructs a new hash lookup table using the known field names from Super Mario Sunshine."""
+        super().__init__(calc_old_hash, os.path.join(os.path.dirname(__file__), "lookup_supermariosunshine.txt"))
+
+
+class LuigisMansionHashTable(JMapHashTable):
+    """A hash table implementation for Luigi's Mansion."""
+
+    def __init__(self):
+        """Constructs a new hash lookup table using the known field names from Luigi's Mansion."""
+        super().__init__(calc_old_hash, os.path.join(os.path.dirname(__file__), "lookup_luigismansion.txt"))
+
+
+# ----------------------------------------------------------------------------------------------------------------------
+# Proper field type declarations
+# ----------------------------------------------------------------------------------------------------------------------
+class JMapFieldType(enum.Enum):
+    """
+    Declarations of all data types that are supported by JMapInfo containers. Every type holds information about their
+    size in bytes, bitmask storage order, default value and data type.
+    """
+
+    def __new__(cls, *args, **kwargs):
+        idx = len(cls.__members__)
+        obj = object.__new__(cls)
+        obj._value_ = idx
+        return obj
+
+    def __init__(self, size: int, mask: int, order: int, data_type, defval):
+        self._size_ = size
+        self._mask_ = mask
+        self._order_ = order
+        self._data_type_ = data_type
+        self._default_ = defval
+
+    @property
+    def size(self) -> int:
+        """The data size in bytes."""
+        return self._size_
+
+    @property
+    def mask(self) -> int:
+        """The data mask."""
+        return self._mask_
+
+    @property
+    def order(self) -> int:
+        """The sorting order index."""
+        return self._order_
+
+    @property
+    def data_type(self):
+        """The data type."""
+        return self._data_type_
+
+    @property
+    def default(self):
+        """The default value."""
+        return self._default_
+
+    LONG          =  4, 0xFFFFFFFF,  2, int, 0
+    STRING        = 32, 0x00000000,  0, str, ""
+    FLOAT         =  4, 0xFFFFFFFF,  1, float, 0.0
+    UNSIGNED_LONG =  4, 0xFFFFFFFF,  3, int, 0
+    SHORT         =  2, 0x0000FFFF,  4, int, 0
+    CHAR          =  1, 0x000000FF,  5, int, 0
+    STRING_OFFSET =  4, 0xFFFFFFFF,  6, str, ""
+
+
+# ----------------------------------------------------------------------------------------------------------------------
+# JMap field and entry definitions
+# ----------------------------------------------------------------------------------------------------------------------
+class JMapField:
+    """
+    Represents the information for a field (or column) of a JMapInfo container. This specifies the name, hash, type,
+    mask, shift amount and offset for an individual field. Actual fields should be created by the JMapInfo instance
+    itself. All entries in a JMapInfo should contain data for every field in the container.
+    """
+
+    # Structures for parsing and packing
+    __STRUCT_BE__ = struct.Struct(">2IH2b")  # Big-endian
+    __STRUCT_LE__ = struct.Struct("<2IH2b")  # Little-endian
+
+    def __init__(self, jmap, field_hash: int = 0, field_type: JMapFieldType = None,
+                 mask: int = 0xFFFFFFFF, shift_amount: int = 0, offset: int = 0, defval=None):
+        """
+        Low-level constructor for a new JMapField. The given JMapInfo container will be assigned to the field. This
+        should not be called directly. Instead, use JMapInfo's methods for creating new fields.
+
+        :param jmap: the JMapInfo container that this field belongs to.
+        :param field_hash: the field's hash.
+        :param field_type: the data type of this field.
+        :param mask: the bitmask.
+        :param shift_amount: the amount of bits to be shifted.
+        :param offset: the data offset into an entry.
+        :param defval: the default value for the field.
+        """
+        self._jmap_ = jmap
+        self._hash_ = field_hash
+        self._type_ = field_type
+        self.mask = mask
+        self.shift = shift_amount
+        self._default_ = defval
+        self._offset_ = offset
+
+    @property
+    def jmap(self):
+        """The JMapInfo container that the field belongs to."""
+        return self._jmap_
+
+    @property
+    def name(self) -> str:
+        """The field's name."""
+        return self._jmap_.hash_table.find(self._hash_)
+
+    @property
+    def hash(self) -> int:
+        """The field's hash."""
+        return self._hash_
+
+    @property
+    def type(self) -> JMapFieldType:
+        """The field's data type."""
+        return self._type_
+
+    @property
+    def offset(self) -> int:
+        """The field's offset. -1 indicates that the container will calculate the offset automatically."""
+        return self._offset_ if self._jmap_.manual_offsets else -1
+
+    @offset.setter
+    def offset(self, val: int):
+        """Sets the field's offset. Should always be specified for containers with manually assigned field offsets."""
+        self._offset_ = val
+
+    @property
+    def default(self):
+        """The field's default value."""
+        return self._default_
+
+    def __repr__(self):
+        return self._jmap_.hash_table.find(self._hash_)
+
+    def _unpack_(self, data, off: int, is_big_endian: bool):
+        strct = self.__STRUCT_BE__ if is_big_endian else self.__STRUCT_LE__
+        self._hash_, self.mask, self._offset_, self.shift, raw_type = strct.unpack_from(data, off)
+
+        if raw_type < 0 or raw_type >= 7:  # Invalid or unknown type
+            raise JMapException(f"Invalid JMap field type with ID 0x{raw_type:02X} found!")
+
+        self._type_ = JMapFieldType(raw_type)
+        self._default_ = self._type_.default
+
+    def _pack_(self, data, off: int, is_big_endian: bool):
+        strct = self.__STRUCT_BE__ if is_big_endian else self.__STRUCT_LE__
+        strct.pack_into(data, off, self._hash_, self.mask, self._offset_, self.shift, self._type_.value)
+
+
+class JMapEntry:
+    """
+    An entry (or row) of a JMapInfo container that holds the actual data. Every entry should contain data for all fields
+    in a JMap container. The data is stored as hash-value pairs. Data can be accessed using the field hash or name.
+    """
+
+    def __init__(self, jmap):
+        """
+        Constructs a new JMapEntry for the given JMapInfo container. This should not be called outside JMapInfo itself.
+        Instead, use JMapInfo's methods for creating new entries.
+
+        :param jmap: The JMapInfo container that this field belongs to.
+        """
+        self._jmap_ = jmap
+        self._data_ = dict()
+
+    @property
+    def jmap(self):
+        """The JMapInfo container that the entry belongs to."""
+        return self._jmap_
+
+    def data(self):
+        """Returns a view of hash-value pairs."""
+        return self._data_.items()
+
+    def __repr__(self):
+        string = "{"
+        first = True
+
+        for field_hash, value in self._data_.items():
+            if not first:
+                string += ", "
+            else:
+                first = False
+            string += repr(self._jmap_.hash_table.find(field_hash)) + ": " + repr(value)
+
+        return string + "}"
+
+    def __len__(self):
+        return len(self._data_)
+
+    def __getitem__(self, field_key):
+        if isinstance(field_key, str):
+            field_hash = self._jmap_.hash_table.calc(field_key)
+
+            if field_hash not in self._data_:
+                raise KeyError(f"Entry does not contain the field \"{field_key}\"")
+            else:
+                return self._data_[field_hash]
+        elif isinstance(field_key, int):
+            if field_key not in self._data_:
+                raise KeyError(f"Entry does not contain the field [{field_key:08X}]")
+            else:
+                return self._data_[field_key]
+        else:
+            raise TypeError("Key must be a str or int!")
+
+    def __setitem__(self, field_key, value):
+        if isinstance(field_key, str):
+            field_hash = self._jmap_.hash_table.calc(field_key)
+
+            if field_hash not in self._data_:
+                raise KeyError(f"Entry does not contain the field \"{field_key}\"")
+            elif type(self._data_[field_hash]) != type(value):
+                raise TypeError(f"Wrong data type for field \"{field_key}\": Expected {str(type(self._data_[field_hash]))}, found {type(value)} instead.")
+            else:
+                self._data_[field_hash] = value
+        elif isinstance(field_key, int):
+            if field_key not in self._data_:
+                raise KeyError(f"Entry does not contain the field [{field_key:08X}]")
+            elif type(self._data_[field_key]) != type(value):
+                raise TypeError(f"Wrong data type for field [{field_key:08X}]: Expected {str(type(self._data_[field_key]))}, found {type(value)} instead.")
+            else:
+                self._data_[field_key] = value
+        else:
+            raise TypeError("Key must be a str or int!")
+
+    def __contains__(self, field_key):
+        if isinstance(field_key, str):
+            return self._jmap_.hash_table.calc(field_key) in self._data_
+        elif isinstance(field_key, int):
+            return field_key in self._data_
+        else:
+            raise TypeError("Key must be a str or int!")
+
+
+# ----------------------------------------------------------------------------------------------------------------------
+# JMapInfo implementation according to the BCSV / JMap format
+# ----------------------------------------------------------------------------------------------------------------------
+class JMapInfo:
+    """
+    The table-like JMap container that consists of individual fields and entries. Provides high-level access to entries.
+    A hash lookup table is used to retrieve proper names for fields.
+    """
+
+    # Structures for parsing and packing
+    __STRUCT_BE__ = struct.Struct(">4I")  # Big-endian
+    __STRUCT_LE__ = struct.Struct("<4I")  # Little-endian
+    __U16_BE__ = struct.Struct(">H")
+    __U16_LE__ = struct.Struct("<H")
+    __U32_BE__ = struct.Struct(">I")
+    __U32_LE__ = struct.Struct("<I")
+    __F32_BE__ = struct.Struct(">f")
+    __F32_LE__ = struct.Struct("<f")
+
+    def __init__(self, hash_table: JMapHashTable):
+        """
+        Constructs a new JMapInfo container with no fields or entries. The specified lookup hash table will be used to
+        retrieve proper names for hashes.
+
+        :param hash_table: the hash lookup table to be used.
+        """
+        self._fields_ = dict()          # Maps fields to hashes for quick access.
+        self._entries_ = list()         # List of actual entries.
+        self._hash_table_ = hash_table  # The lookup hash table that is used to retrieve proper field names.
+        self._entry_size_ = -1          # Size of a single entry.
+        self.manual_offsets = False     # Requires manually-specified field offsets. Necessary for PA collision data.
+
+    @property
+    def hash_table(self):
+        """
+        Returns the hash lookup table used by this container.
+
+        :return: the hash lookup table.
+        """
+        return self._hash_table_
+
+    @property
+    def fields(self) -> tuple:
+        """
+        Returns a tuple of all fields in this container.
+
+        :return: the tuple of all fields.
+        """
+        return tuple(self._fields_.values())
+
+    def __iter__(self):
+        return iter(self._entries_)
+
+    def __reversed__(self):
+        return reversed(self._entries_)
+
+    def __repr__(self):
+        return repr(self._entries_)
+
+    def __len__(self):
+        return len(self._entries_)
+
+    def __getitem__(self, key):
+        return self._entries_[key]
+
+    def __delitem__(self, key):
+        if isinstance(key, slice):
+            for i in reversed(range(*key.indices(len(self._entries_)))):
+                self._entries_[i]._jmap_ = None
+                del self._entries_[i]
+        else:
+            self._entries_[key]._jmap_ = None
+            del self._entries_[key]
+
+    def __contains__(self, field_key):
+        if isinstance(field_key, str):
+            return self._hash_table_.calc(field_key) in self._fields_
+        elif isinstance(field_key, int):
+            return field_key in self._fields_
+        else:
+            raise TypeError("Key must be a str or int!")
+
+    def get_field(self, field_key) -> JMapField:
+        """
+        Retrieves the field using the specified key (hash or name) and remove's the field's data from all entries.
+
+        :param field_key: the field's key (hash or name).
+        :return: the field that corresponds to the key.
+        """
+        if isinstance(field_key, str):
+            field_hash = self._hash_table_.calc(field_key)
+
+            if field_hash not in self._fields_:
+                raise KeyError(f"Entry does not contain the field \"{field_key}\"")
+            else:
+                return self._fields_[field_hash]
+        elif isinstance(field_key, int):
+            if field_key not in self._fields_:
+                raise KeyError(f"Entry does not contain the field [{field_key}]")
+            else:
+                return self._fields_[field_key]
+        else:
+            raise TypeError("Key must be a str or int!")
+
+    def create_field(self, field_name, field_type: JMapFieldType, defval, mask: int = -1, shift_amount: int = 0, offset: int = 0):
+        """
+        Creates a new field with the given name, type, mask and shift amount. This also sets the field's data to the
+        default value for every entry. If a field with the same hash already exists, a JMapException will be raised. The
+        field's offset should only be provided when the JMapInfo container uses manually-specified field offsets.
+
+        :param field_name: the new field's name.
+        :param field_type: the new field's data type.
+        :param defval: the new field's default value.
+        :param mask: the new field's bitmask. If negative, the field type's default mask will be used.
+        :param shift_amount: the new field's shift amount.
+        :param offset: the field's offset.
+        :raises JMapException: if a field with the same hash already exists.
+        """
+        if not isinstance(defval, field_type.data_type):
+            raise TypeError(f"Default value type {repr(type(defval))} is not expected type {repr(field_type.data_type)}!")
+
+        # Calculate hash and check if field with same hash already exists
+        field_hash = self._hash_table_.add(field_name)
+
+        if field_hash in self._fields_:
+            raise JMapException(f"Field \"{field_name}\" already exists!")
+
+        # Create the actual field
+        mask = field_type.mask if mask < 0 else mask
+        field = JMapField(self, field_hash, field_type, mask, shift_amount, offset, defval)
+        self._fields_[field_hash] = field
+
+        # Set default values for all entries
+        for entry in self._entries_:
+            entry._data_[field_hash] = field.default
+
+    def drop_field(self, field_key):
+        """
+        Drops the field with the specified key (hash or name) and remove's the field's data from all entries.
+
+        :param field_key: the field's key (hash or name).
+        """
+        def dropfield0(field_hash):
+            field = self._fields_[field_hash]
+            field._jmap_ = None  # Unlink
+            del self._fields_[field_hash]
+
+            for entry in self._entries_:
+                del entry._data_[field_hash]
+
+        if isinstance(field_key, str):
+            field_hash = self._hash_table_.calc(field_key)
+
+            if field_hash not in self._fields_:
+                raise KeyError(f"Field \"{field_key}\" does not exist!")
+            else:
+                dropfield0(field_hash)
+        elif isinstance(field_key, int):
+            if field_key not in self._fields_:
+                raise KeyError(f"Field [{field_key:08X}] does not exist!")
+            else:
+                dropfield0(field_key)
+        else:
+            raise TypeError("Key must be a str or int!")
+
+    def create_entry(self) -> JMapEntry:
+        """
+        Creates a new entry, populates it with default values for all fields and appends it at the end of the container.
+
+        :return: the newly created entry.
+        """
+        entry = JMapEntry(self)
+
+        for field in self._fields_.values():
+            entry._data_[field.hash] = field.default
+
+        self._entries_.append(entry)
+
+        return entry
+
+    def remove_entry(self, index: int):
+        """
+        Removes and unlinks the entry at the given index.
+
+        :param index: the entry's index.
+        """
+        entry = self._entries_[index]
+        entry._jmap_ = None
+        del self._entries_[index]
+
+    def clear_entries(self):
+        """
+        Removes and unlinks all entries from this container.
+        """
+        for entry in self._entries_:
+            entry._jmap_ = None
+
+        self._entries_.clear()
+
+    def sort_entries(self, key, reverse: bool = False):
+        """
+        Sorts the entries using the given sorting key.
+
+        :param key: the sorting key function.
+        :param reverse: reverse sorting order.
+        """
+        self._entries_.sort(key=key, reverse=reverse)
+
+    def copy(self):
+        clone = JMapInfo(self._hash_table_)
+        clone._entry_size_ = self._entry_size_
+
+        for field_hash, field in self._fields_.items():
+            clone_field = JMapField(clone, field_hash, field.type, field.mask, field.shift, field.default)
+            clone._fields_[field_hash] = clone_field
+
+        for entry in self._entries_:
+            clone_entry = JMapEntry(clone)
+            clone_entry._data_ = entry._data_.copy()
+            clone._entries_.append(clone_entry)
+
+        return clone
+
+    __copy__ = copy
+    __deepcopy__ = copy
+
+    def _unpack_(self, data, off: int, is_big_endian: bool, encoding: str):
+        # Unpack header and calculate string pool offset
+        strct = self.__STRUCT_BE__ if is_big_endian else self.__STRUCT_LE__
+        num_entries, num_fields, off_data, self._entry_size_ = strct.unpack_from(data, off)
+        off_strings = off + off_data + (num_entries * self._entry_size_)
+
+        # Unpack fields
+        off_tmp = off + 0x10
+
+        for i in range(num_fields):
+            field = JMapField(self)
+            field._unpack_(data, off_tmp, is_big_endian)
+            self._fields_[field.hash] = field
+            off_tmp += 0xC
+
+        # Unpack entries
+        off_tmp = off + off_data
+
+        strct_u16 = self.__U16_BE__ if is_big_endian else self.__U16_LE__
+        strct_u32 = self.__U32_BE__ if is_big_endian else self.__U32_LE__
+        strct_f32 = self.__F32_BE__ if is_big_endian else self.__F32_LE__
+
+        for i in range(num_entries):
+            entry = JMapEntry(self)
+
+            for field in self._fields_.values():
+                field_type = field.type
+                off_val = off_tmp + field._offset_
+                val = None
+
+                # Read long
+                if field_type == JMapFieldType.LONG or field_type == JMapFieldType.UNSIGNED_LONG:
+                    val = (strct_u32.unpack_from(data, off_val)[0] & field.mask) >> field.shift
+                    val |= ~0xFFFFFFFF if val & 0x80000000 else 0
+
+                # Read string
+                elif field_type == JMapFieldType.STRING:
+                    # Read 32 bytes maximum
+                    end = data.index(0x00, off_val, off_val + 32)
+                    end = 32 if end < 0 else end
+                    val = data[off_val:end].decode(encoding)
+
+                # Read float
+                elif field_type == JMapFieldType.FLOAT:
+                    val = strct_f32.unpack_from(data, off_val)[0]
+
+                # Read short
+                elif field_type == JMapFieldType.SHORT:
+                    val = (strct_u16.unpack_from(data, off_val)[0] & field.mask) >> field.shift
+                    val |= ~0xFFFF if val & 0x8000 else 0
+
+                # Read char
+                elif field_type == JMapFieldType.CHAR:
+                    val = (data[off_val] & field.mask) >> field.shift
+                    val |= ~0xFF if val & 0x80 else 0
+
+                # Read string at offset
+                elif field_type == JMapFieldType.STRING_OFFSET:
+                    off_val = off_strings + strct_u32.unpack_from(data, off_val)[0]
+                    end_str = off_val
+
+                    while data[end_str]:
+                        end_str += 1
+
+                    val = data[off_val:end_str].decode(encoding)
+
+                entry._data_[field.hash] = val
+
+            self._entries_.append(entry)
+            off_tmp += self._entry_size_
+
+    def makebin(self, is_big_endian: bool, encoding: str) -> bytearray:
+        """
+        Packs the container's contents according to the BCSV format and returns the resulting bytearray buffer.
+
+        :param is_big_endian: the endianness of the data.
+        :param encoding: the encoding for strings.
+        :return: the packed bytearray buffer.
+        """
+        # Prepare header information
+        num_entries = len(self._entries_)
+        num_fields = len(self._fields_)
+        off_data = 0x10 + num_fields * 0xC
+
+        # Calculate entry size and field offsets
+        len_data_entry = 0
+
+        if self.manual_offsets:
+            for field in self._fields_.values():
+                potential_len = field.offset + field.type.size
+
+                if potential_len > len_data_entry:
+                    len_data_entry = potential_len
+        else:
+            for field in sorted(self._fields_.values(), key=lambda k: k.type.order):
+                field._offset_ = len_data_entry
+                len_data_entry += field.type.size
+
+        # Align total entry size to 4 bytes
+        self._entry_size_ = len_data_entry + 3 & ~3
+
+        # Prepare output buffer and write header
+        buffer = bytearray(off_data + num_entries * self._entry_size_)
+        strct = self.__STRUCT_BE__ if is_big_endian else self.__STRUCT_LE__
+        strct.pack_into(buffer, 0, num_entries, num_fields, off_data, self._entry_size_)
+
+        # Pack fields
+        off_tmp = 0x10
+
+        for field in self._fields_.values():
+            field._pack_(buffer, off_tmp, is_big_endian)
+            off_tmp += 0xC
+
+        # Pack entries and prepare the string pool
+        off_strings = len(buffer)
+        string_offsets = dict()
+
+        strct_u16 = self.__U16_BE__ if is_big_endian else self.__U16_LE__
+        strct_u32 = self.__U32_BE__ if is_big_endian else self.__U32_LE__
+        strct_f32 = self.__F32_BE__ if is_big_endian else self.__F32_LE__
+
+        for entry in self._entries_:
+            for field in self._fields_.values():
+                field_type = field.type
+                off_val = off_tmp + field._offset_
+                val = entry._data_[field.hash]
+
+                # Pack long
+                if field_type == JMapFieldType.LONG or field_type == JMapFieldType.UNSIGNED_LONG:
+                    prev = strct_u32.unpack_from(buffer, off_val)[0] & ~field.mask
+                    val = ((val << field.shift) & field.mask) | prev
+                    strct_u32.pack_into(buffer, off_val, val)
+
+                # Pack string
+                elif field_type == JMapFieldType.STRING:
+                    enc_string = val.encode(encoding)
+                    if len(enc_string) >= 32:
+                        warnings.warn("String is too long to be embedded. String will be chopped to fit 32 bytes!")
+                    len_string = min(len(enc_string), 32)
+                    buffer[off_val:off_val + len_string] = enc_string[:len_string]
+
+                # Pack float
+                elif field_type == JMapFieldType.FLOAT:
+                    strct_f32.pack_into(buffer, off_val, val)
+
+                # Pack short
+                elif field_type == JMapFieldType.SHORT:
+                    prev = strct_u16.unpack_from(buffer, off_val)[0] & ~field.mask
+                    val = ((val << field.shift) & field.mask) | prev
+                    strct_u16.pack_into(buffer, off_val, val)
+
+                # Pack char
+                elif field_type == JMapFieldType.CHAR:
+                    buffer[off_val] = (val << field.shift) & field.mask
+
+                # Pack string at offset
+                elif field_type == JMapFieldType.STRING_OFFSET:
+                    if val in string_offsets:
+                        off_string = string_offsets[val]
+                    else:
+                        off_string = len(buffer) - off_strings
+                        string_offsets[val] = off_string
+
+                        # shift_jis appears to truncate the zero terminator sometimes...
+                        buffer += val.encode(encoding)
+                        buffer += "\0".encode(encoding)
+
+                    strct_u32.pack_into(buffer, off_val, off_string)
+
+            off_tmp += self._entry_size_
+
+        # Align buffer to 32 bytes
+        len_buf = len(buffer)
+        buffer += bytearray([0x40] * ((len_buf + 31 & ~31) - len_buf))
+
+        return buffer
+
+
+# ----------------------------------------------------------------------------------------------------------------------
+# Helper I/O functions
+# ----------------------------------------------------------------------------------------------------------------------
+def from_buffer(hashtable: JMapHashTable, buffer, offset: int, big_endian: bool = True, encoding: str = "shift_jisx0213") -> JMapInfo:
+    """
+    Creates and returns a new JMapInfo container by unpacking the content from the specified buffer. The data is
+    expected to be stored in the JMap / BCSV format.
+
+    :param hashtable: the hash lookup table to be used.
+    :param buffer: the byte buffer.
+    :param offset: the offset into the buffer.
+    :param big_endian: the endianness of the data.
+    :param encoding: the encoding for strings.
+    :return: the unpacked JMapInfo container.
+    """
+    jmap = JMapInfo(hashtable)
+    jmap._unpack_(buffer, offset, big_endian, encoding)
+    return jmap
+
+
+def pack_buffer(jmap: JMapInfo, big_endian: bool = True, encoding: str = "shift_jisx0213") -> bytearray:
+    """
+    Packs the given JMapInfo's contents according to the BCSV format and returns the resulting bytearray buffer.
+
+    :param jmap: the JMapInfo container.
+    :param big_endian: the endianness of the data.
+    :param encoding: the encoding for strings.
+    :return: the buffer containing the stored data.
+    """
+    return jmap.makebin(big_endian, encoding)
+
+
+def from_file(hashtable: JMapHashTable, file_path: str, big_endian: bool = True, encoding: str = "shift_jisx0213") -> JMapInfo:
+    """
+    Creates and returns a new JMapInfo container by unpacking the contents from the given file path. The data is
+    expected to be stored in the JMap / BCSV format.
+
+    :param hashtable: the hash lookup table to be used.
+    :param file_path: the file path to the JMap / BCSV file.
+    :param big_endian: the endianness of the data.
+    :param encoding: the encoding for strings.
+    :return: the unpacked JMapInfo container.
+    """
+    jmap = JMapInfo(hashtable)
+    with open(file_path, "rb") as f:
+        jmap._unpack_(f.read(), 0, big_endian, encoding)
+    return jmap
+
+
+def write_file(jmap: JMapInfo, file_path: str, big_endian: bool = True, encoding: str = "shift_jisx0213"):
+    """
+    Packs the given JMapInfo's contents according to the BCSV format and writes the resulting buffer's contents to the
+    specified file.
+
+    :param jmap: the JMapInfo container.
+    :param file_path: the file path to write the contents to.
+    :param big_endian: the endianness of the data.
+    :param encoding: the encoding for strings.
+    """
+    buffer = jmap.makebin(big_endian, encoding)
+
+    with open(file_path, "wb") as f:
+        f.write(buffer)
+        f.flush()
+
+
+__CSV_FIELD_TYPES__ = ["Int", "EmbeddedString", "Float", "UnsignedInt", "Short", "Char", "String"]
+__CSV_FIELD_DEFAULTS__ = ["0", "0", "0.0", "0", "0", "0", "0"]
+__CSV_FIELD_PRIMARIES__ = [int, str, float, int, int, int, str]
+
+
+def from_csv(hashtable: JMapHashTable, file_path: str, encoding: str = "utf-8") -> JMapInfo:
+    """
+    Creates a new JMapInfo container using the raw CSV data found in the specified file. The CSV files have to be comma-
+    delimited and may use quote marks for quotes strings. The information of each field consists of three components
+    that are separated by double-colons. Example: GroupName:String:0. The components describe the field's name, type and
+    default value, respectively.
+
+    :param hashtable: the hash lookup table to be used.
+    :param file_path: the file path to the CSV file.
+    :param encoding: the CSV file's encoding, expects utf-8 by default.
+    :return: the created JMapInfo container.
+    """
+    jmap = JMapInfo(hashtable)
+
+    with open(file_path, "r", encoding=encoding, newline="") as f:
+        csvreader = csv.reader(f, delimiter=",", quotechar='"')
+
+        # Create fields
+        field_descs = next(csvreader, None)
+
+        if field_descs is None:
+            raise SyntaxError("CSV file is empty.")
+
+        for field_desc in field_descs:
+            # Get field descriptor information
+            field_desc = field_desc.split(":")
+
+            if len(field_desc) != 3:
+                raise SyntaxError("Number of field descriptor details is not 3!")
+
+            field_name, field_type, field_default = field_desc
+
+            if len(field_name) == 0:
+                raise SyntaxError("Field name cannot be empty!")
+
+            # Get proper JMapFieldType and default value from descriptor
+            if field_type == "Int":
+                actual_type = JMapFieldType.LONG
+                actual_default = int(field_default)
+            elif field_type == "EmbeddedString":
+                actual_type = JMapFieldType.STRING
+                actual_default = ""
+            elif field_type == "Float":
+                actual_type = JMapFieldType.FLOAT
+                actual_default = float(field_default)
+            elif field_type == "UnsignedInt":
+                actual_type = JMapFieldType.UNSIGNED_LONG
+                actual_default = int(field_default)
+            elif field_type == "Short":
+                actual_type = JMapFieldType.SHORT
+                actual_default = int(field_default)
+            elif field_type == "Char":
+                actual_type = JMapFieldType.CHAR
+                actual_default = int(field_default)
+            elif field_type == "String":
+                actual_type = JMapFieldType.STRING_OFFSET
+                actual_default = ""
+            else:
+                raise SyntaxError(f"Unknown CSV field type {field_type} for field {field_name}")
+
+            # Check if field name is a hash
+            if field_name[0] == "[" and field_name[-1] == "]":
+                field_hash = int(field_name[1:-1], 16)
+            else:
+                field_hash = hashtable.add(field_name)
+
+            field = JMapField(jmap, field_hash, actual_type, mask=actual_type.mask, defval=actual_default)
+            jmap._fields_[field.hash] = field
+
+        # Create entries
+        for entry_row in csvreader:
+            entry = JMapEntry(jmap)
+            jmap._entries_.append(entry)
+
+            for i, field in enumerate(jmap._fields_.values()):
+                if len(entry_row[i]) == 0:
+                    entry._data_[field.hash] = field.default
+                else:
+                    entry._data_[field.hash] = __CSV_FIELD_PRIMARIES__[field.type.value](entry_row[i])
+
+    return jmap
+
+
+def dump_csv(jmap: JMapInfo, file_path: str, encoding: str = "utf-8"):
+    """
+    Dumps the JMapInfo's data to the specified CSV file. The CSV file is comma-delimited and may use quote marks for
+    quoted strings.
+
+    :param jmap: the JMapInfo container.
+    :param file_path: the file path to the CSV file.
+    :param encoding: the CSV file's encoding, expects utf-8 by default.
+    """
+    with open(file_path, "w", encoding=encoding, newline="") as f:
+        csv_writer = csv.writer(f, delimiter=",", quotechar='"', quoting=csv.QUOTE_MINIMAL)
+
+        # Write fields header
+        field_descs = [
+            f"{field.name}:{__CSV_FIELD_TYPES__[field.type.value]}:{__CSV_FIELD_DEFAULTS__[field.type.value]}"
+            for field in jmap._fields_.values()
+        ]
+        csv_writer.writerow(field_descs)
+
+        # Write entries
+        for entry in jmap:
+            csv_writer.writerow([str(entry[field.hash]) for field in jmap._fields_.values()])
+
+        f.flush()