flatdata-py 0.4.10__tar.gz → 0.4.11__tar.gz

{flatdata_py-0.4.10 → flatdata_py-0.4.11}/PKG-INFO

@@ -1,13 +1,13 @@
 Metadata-Version: 2.4
 Name: flatdata-py
-Version: 0.4.10
+Version: 0.4.11
 Summary: Python 3 implementation of Flatdata
 Project-URL: Homepage, https://github.com/heremaps/flatdata
 Author: Flatdata Developers
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
-Requires-Dist: flatdata-generator==0.4.10
+Requires-Dist: flatdata-generator==0.4.11
 Requires-Dist: numpy
 Requires-Dist: pandas
 Provides-Extra: inspector
@@ -34,6 +34,37 @@ Once you have [created a flatdata schema file](../README.md#creating-a-schema),
 flatdata-generator --gen py --schema locations.flatdata --output-file locations.py
 ```
 
+## Performance tips
+
+`flatdata-py` supports two data access patterns with very different performance characteristics on large archives.
+
+Iterating over a vector yields one Python object per element. Each field access unpacks bits from the underlying memory-mapped data. This is fine for accessing individual elements or small ranges, but has significant per-element overhead for bulk operations:
+
+```python
+count = sum(1 for x in archive.links if x.speed_limit > 100)
+```
+
+For bulk operations, use the vectorized access methods that read fields directly into NumPy arrays:
+
+```python
+# single column access, returns a pandas DataFrame
+df = archive.links.speed_limit
+count = len(df[df['speed_limit'] > 100])
+
+# full NumPy structured array with all fields
+arr = archive.links.to_numpy()
+count = int(np.sum(arr['speed_limit'] > 100))
+
+# slices work too
+arr = archive.links[1000:2000].to_numpy()
+df = archive.links[::10].to_data_frame()
+```
+
+* Use `vector.field_name` (column access) when you only need one or a few fields.
+* Use `vector.to_numpy()` or `vector.to_data_frame()` when you need all fields at once.
+* Use `vector[i].field` for random access to individual elements.
+* The underlying data is memory-mapped; the OS pages it from disk on demand. Vectorized results are materialized as NumPy arrays in RAM.
+
 ## Using the inspector
 
 `flatdata-py` comes with a handy tool called the `flatdata-inspector` to inspect the contents of an archive:

{flatdata_py-0.4.10 → flatdata_py-0.4.11}/README.md

@@ -18,6 +18,37 @@ Once you have [created a flatdata schema file](../README.md#creating-a-schema),
 flatdata-generator --gen py --schema locations.flatdata --output-file locations.py
 ```
 
+## Performance tips
+
+`flatdata-py` supports two data access patterns with very different performance characteristics on large archives.
+
+Iterating over a vector yields one Python object per element. Each field access unpacks bits from the underlying memory-mapped data. This is fine for accessing individual elements or small ranges, but has significant per-element overhead for bulk operations:
+
+```python
+count = sum(1 for x in archive.links if x.speed_limit > 100)
+```
+
+For bulk operations, use the vectorized access methods that read fields directly into NumPy arrays:
+
+```python
+# single column access, returns a pandas DataFrame
+df = archive.links.speed_limit
+count = len(df[df['speed_limit'] > 100])
+
+# full NumPy structured array with all fields
+arr = archive.links.to_numpy()
+count = int(np.sum(arr['speed_limit'] > 100))
+
+# slices work too
+arr = archive.links[1000:2000].to_numpy()
+df = archive.links[::10].to_data_frame()
+```
+
+* Use `vector.field_name` (column access) when you only need one or a few fields.
+* Use `vector.to_numpy()` or `vector.to_data_frame()` when you need all fields at once.
+* Use `vector[i].field` for random access to individual elements.
+* The underlying data is memory-mapped; the OS pages it from disk on demand. Vectorized results are materialized as NumPy arrays in RAM.
+
 ## Using the inspector
 
 `flatdata-py` comes with a handy tool called the `flatdata-inspector` to inspect the contents of an archive:

{flatdata_py-0.4.10 → flatdata_py-0.4.11}/flatdata/lib/archive.py

@@ -39,9 +39,9 @@ class Archive:
             self.__getattr__(name)
 
     def __getattr__(self, name):
-        if name not in list(self._RESOURCES.keys()):
+        if name not in self._RESOURCES:
             raise AttributeError("Resource %s not defined in archive." % name)
-        if name not in list(self._loaded_resources.keys()):
+        if name not in self._loaded_resources:
             self._loaded_resources[name] = self._open_resource(name)
         return self._loaded_resources[name]
 

flatdata_py-0.4.11/flatdata/lib/data_access.py

@@ -0,0 +1,168 @@
+'''
+ Copyright (c) 2017 HERE Europe B.V.
+ See the LICENSE file in the root of this project for license details.
+'''
+
+import numpy as np
+
+# Sign bits cache for the value reading.
+_SIGN_BITS = [0] + [(1 << (bits - 1)) for bits in range(1, 65)]
+
+
+def make_field_reader(offset_bits, num_bits, is_signed):
+    """Build a specialized closure for reading a single field from a structure.
+
+    Returns a function reader(data, pos_bytes) that reads the field value
+    from ``data`` at byte position ``pos_bytes``.  All constants (byte offset,
+    bit shift, mask, sign handling) are pre-computed and captured by the
+    closure so the hot path does minimal work.
+    """
+    offset_bytes, offset_extra = divmod(offset_bits, 8)
+    total_bytes = (num_bits + 7) // 8
+    end_byte = offset_bytes + total_bytes
+    mask = (1 << num_bits) - 1
+    needs_extra = (total_bytes * 8 - offset_extra) < num_bits
+    extra_shift = total_bytes * 8 - offset_extra
+
+    if num_bits == 1:
+        bit_mask = 1 << offset_extra
+        def reader(data, pos):
+            return int((data[pos + offset_bytes] & bit_mask) != 0)
+        return reader
+
+    if is_signed:
+        sign_bit = _SIGN_BITS[num_bits]
+        sign_mask = sign_bit - 1
+        if needs_extra:
+            def reader(data, pos):
+                result = int.from_bytes(
+                    data[pos + offset_bytes: pos + end_byte], byteorder="little")
+                result >>= offset_extra
+                result |= data[pos + end_byte] << extra_shift
+                result &= mask
+                return (result & sign_mask) - (result & sign_bit)
+        elif offset_extra:
+            def reader(data, pos):
+                result = (int.from_bytes(
+                    data[pos + offset_bytes: pos + end_byte],
+                    byteorder="little") >> offset_extra) & mask
+                return (result & sign_mask) - (result & sign_bit)
+        else:
+            def reader(data, pos):
+                result = int.from_bytes(
+                    data[pos + offset_bytes: pos + end_byte],
+                    byteorder="little") & mask
+                return (result & sign_mask) - (result & sign_bit)
+        return reader
+
+    # Unsigned paths
+    if needs_extra:
+        def reader(data, pos):
+            result = int.from_bytes(
+                data[pos + offset_bytes: pos + end_byte], byteorder="little")
+            result >>= offset_extra
+            result |= data[pos + end_byte] << extra_shift
+            return result & mask
+    elif offset_extra:
+        def reader(data, pos):
+            return (int.from_bytes(
+                data[pos + offset_bytes: pos + end_byte],
+                byteorder="little") >> offset_extra) & mask
+    else:
+        def reader(data, pos):
+            return int.from_bytes(
+                data[pos + offset_bytes: pos + end_byte],
+                byteorder="little") & mask
+    return reader
+
+
+def read_field_vectorized(raw_bytes_2d, field_offset_bits, field_width_bits, is_signed):
+    """Read a bit-packed field from all elements at once, returning a numpy array.
+
+    :param raw_bytes_2d: numpy uint8 array shaped (num_elements, struct_size_bytes)
+    :param field_offset_bits: bit offset of the field within each element
+    :param field_width_bits: width of the field in bits (max 64)
+    :param is_signed: whether to sign-extend the result
+    :return: numpy array of field values
+    """
+    if field_width_bits == 1:
+        byte_idx = field_offset_bits // 8
+        bit_idx = field_offset_bits % 8
+        return ((raw_bytes_2d[:, byte_idx].astype(np.uint64) >> np.uint64(bit_idx)) &
+                np.uint64(1))
+
+    byte_start = field_offset_bits // 8
+    bit_shift = field_offset_bits % 8
+    bytes_needed = (bit_shift + field_width_bits + 7) // 8
+
+    # Use Python int arithmetic for the shift to avoid numpy overflow,
+    # then broadcast back to the array.
+    result = np.zeros(raw_bytes_2d.shape[0], dtype=np.uint64)
+    for b in range(min(bytes_needed, 8)):
+        result |= raw_bytes_2d[:, byte_start + b].astype(np.uint64) << np.uint64(b * 8)
+    result >>= np.uint64(bit_shift)
+
+    # If the field spans more than 8 bytes (unaligned 64-bit field), merge the extra byte.
+    bits_so_far = 8 * min(bytes_needed, 8) - bit_shift
+    if bits_so_far < field_width_bits and bytes_needed > 8:
+        extra = raw_bytes_2d[:, byte_start + 8].astype(np.uint64)
+        result |= extra << np.uint64(bits_so_far)
+
+    if field_width_bits < 64:
+        result &= np.uint64((1 << field_width_bits) - 1)
+
+    if is_signed:
+        if field_width_bits == 64:
+            return result.view(np.int64)
+        sign_bit = np.uint64(1 << (field_width_bits - 1))
+        offset = -(1 << field_width_bits)
+        signed = result.astype(np.int64) + np.int64(offset)
+        result = np.where(result & sign_bit, signed, result.astype(np.int64))
+
+    return result
+
+
+def read_value(data, offset_bits, num_bits, is_signed):
+    """Read a bit-packed value from data at the given bit offset.
+
+    This is a convenience wrapper around :func:`make_field_reader` for one-off
+    reads.  For repeated reads of the same field, prefer building a reader once
+    with ``make_field_reader`` and reusing it.
+    """
+    reader = make_field_reader(offset_bits, num_bits, is_signed)
+    return reader(data, 0)
+
+
+def write_value(data, offset_bits, num_bits, is_signed, value):
+    assert num_bits <= 64, f'Number of bits to write is greater than 64'
+
+    offset_bytes, offset_extra_bits = divmod(offset_bits, 8)
+    total_bytes = (num_bits + 7) // 8
+
+    if num_bits == 1:
+        if value == 1:
+            data[offset_bytes] |= 1 << offset_extra_bits
+        else:
+            data[offset_bytes] &= ~(1 << offset_extra_bits)
+        return
+
+    mask = (1 << num_bits) - 1
+    value <<= offset_extra_bits
+    value &= mask << offset_extra_bits
+    value_in_little_endian = value.to_bytes(total_bytes + 1, byteorder="little", signed=is_signed)
+    surrounding_bits = data[offset_bytes] & ((1 << offset_bits) - 1)
+
+    byte_idx = 0
+    data[offset_bytes] = value_in_little_endian[byte_idx]
+    data[offset_bytes] |= surrounding_bits
+
+    byte_idx += 1
+    while byte_idx < total_bytes:
+        data[offset_bytes + byte_idx] = value_in_little_endian[byte_idx]
+        byte_idx += 1
+
+    bits_written = total_bytes * 8 - offset_extra_bits
+    if bits_written < num_bits:
+        surrounding_bits = data[offset_bytes + byte_idx] & ~((1 << offset_bits) - 1)
+        data[offset_bytes + byte_idx] = value_in_little_endian[byte_idx] & ((1 << (8 - (bits_written % 8))) - 1)
+        data[offset_bytes + byte_idx] |= surrounding_bits

flatdata_py-0.4.11/flatdata/lib/data_access.py.orig

@@ -0,0 +1,204 @@
+'''
+ Copyright (c) 2017 HERE Europe B.V.
+ See the LICENSE file in the root of this project for license details.
+'''
+
+# Sign bits cache for the value reading.
+_SIGN_BITS = [0] + [(1 << (bits - 1)) for bits in range(1, 65)]
+
+
+def make_field_reader(offset_bits, num_bits, is_signed):
+    """Build a specialized closure for reading a single field from a structure.
+
+    Returns a function reader(data, pos_bytes) that reads the field value
+    from ``data`` at byte position ``pos_bytes``.  All constants (byte offset,
+    bit shift, mask, sign handling) are pre-computed and captured by the
+    closure so the hot path does minimal work.
+    """
+    offset_bytes, offset_extra = divmod(offset_bits, 8)
+    total_bytes = (num_bits + 7) // 8
+<<<<<<< HEAD
+
+    if num_bits == 1:
+        return int((data[offset_bytes] & (1 << offset_extra_bits)) != 0)
+
+    result = int.from_bytes(data[offset_bytes: offset_bytes + total_bytes], byteorder="little")
+    result >>= offset_extra_bits
+    if (total_bytes * 8 - offset_extra_bits) < num_bits:
+        remainder = data[offset_bytes + total_bytes]
+        result |= remainder << (total_bytes * 8 - offset_extra_bits)
+
+    if num_bits < 64:
+        result = result & ((1 << num_bits) - 1)
+
+    if not is_signed:
+        return result
+
+    return (result & (_SIGN_BITS[num_bits] - 1)) - (result & _SIGN_BITS[num_bits])
+
+
+def write_value(data, offset_bits, num_bits, is_signed, value):
+    assert num_bits <= 64, f'Number of bits to write is greater than 64'
+
+    offset_bytes, offset_extra_bits = divmod(offset_bits, 8)
+    total_bytes = (num_bits + 7) // 8
+
+    if num_bits == 1:
+        if value == 1:
+            data[offset_bytes] |= 1 << offset_extra_bits
+        else:
+            data[offset_bytes] &= ~(1 << offset_extra_bits)
+        return
+
+=======
+    end_byte = offset_bytes + total_bytes
+>>>>>>> e486615 (Also improve scalar readers by caching)
+    mask = (1 << num_bits) - 1
+    needs_extra = (total_bytes * 8 - offset_extra) < num_bits
+    extra_shift = total_bytes * 8 - offset_extra
+
+    if num_bits == 1:
+        bit_mask = 1 << offset_extra
+        def reader(data, pos):
+            return int((data[pos + offset_bytes] & bit_mask) != 0)
+        return reader
+
+    if is_signed:
+        sign_bit = _SIGN_BITS[num_bits]
+        sign_mask = sign_bit - 1
+        if needs_extra:
+            def reader(data, pos):
+                result = int.from_bytes(
+                    data[pos + offset_bytes: pos + end_byte], byteorder="little")
+                result >>= offset_extra
+                result |= data[pos + end_byte] << extra_shift
+                result &= mask
+                return (result & sign_mask) - (result & sign_bit)
+        elif offset_extra:
+            def reader(data, pos):
+                result = (int.from_bytes(
+                    data[pos + offset_bytes: pos + end_byte],
+                    byteorder="little") >> offset_extra) & mask
+                return (result & sign_mask) - (result & sign_bit)
+        else:
+            def reader(data, pos):
+                result = int.from_bytes(
+                    data[pos + offset_bytes: pos + end_byte],
+                    byteorder="little") & mask
+                return (result & sign_mask) - (result & sign_bit)
+        return reader
+
+<<<<<<< HEAD
+=======
+    # Unsigned paths
+    if needs_extra:
+        def reader(data, pos):
+            result = int.from_bytes(
+                data[pos + offset_bytes: pos + end_byte], byteorder="little")
+            result >>= offset_extra
+            result |= data[pos + end_byte] << extra_shift
+            return result & mask
+    elif offset_extra:
+        def reader(data, pos):
+            return (int.from_bytes(
+                data[pos + offset_bytes: pos + end_byte],
+                byteorder="little") >> offset_extra) & mask
+    else:
+        def reader(data, pos):
+            return int.from_bytes(
+                data[pos + offset_bytes: pos + end_byte],
+                byteorder="little") & mask
+    return reader
+
+
+def read_field_vectorized(raw_bytes_2d, field_offset_bits, field_width_bits, is_signed):
+    """Read a bit-packed field from all elements at once, returning a numpy array.
+
+    :param raw_bytes_2d: numpy uint8 array shaped (num_elements, struct_size_bytes)
+    :param field_offset_bits: bit offset of the field within each element
+    :param field_width_bits: width of the field in bits (max 64)
+    :param is_signed: whether to sign-extend the result
+    :return: numpy array of field values
+    """
+    if field_width_bits == 1:
+        byte_idx = field_offset_bits // 8
+        bit_idx = field_offset_bits % 8
+        return ((raw_bytes_2d[:, byte_idx].astype(np.uint64) >> np.uint64(bit_idx)) &
+                np.uint64(1))
+
+    byte_start = field_offset_bits // 8
+    bit_shift = field_offset_bits % 8
+    bytes_needed = (bit_shift + field_width_bits + 7) // 8
+
+    # Use Python int arithmetic for the shift to avoid numpy overflow,
+    # then broadcast back to the array.
+    result = np.zeros(raw_bytes_2d.shape[0], dtype=np.uint64)
+    for b in range(min(bytes_needed, 8)):
+        result |= raw_bytes_2d[:, byte_start + b].astype(np.uint64) << np.uint64(b * 8)
+    result >>= np.uint64(bit_shift)
+
+    # If the field spans more than 8 bytes (unaligned 64-bit field), merge the extra byte.
+    bits_so_far = 8 * min(bytes_needed, 8) - bit_shift
+    if bits_so_far < field_width_bits and bytes_needed > 8:
+        extra = raw_bytes_2d[:, byte_start + 8].astype(np.uint64)
+        result |= extra << np.uint64(bits_so_far)
+
+    if field_width_bits < 64:
+        result &= np.uint64((1 << field_width_bits) - 1)
+
+    if is_signed:
+        if field_width_bits == 64:
+            return result.view(np.int64)
+        sign_bit = np.uint64(1 << (field_width_bits - 1))
+        offset = -(1 << field_width_bits)
+        signed = result.astype(np.int64) + np.int64(offset)
+        result = np.where(result & sign_bit, signed, result.astype(np.int64))
+
+    return result
+
+
+def read_value(data, offset_bits, num_bits, is_signed):
+    """Read a bit-packed value from data at the given bit offset.
+
+    This is a convenience wrapper around :func:`make_field_reader` for one-off
+    reads.  For repeated reads of the same field, prefer building a reader once
+    with ``make_field_reader`` and reusing it.
+    """
+    reader = make_field_reader(offset_bits, num_bits, is_signed)
+    return reader(data, 0)
+
+
+def write_value(data, offset_bits, num_bits, is_signed, value):
+    assert num_bits <= 64, f'Number of bits to write is greater than 64'
+
+    offset_bytes, offset_extra_bits = divmod(offset_bits, 8)
+    total_bytes = (num_bits + 7) // 8
+
+    if num_bits == 1:
+        if value == 1:
+            data[offset_bytes] |= 1 << offset_extra_bits
+        else:
+            data[offset_bytes] &= ~(1 << offset_extra_bits)
+        return
+
+    mask = (1 << num_bits) - 1
+    value <<= offset_extra_bits
+    value &= mask << offset_extra_bits
+    value_in_little_endian = value.to_bytes(total_bytes + 1, byteorder="little", signed=is_signed)
+    surrounding_bits = data[offset_bytes] & ((1 << offset_bits) - 1)
+
+    byte_idx = 0
+    data[offset_bytes] = value_in_little_endian[byte_idx]
+    data[offset_bytes] |= surrounding_bits
+
+    byte_idx += 1
+    while byte_idx < total_bytes:
+        data[offset_bytes + byte_idx] = value_in_little_endian[byte_idx]
+        byte_idx += 1
+
+>>>>>>> e486615 (Also improve scalar readers by caching)
+    bits_written = total_bytes * 8 - offset_extra_bits
+    if bits_written < num_bits:
+        surrounding_bits = data[offset_bytes + byte_idx] & ~((1 << offset_bits) - 1)
+        data[offset_bytes + byte_idx] = value_in_little_endian[byte_idx] & ((1 << (8 - (bits_written % 8))) - 1)
+        data[offset_bytes + byte_idx] |= surrounding_bits

flatdata_py-0.4.11/flatdata/lib/data_access_BACKUP_91129.py

@@ -0,0 +1,219 @@
+'''
+ Copyright (c) 2017 HERE Europe B.V.
+ See the LICENSE file in the root of this project for license details.
+'''
+
+import numpy as np
+
+# Sign bits cache for the value reading.
+_SIGN_BITS = [0] + [(1 << (bits - 1)) for bits in range(1, 65)]
+
+
+def make_field_reader(offset_bits, num_bits, is_signed):
+    """Build a specialized closure for reading a single field from a structure.
+
+    Returns a function reader(data, pos_bytes) that reads the field value
+    from ``data`` at byte position ``pos_bytes``.  All constants (byte offset,
+    bit shift, mask, sign handling) are pre-computed and captured by the
+    closure so the hot path does minimal work.
+    """
+    offset_bytes, offset_extra = divmod(offset_bits, 8)
+    total_bytes = (num_bits + 7) // 8
+    end_byte = offset_bytes + total_bytes
+    mask = (1 << num_bits) - 1
+    needs_extra = (total_bytes * 8 - offset_extra) < num_bits
+    extra_shift = total_bytes * 8 - offset_extra
+
+    if num_bits == 1:
+        bit_mask = 1 << offset_extra
+        def reader(data, pos):
+            return int((data[pos + offset_bytes] & bit_mask) != 0)
+        return reader
+
+    if is_signed:
+        sign_bit = _SIGN_BITS[num_bits]
+        sign_mask = sign_bit - 1
+        if needs_extra:
+            def reader(data, pos):
+                result = int.from_bytes(
+                    data[pos + offset_bytes: pos + end_byte], byteorder="little")
+                result >>= offset_extra
+                result |= data[pos + end_byte] << extra_shift
+                result &= mask
+                return (result & sign_mask) - (result & sign_bit)
+        elif offset_extra:
+            def reader(data, pos):
+                result = (int.from_bytes(
+                    data[pos + offset_bytes: pos + end_byte],
+                    byteorder="little") >> offset_extra) & mask
+                return (result & sign_mask) - (result & sign_bit)
+        else:
+            def reader(data, pos):
+                result = int.from_bytes(
+                    data[pos + offset_bytes: pos + end_byte],
+                    byteorder="little") & mask
+                return (result & sign_mask) - (result & sign_bit)
+        return reader
+
+<<<<<<< HEAD
+    if num_bits < 64 or offset_extra_bits > 0:
+        result = result & ((1 << num_bits) - 1)
+=======
+    # Unsigned paths
+    if needs_extra:
+        def reader(data, pos):
+            result = int.from_bytes(
+                data[pos + offset_bytes: pos + end_byte], byteorder="little")
+            result >>= offset_extra
+            result |= data[pos + end_byte] << extra_shift
+            return result & mask
+    elif offset_extra:
+        def reader(data, pos):
+            return (int.from_bytes(
+                data[pos + offset_bytes: pos + end_byte],
+                byteorder="little") >> offset_extra) & mask
+    else:
+        def reader(data, pos):
+            return int.from_bytes(
+                data[pos + offset_bytes: pos + end_byte],
+                byteorder="little") & mask
+    return reader
+>>>>>>> c635308 (Also improve scalar readers by caching)
+
+
+def read_field_vectorized(raw_bytes_2d, field_offset_bits, field_width_bits, is_signed):
+    """Read a bit-packed field from all elements at once, returning a numpy array.
+
+    :param raw_bytes_2d: numpy uint8 array shaped (num_elements, struct_size_bytes)
+    :param field_offset_bits: bit offset of the field within each element
+    :param field_width_bits: width of the field in bits (max 64)
+    :param is_signed: whether to sign-extend the result
+    :return: numpy array of field values
+    """
+    if field_width_bits == 1:
+        byte_idx = field_offset_bits // 8
+        bit_idx = field_offset_bits % 8
+        return ((raw_bytes_2d[:, byte_idx].astype(np.uint64) >> np.uint64(bit_idx)) &
+                np.uint64(1))
+
+    byte_start = field_offset_bits // 8
+    bit_shift = field_offset_bits % 8
+    bytes_needed = (bit_shift + field_width_bits + 7) // 8
+
+    # Use Python int arithmetic for the shift to avoid numpy overflow,
+    # then broadcast back to the array.
+    result = np.zeros(raw_bytes_2d.shape[0], dtype=np.uint64)
+    for b in range(min(bytes_needed, 8)):
+        result |= raw_bytes_2d[:, byte_start + b].astype(np.uint64) << np.uint64(b * 8)
+    result >>= np.uint64(bit_shift)
+
+    # If the field spans more than 8 bytes (unaligned 64-bit field), merge the extra byte.
+    bits_so_far = 8 * min(bytes_needed, 8) - bit_shift
+    if bits_so_far < field_width_bits and bytes_needed > 8:
+        extra = raw_bytes_2d[:, byte_start + 8].astype(np.uint64)
+        result |= extra << np.uint64(bits_so_far)
+
+    if field_width_bits < 64:
+        result &= np.uint64((1 << field_width_bits) - 1)
+
+    if is_signed:
+        if field_width_bits == 64:
+            return result.view(np.int64)
+        sign_bit = np.uint64(1 << (field_width_bits - 1))
+        offset = -(1 << field_width_bits)
+        signed = result.astype(np.int64) + np.int64(offset)
+        result = np.where(result & sign_bit, signed, result.astype(np.int64))
+
+    return result
+
+
+def read_value(data, offset_bits, num_bits, is_signed):
+    """Read a bit-packed value from data at the given bit offset.
+
+    This is a convenience wrapper around :func:`make_field_reader` for one-off
+    reads.  For repeated reads of the same field, prefer building a reader once
+    with ``make_field_reader`` and reusing it.
+    """
+    reader = make_field_reader(offset_bits, num_bits, is_signed)
+    return reader(data, 0)
+
+
+def write_value(data, offset_bits, num_bits, is_signed, value):
+    assert num_bits <= 64, f'Number of bits to write is greater than 64'
+
+    offset_bytes, offset_extra_bits = divmod(offset_bits, 8)
+    total_bytes = (num_bits + 7) // 8
+
+    if num_bits == 1:
+        if value == 1:
+            data[offset_bytes] |= 1 << offset_extra_bits
+        else:
+            data[offset_bytes] &= ~(1 << offset_extra_bits)
+        return
+
+    mask = (1 << num_bits) - 1
+    value <<= offset_extra_bits
+    value &= mask << offset_extra_bits
+    value_in_little_endian = value.to_bytes(total_bytes + 1, byteorder="little", signed=is_signed)
+    surrounding_bits = data[offset_bytes] & ((1 << offset_bits) - 1)
+
+    byte_idx = 0
+    data[offset_bytes] = value_in_little_endian[byte_idx]
+    data[offset_bytes] |= surrounding_bits
+
+    byte_idx += 1
+    while byte_idx < total_bytes:
+        data[offset_bytes + byte_idx] = value_in_little_endian[byte_idx]
+        byte_idx += 1
+
+    bits_written = total_bytes * 8 - offset_extra_bits
+    if bits_written < num_bits:
+        surrounding_bits = data[offset_bytes + byte_idx] & ~((1 << offset_bits) - 1)
+        data[offset_bytes + byte_idx] = value_in_little_endian[byte_idx] & ((1 << (8 - (bits_written % 8))) - 1)
+        data[offset_bytes + byte_idx] |= surrounding_bits
+
+
+def read_field_vectorized(raw_bytes_2d, field_offset_bits, field_width_bits, is_signed):
+    """Read a bit-packed field from all elements at once, returning a numpy array.
+
+    :param raw_bytes_2d: numpy uint8 array shaped (num_elements, struct_size_bytes)
+    :param field_offset_bits: bit offset of the field within each element
+    :param field_width_bits: width of the field in bits (max 64)
+    :param is_signed: whether to sign-extend the result
+    :return: numpy array of field values
+    """
+    if field_width_bits == 1:
+        byte_idx = field_offset_bits // 8
+        bit_idx = field_offset_bits % 8
+        return ((raw_bytes_2d[:, byte_idx].astype(np.uint64) >> np.uint64(bit_idx)) &
+                np.uint64(1))
+
+    byte_start = field_offset_bits // 8
+    bit_shift = field_offset_bits % 8
+    bytes_needed = (bit_shift + field_width_bits + 7) // 8
+
+    # Use Python int arithmetic for the shift to avoid numpy overflow,
+    # then broadcast back to the array.
+    result = np.zeros(raw_bytes_2d.shape[0], dtype=np.uint64)
+    for b in range(min(bytes_needed, 8)):
+        result |= raw_bytes_2d[:, byte_start + b].astype(np.uint64) << np.uint64(b * 8)
+    result >>= np.uint64(bit_shift)
+
+    # If the field spans more than 8 bytes (unaligned 64-bit field), merge the extra byte.
+    bits_so_far = 8 * min(bytes_needed, 8) - bit_shift
+    if bits_so_far < field_width_bits and bytes_needed > 8:
+        extra = raw_bytes_2d[:, byte_start + 8].astype(np.uint64)
+        result |= extra << np.uint64(bits_so_far)
+
+    if field_width_bits < 64:
+        result &= np.uint64((1 << field_width_bits) - 1)
+
+    if is_signed:
+        if field_width_bits == 64:
+            return result.view(np.int64)
+        sign_bit = np.uint64(1 << (field_width_bits - 1))
+        offset = -(1 << field_width_bits)
+        signed = result.astype(np.int64) + np.int64(offset)
+        result = np.where(result & sign_bit, signed, result.astype(np.int64))
+
+    return result

flatdata_py-0.4.11/flatdata/lib/data_access_LOCAL_91129.py

@@ -0,0 +1,112 @@
+'''
+ Copyright (c) 2017 HERE Europe B.V.
+ See the LICENSE file in the root of this project for license details.
+'''
+
+import numpy as np
+
+# Sign bits cache for the value reading.
+_SIGN_BITS = [0] + [(1 << (bits - 1)) for bits in range(1, 65)]
+
+
+def read_value(data, offset_bits, num_bits, is_signed):
+    offset_bytes, offset_extra_bits = divmod(offset_bits, 8)
+    total_bytes = (num_bits + 7) // 8
+
+    if num_bits == 1:
+        return int((data[offset_bytes] & (1 << offset_extra_bits)) != 0)
+
+    result = int.from_bytes(data[offset_bytes: offset_bytes + total_bytes], byteorder="little")
+    result >>= offset_extra_bits
+    if (total_bytes * 8 - offset_extra_bits) < num_bits:
+        remainder = data[offset_bytes + total_bytes]
+        result |= remainder << (total_bytes * 8 - offset_extra_bits)
+
+    if num_bits < 64 or offset_extra_bits > 0:
+        result = result & ((1 << num_bits) - 1)
+
+    if not is_signed:
+        return result
+
+    return (result & (_SIGN_BITS[num_bits] - 1)) - (result & _SIGN_BITS[num_bits])
+
+
+def write_value(data, offset_bits, num_bits, is_signed, value):
+    assert num_bits <= 64, f'Number of bits to write is greater than 64'
+
+    offset_bytes, offset_extra_bits = divmod(offset_bits, 8)
+    total_bytes = (num_bits + 7) // 8
+
+    if num_bits == 1:
+        if value == 1:
+            data[offset_bytes] |= 1 << offset_extra_bits
+        else:
+            data[offset_bytes] &= ~(1 << offset_extra_bits)
+        return
+
+    mask = (1 << num_bits) - 1
+    value <<= offset_extra_bits
+    value &= mask << offset_extra_bits
+    value_in_little_endian = value.to_bytes(total_bytes + 1, byteorder="little", signed=is_signed)
+    surrounding_bits = data[offset_bytes] & ((1 << offset_bits) - 1)
+
+    byte_idx = 0
+    data[offset_bytes] = value_in_little_endian[byte_idx]
+    data[offset_bytes] |= surrounding_bits
+
+    byte_idx += 1
+    while byte_idx < total_bytes:
+        data[offset_bytes + byte_idx] = value_in_little_endian[byte_idx]
+        byte_idx += 1
+
+    bits_written = total_bytes * 8 - offset_extra_bits
+    if bits_written < num_bits:
+        surrounding_bits = data[offset_bytes + byte_idx] & ~((1 << offset_bits) - 1)
+        data[offset_bytes + byte_idx] = value_in_little_endian[byte_idx] & ((1 << (8 - (bits_written % 8))) - 1)
+        data[offset_bytes + byte_idx] |= surrounding_bits
+
+
+def read_field_vectorized(raw_bytes_2d, field_offset_bits, field_width_bits, is_signed):
+    """Read a bit-packed field from all elements at once, returning a numpy array.
+
+    :param raw_bytes_2d: numpy uint8 array shaped (num_elements, struct_size_bytes)
+    :param field_offset_bits: bit offset of the field within each element
+    :param field_width_bits: width of the field in bits (max 64)
+    :param is_signed: whether to sign-extend the result
+    :return: numpy array of field values
+    """
+    if field_width_bits == 1:
+        byte_idx = field_offset_bits // 8
+        bit_idx = field_offset_bits % 8
+        return ((raw_bytes_2d[:, byte_idx].astype(np.uint64) >> np.uint64(bit_idx)) &
+                np.uint64(1))
+
+    byte_start = field_offset_bits // 8
+    bit_shift = field_offset_bits % 8
+    bytes_needed = (bit_shift + field_width_bits + 7) // 8
+
+    # Use Python int arithmetic for the shift to avoid numpy overflow,
+    # then broadcast back to the array.
+    result = np.zeros(raw_bytes_2d.shape[0], dtype=np.uint64)
+    for b in range(min(bytes_needed, 8)):
+        result |= raw_bytes_2d[:, byte_start + b].astype(np.uint64) << np.uint64(b * 8)
+    result >>= np.uint64(bit_shift)
+
+    # If the field spans more than 8 bytes (unaligned 64-bit field), merge the extra byte.
+    bits_so_far = 8 * min(bytes_needed, 8) - bit_shift
+    if bits_so_far < field_width_bits and bytes_needed > 8:
+        extra = raw_bytes_2d[:, byte_start + 8].astype(np.uint64)
+        result |= extra << np.uint64(bits_so_far)
+
+    if field_width_bits < 64:
+        result &= np.uint64((1 << field_width_bits) - 1)
+
+    if is_signed:
+        if field_width_bits == 64:
+            return result.view(np.int64)
+        sign_bit = np.uint64(1 << (field_width_bits - 1))
+        offset = -(1 << field_width_bits)
+        signed = result.astype(np.int64) + np.int64(offset)
+        result = np.where(result & sign_bit, signed, result.astype(np.int64))
+
+    return result

flatdata_py-0.4.11/flatdata/lib/data_access_REMOTE_91129.py

@@ -0,0 +1,168 @@
+'''
+ Copyright (c) 2017 HERE Europe B.V.
+ See the LICENSE file in the root of this project for license details.
+'''
+
+import numpy as np
+
+# Sign bits cache for the value reading.
+_SIGN_BITS = [0] + [(1 << (bits - 1)) for bits in range(1, 65)]
+
+
+def make_field_reader(offset_bits, num_bits, is_signed):
+    """Build a specialized closure for reading a single field from a structure.
+
+    Returns a function reader(data, pos_bytes) that reads the field value
+    from ``data`` at byte position ``pos_bytes``.  All constants (byte offset,
+    bit shift, mask, sign handling) are pre-computed and captured by the
+    closure so the hot path does minimal work.
+    """
+    offset_bytes, offset_extra = divmod(offset_bits, 8)
+    total_bytes = (num_bits + 7) // 8
+    end_byte = offset_bytes + total_bytes
+    mask = (1 << num_bits) - 1
+    needs_extra = (total_bytes * 8 - offset_extra) < num_bits
+    extra_shift = total_bytes * 8 - offset_extra
+
+    if num_bits == 1:
+        bit_mask = 1 << offset_extra
+        def reader(data, pos):
+            return int((data[pos + offset_bytes] & bit_mask) != 0)
+        return reader
+
+    if is_signed:
+        sign_bit = _SIGN_BITS[num_bits]
+        sign_mask = sign_bit - 1
+        if needs_extra:
+            def reader(data, pos):
+                result = int.from_bytes(
+                    data[pos + offset_bytes: pos + end_byte], byteorder="little")
+                result >>= offset_extra
+                result |= data[pos + end_byte] << extra_shift
+                result &= mask
+                return (result & sign_mask) - (result & sign_bit)
+        elif offset_extra:
+            def reader(data, pos):
+                result = (int.from_bytes(
+                    data[pos + offset_bytes: pos + end_byte],
+                    byteorder="little") >> offset_extra) & mask
+                return (result & sign_mask) - (result & sign_bit)
+        else:
+            def reader(data, pos):
+                result = int.from_bytes(
+                    data[pos + offset_bytes: pos + end_byte],
+                    byteorder="little") & mask
+                return (result & sign_mask) - (result & sign_bit)
+        return reader
+
+    # Unsigned paths
+    if needs_extra:
+        def reader(data, pos):
+            result = int.from_bytes(
+                data[pos + offset_bytes: pos + end_byte], byteorder="little")
+            result >>= offset_extra
+            result |= data[pos + end_byte] << extra_shift
+            return result & mask
+    elif offset_extra:
+        def reader(data, pos):
+            return (int.from_bytes(
+                data[pos + offset_bytes: pos + end_byte],
+                byteorder="little") >> offset_extra) & mask
+    else:
+        def reader(data, pos):
+            return int.from_bytes(
+                data[pos + offset_bytes: pos + end_byte],
+                byteorder="little") & mask
+    return reader
+
+
+def read_field_vectorized(raw_bytes_2d, field_offset_bits, field_width_bits, is_signed):
+    """Read a bit-packed field from all elements at once, returning a numpy array.
+
+    :param raw_bytes_2d: numpy uint8 array shaped (num_elements, struct_size_bytes)
+    :param field_offset_bits: bit offset of the field within each element
+    :param field_width_bits: width of the field in bits (max 64)
+    :param is_signed: whether to sign-extend the result
+    :return: numpy array of field values
+    """
+    if field_width_bits == 1:
+        byte_idx = field_offset_bits // 8
+        bit_idx = field_offset_bits % 8
+        return ((raw_bytes_2d[:, byte_idx].astype(np.uint64) >> np.uint64(bit_idx)) &
+                np.uint64(1))
+
+    byte_start = field_offset_bits // 8
+    bit_shift = field_offset_bits % 8
+    bytes_needed = (bit_shift + field_width_bits + 7) // 8
+
+    # Use Python int arithmetic for the shift to avoid numpy overflow,
+    # then broadcast back to the array.
+    result = np.zeros(raw_bytes_2d.shape[0], dtype=np.uint64)
+    for b in range(min(bytes_needed, 8)):
+        result |= raw_bytes_2d[:, byte_start + b].astype(np.uint64) << np.uint64(b * 8)
+    result >>= np.uint64(bit_shift)
+
+    # If the field spans more than 8 bytes (unaligned 64-bit field), merge the extra byte.
+    bits_so_far = 8 * min(bytes_needed, 8) - bit_shift
+    if bits_so_far < field_width_bits and bytes_needed > 8:
+        extra = raw_bytes_2d[:, byte_start + 8].astype(np.uint64)
+        result |= extra << np.uint64(bits_so_far)
+
+    if field_width_bits < 64:
+        result &= np.uint64((1 << field_width_bits) - 1)
+
+    if is_signed:
+        if field_width_bits == 64:
+            return result.view(np.int64)
+        sign_bit = np.uint64(1 << (field_width_bits - 1))
+        offset = -(1 << field_width_bits)
+        signed = result.astype(np.int64) + np.int64(offset)
+        result = np.where(result & sign_bit, signed, result.astype(np.int64))
+
+    return result
+
+
+def read_value(data, offset_bits, num_bits, is_signed):
+    """Read a bit-packed value from data at the given bit offset.
+
+    This is a convenience wrapper around :func:`make_field_reader` for one-off
+    reads.  For repeated reads of the same field, prefer building a reader once
+    with ``make_field_reader`` and reusing it.
+    """
+    reader = make_field_reader(offset_bits, num_bits, is_signed)
+    return reader(data, 0)
+
+
+def write_value(data, offset_bits, num_bits, is_signed, value):
+    assert num_bits <= 64, f'Number of bits to write is greater than 64'
+
+    offset_bytes, offset_extra_bits = divmod(offset_bits, 8)
+    total_bytes = (num_bits + 7) // 8
+
+    if num_bits == 1:
+        if value == 1:
+            data[offset_bytes] |= 1 << offset_extra_bits
+        else:
+            data[offset_bytes] &= ~(1 << offset_extra_bits)
+        return
+
+    mask = (1 << num_bits) - 1
+    value <<= offset_extra_bits
+    value &= mask << offset_extra_bits
+    value_in_little_endian = value.to_bytes(total_bytes + 1, byteorder="little", signed=is_signed)
+    surrounding_bits = data[offset_bytes] & ((1 << offset_bits) - 1)
+
+    byte_idx = 0
+    data[offset_bytes] = value_in_little_endian[byte_idx]
+    data[offset_bytes] |= surrounding_bits
+
+    byte_idx += 1
+    while byte_idx < total_bytes:
+        data[offset_bytes + byte_idx] = value_in_little_endian[byte_idx]
+        byte_idx += 1
+
+    bits_written = total_bytes * 8 - offset_extra_bits
+    if bits_written < num_bits:
+        surrounding_bits = data[offset_bytes + byte_idx] & ~((1 << offset_bits) - 1)
+        data[offset_bytes + byte_idx] = value_in_little_endian[byte_idx] & ((1 << (8 - (bits_written % 8))) - 1)
+        data[offset_bytes + byte_idx] |= surrounding_bits

{flatdata_py-0.4.10 → flatdata_py-0.4.11}/flatdata/lib/resources.py

@@ -8,7 +8,7 @@ import json
 import pandas as pd
 import numpy as np
 
-from .data_access import read_value
+from .data_access import read_value, read_field_vectorized
 from .errors import CorruptResourceError
 
 SIZE_OFFSET_IN_BITS = 64
@@ -24,6 +24,7 @@ class ResourceBase:
         self._element_type = element_type
         self._element_types = [element_type]
         self._type_size_in_bytes = self._element_type._SIZE_IN_BYTES if self._element_type else 1
+        self._raw_numpy_2d = None
 
     def size_in_bytes(self):
         return len(self._mem)
@@ -35,6 +36,20 @@ class ResourceBase:
         offset = self._item_offset(index)
         return self._element_type(self._mem, offset)
 
+    def _as_numpy_2d(self):
+        """Return the raw data as a 2D numpy uint8 array of shape (n, struct_size).
+        Zero-copy via np.frombuffer on the mmap'd memory. Cached after first call.
+        """
+        if self._raw_numpy_2d is None:
+            n = len(self)
+            struct_size = self._type_size_in_bytes
+            raw = np.frombuffer(
+                self._mem[SIZE_OFFSET_IN_BYTES:SIZE_OFFSET_IN_BYTES + n * struct_size],
+                dtype=np.uint8,
+            )
+            self._raw_numpy_2d = raw.reshape(n, struct_size)
+        return self._raw_numpy_2d
+
     def _repr_attributes(self):
         return {
             "container_type": self.__class__.__name__,
@@ -60,14 +75,18 @@ class _VectorSlice:
         self._sequence = sequence
 
     def to_numpy(self, limit=None):
-        indices = self._slice.indices(len(self._sequence))
-        num_items = len(range(*indices)) if not limit else limit
-        result = np.empty(
-            shape=num_items,
-            dtype=self._sequence._element_type.dtype()
-        )
-        for index, item in enumerate(self):
-            result[index] = item.as_tuple()
+        raw_2d = self._sequence._as_numpy_2d()
+        sliced = raw_2d[self._slice]
+        if limit is not None:
+            sliced = sliced[:limit]
+
+        fields = self._sequence._element_type._FIELDS
+        dtype = self._sequence._element_type.dtype()
+        result = np.empty(sliced.shape[0], dtype=dtype)
+        for name, field in fields.items():
+            result[name] = read_field_vectorized(
+                sliced, field.offset, field.width, field.is_signed
+            )
         return result
 
     def to_data_frame(self, limit=None):
@@ -78,7 +97,13 @@ class _VectorSlice:
             yield self._sequence[i]
 
     def __getattr__(self, name):
-        return pd.DataFrame(data=[[getattr(item, name)] for item in self], columns=[name])
+        try:
+            field = self._sequence._element_type._FIELDS[name]
+        except KeyError:
+            raise AttributeError("Field %s not found in structure" % name)
+        raw_2d = self._sequence._as_numpy_2d()[self._slice]
+        values = read_field_vectorized(raw_2d, field.offset, field.width, field.is_signed)
+        return pd.DataFrame(data=values, columns=[name])
 
     def __repr__(self):
         return "Displaying first 100 records:\n" + self.to_data_frame(limit=100).__repr__()
@@ -92,8 +117,20 @@ class Vector(ResourceBase):
         assert rem == 0, "Malformed vector"
         self._size = size
 
+    def to_numpy(self):
+        """Convert entire vector to a numpy structured array (vectorized)."""
+        raw_2d = self._as_numpy_2d()
+        fields = self._element_type._FIELDS
+        dtype = self._element_type.dtype()
+        result = np.empty(self._size, dtype=dtype)
+        for name, field in fields.items():
+            result[name] = read_field_vectorized(
+                raw_2d, field.offset, field.width, field.is_signed
+            )
+        return result
+
     def to_data_frame(self):
-        return self[:].to_data_frame()
+        return pd.DataFrame(data=self.to_numpy())
 
     def __getitem__(self, index):
         if isinstance(index, slice):
@@ -106,11 +143,20 @@ class Vector(ResourceBase):
         return self._get_item(index)
 
     def __iter__(self):
-        for i in range(len(self)):
-            yield self._get_item(i)
+        mem = self._mem
+        element_type = self._element_type
+        size_bytes = self._type_size_in_bytes
+        for i in range(self._size):
+            yield element_type(mem, SIZE_OFFSET_IN_BYTES + size_bytes * i)
 
     def __getattr__(self, name):
-        return pd.DataFrame(data=[[getattr(item, name)] for item in self], columns=[name])
+        try:
+            field = self._element_type._FIELDS[name]
+        except KeyError:
+            raise AttributeError("Field %s not found in structure" % name)
+        raw_2d = self._as_numpy_2d()
+        values = read_field_vectorized(raw_2d, field.offset, field.width, field.is_signed)
+        return pd.DataFrame(data=values, columns=[name])
 
     def __len__(self):
         return self._size

{flatdata_py-0.4.10 → flatdata_py-0.4.11}/flatdata/lib/structure.py

@@ -2,26 +2,33 @@ from collections import namedtuple
 import json
 import numpy as np
 
-from .data_access import read_value
+from .data_access import make_field_reader
 
 FieldSignature = namedtuple(
     "FieldSignature", ["offset", "width", "is_signed", "dtype"])
 
 
 class Structure:
+    __slots__ = ('_mem', '_pos')
+    _READERS = {}
+
+    def __init_subclass__(cls, **kwargs):
+        super().__init_subclass__(**kwargs)
+        fields = cls.__dict__.get('_FIELDS')
+        if fields is not None:
+            cls._READERS = {name: make_field_reader(f.offset, f.width, f.is_signed)
+                            for name, f in fields.items()}
+
     def __init__(self, mem, pos):
         self._mem = mem
         self._pos = pos
 
     def __getattr__(self, name):
         try:
-            field = self._FIELDS[name]
+            reader = self._READERS[name]
         except KeyError:
             raise AttributeError("Field %s not found in structure" % name)
-        return self._get_value(field)
-
-    def _get_value(self, field):
-        return read_value(self._mem, self._pos * 8 + field.offset, field.width, field.is_signed)
+        return reader(self._mem, self._pos)
 
     def __dir__(self):
         return self._FIELD_KEYS
@@ -31,20 +38,24 @@ class Structure:
             yield getattr(self, name)
 
     def as_dict(self):
-        return {name: self._get_value(field) for name, field in self._FIELDS.items()}
+        mem, pos = self._mem, self._pos
+        return {name: reader(mem, pos) for name, reader in self._READERS.items()}
 
     def as_list(self):
-        return [self._get_value(field) for field in self._FIELDS.values()]
+        mem, pos = self._mem, self._pos
+        return [reader(mem, pos) for reader in self._READERS.values()]
 
     def as_tuple(self):
-        return tuple(self._get_value(field) for field in self._FIELDS.values())
+        mem, pos = self._mem, self._pos
+        return tuple(reader(mem, pos) for reader in self._READERS.values())
 
     @classmethod
     def dtype(cls):
         return [(name, np.dtype(field.dtype)) for name, field in cls._FIELDS.items()]
 
     def as_nparray(self):
-        return np.array([tuple(self._get_value(field) for name, field in self._FIELDS.items())],
+        mem, pos = self._mem, self._pos
+        return np.array([tuple(reader(mem, pos) for reader in self._READERS.values())],
                         dtype=self.dtype())
 
     def schema(self):

flatdata_py-0.4.11/flatdata/lib/structure.py.orig

@@ -0,0 +1,92 @@
+from collections import namedtuple
+import json
+import numpy as np
+
+from .data_access import make_field_reader
+
+FieldSignature = namedtuple(
+    "FieldSignature", ["offset", "width", "is_signed", "dtype"])
+
+
+class Structure:
+<<<<<<< HEAD
+=======
+    __slots__ = ('_mem', '_pos')
+    _READERS = {}
+
+    def __init_subclass__(cls, **kwargs):
+        super().__init_subclass__(**kwargs)
+        fields = cls.__dict__.get('_FIELDS')
+        if fields is not None:
+            cls._READERS = {name: make_field_reader(f.offset, f.width, f.is_signed)
+                            for name, f in fields.items()}
+
+>>>>>>> e486615 (Also improve scalar readers by caching)
+    def __init__(self, mem, pos):
+        self._mem = mem
+        self._pos = pos
+
+    def __getattr__(self, name):
+        try:
+            reader = self._READERS[name]
+        except KeyError:
+            raise AttributeError("Field %s not found in structure" % name)
+        return reader(self._mem, self._pos)
+
+    def __dir__(self):
+        return self._FIELD_KEYS
+
+    def __iter__(self):
+        for name in self._FIELD_KEYS:
+            yield getattr(self, name)
+
+    def as_dict(self):
+        mem, pos = self._mem, self._pos
+        return {name: reader(mem, pos) for name, reader in self._READERS.items()}
+
+    def as_list(self):
+        mem, pos = self._mem, self._pos
+        return [reader(mem, pos) for reader in self._READERS.values()]
+
+    def as_tuple(self):
+        mem, pos = self._mem, self._pos
+        return tuple(reader(mem, pos) for reader in self._READERS.values())
+
+    @classmethod
+    def dtype(cls):
+        return [(name, np.dtype(field.dtype)) for name, field in cls._FIELDS.items()]
+
+    def as_nparray(self):
+        mem, pos = self._mem, self._pos
+        return np.array([tuple(reader(mem, pos) for reader in self._READERS.values())],
+                        dtype=self.dtype())
+
+    def schema(self):
+        return self._SCHEMA
+
+    @classmethod
+    def _repr_attributes(cls):
+        return {
+            "name": cls.__name__,
+            "doc": cls.__doc__,
+            "attributes": [
+                {
+                    "name": name,
+                    "offset": signature.offset,
+                    "width": signature.width,
+                    "is_signed": signature.is_signed
+                }
+                for name, signature in cls._FIELDS.items()]
+        }
+
+    @classmethod
+    def __repr__(cls):
+        return json.dumps(cls._repr_attributes())
+
+    def __repr__(self):
+        return json.dumps({
+            "name": self.__class__.__name__,
+            "attributes":
+                {name: getattr(self, name)
+                 for name, signature in self._FIELDS.items()}
+        }, indent=4)

{flatdata_py-0.4.10 → flatdata_py-0.4.11}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "flatdata-py"
-version = "0.4.10"
+version = "0.4.11"
 description = "Python 3 implementation of Flatdata"
 readme = "README.md"
 authors = [
@@ -16,7 +16,7 @@ classifiers = [
     "Programming Language :: Python :: 3",
 ]
 dependencies = [
-    "flatdata-generator==0.4.10",
+    "flatdata-generator==0.4.11",
     "numpy",
     "pandas",
 ]