python-gdb 0.2.0__tar.gz → 0.2.1__tar.gz

{python_gdb-0.2.0 → python_gdb-0.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python-gdb
-Version: 0.2.0
+Version: 0.2.1
 Classifier: Development Status :: 3 - Alpha
 Classifier: Intended Audience :: Science/Research
 Classifier: Topic :: Scientific/Engineering :: GIS
@@ -190,6 +190,7 @@ zensical serve
   boundary on reverse engineering.
 - [`docs/provenance/`](docs/provenance/index.md) — the original
   research log and write-up this implementation was derived from.
+- [`CHANGELOG.md`](CHANGELOG.md) — what changed in each release.
 
 ## Testing
 

{python_gdb-0.2.0 → python_gdb-0.2.1}/README.md

@@ -152,6 +152,7 @@ zensical serve
   boundary on reverse engineering.
 - [`docs/provenance/`](docs/provenance/index.md) — the original
   research log and write-up this implementation was derived from.
+- [`CHANGELOG.md`](CHANGELOG.md) — what changed in each release.
 
 ## Testing
 

{python_gdb-0.2.0 → python_gdb-0.2.1}/pygdb/gdb.py

@@ -16,7 +16,7 @@ from __future__ import annotations
 import warnings
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Dict, Iterator, List, Optional, Tuple, Union
+from typing import Dict, Iterator, List, Optional, Sequence, Tuple, Union
 
 import numpy as np
 
@@ -60,6 +60,87 @@ LineRef = Union[str, Tuple[str, int], "LineRecord"]
 ChannelRef = Union[str, Tuple[str, int], "ChannelRecord"]
 
 
+def _finite_values(values: np.ndarray) -> np.ndarray:
+    v = np.asarray(values, dtype=float).ravel()
+    return v[np.isfinite(v) & (np.abs(v) < 1e30)]
+
+
+def _roughness(values: np.ndarray) -> Optional[float]:
+    """
+    Median row-to-row step divided by the 5-95% spread of the values.
+
+    Returns
+    -------
+    float or None
+        Small for data that varies smoothly along its rows, large for
+        the same values in a scrambled order. None if there are too few
+        finite values or they are constant.
+    """
+    v = _finite_values(values)
+    if len(v) < 50:
+        return None
+    spread = float(np.percentile(v, 95) - np.percentile(v, 5))
+    if spread == 0.0:
+        return None
+    return float(np.median(np.abs(np.diff(v))) / spread)
+
+
+def _row_order_pick(first: np.ndarray, last: np.ndarray, factor: float = 3.0) -> Optional[int]:
+    """
+    Decide which of two copies of one channel is in acquisition order.
+
+    Parameters
+    ----------
+    first, last : numpy.ndarray
+        The earlier and later copy in the blob chain.
+    factor : float, optional
+        How many times rougher one copy must be than the other.
+
+    Returns
+    -------
+    int or None
+        0 if the *first* copy is the smooth one and the last is a
+        markedly rougher reordering of it; 1 if the last is the smooth
+        one; None if the copies are not a pure reordering of each other
+        (different values or length) or neither is clearly smoother.
+
+    Notes
+    -----
+    A copy that is *itself* perfectly monotone is never judged: a sorted
+    ramp is smoother than any real signal, and a re-sort by a channel's
+    own value (a coordinate re-sorted by itself) leaves exactly that. It
+    could equally be a genuine ID or time channel, and nothing here can
+    tell the two apart, so such a pair is undecided.
+    """
+    if len(first) != len(last):
+        return None
+    a, b = np.asarray(first).ravel(), np.asarray(last).ravel()
+    try:
+        if not np.array_equal(np.sort(a), np.sort(b), equal_nan=True):
+            return None
+    except TypeError:  # dtype without a NaN notion (integers)
+        if not np.array_equal(np.sort(a), np.sort(b)):
+            return None
+    if _is_monotone_reference(a) or _is_monotone_reference(b):
+        return None
+    ra, rb = _roughness(a), _roughness(b)
+    if ra is None or rb is None:
+        return None
+    if rb > 0 and rb >= factor * ra:
+        return 0
+    if ra > 0 and ra >= factor * rb:
+        return 1
+    return None
+
+
+def _is_monotone_reference(values: np.ndarray) -> bool:
+    """A non-constant channel stored in non-decreasing order (an ID, date or time)."""
+    v = _finite_values(values)
+    if len(v) < 50 or v[0] == v[-1]:
+        return False
+    return bool(np.mean(np.diff(v) >= 0) >= 0.999)
+
+
 @dataclass
 class CompressionInfo:
     """
@@ -100,6 +181,20 @@ class GDB:
     ----------
     path : str
         Path to the `.gdb` file.
+    duplicate_blobs : {"last", "row_order"}, optional
+        What to do when a (line, channel) has more than one blob in the
+        blob chain (issue #2). `"last"` (the default) uses the last one
+        in chain order. `"row_order"` is an opt-in **heuristic**, not a
+        decoded field: for a duplicated numeric channel whose two
+        copies hold exactly the same values in a different row order
+        (a stale re-sorted copy), on a line that has an order-defining
+        channel (see Notes), it prefers the copy whose values vary
+        smoothly along the rows -- acquisition order, the order the
+        line's ID/time channels are stored in -- and keeps the last
+        copy in every other case. Either way one `GDBParseWarning`
+        names the affected pairs and, for `"row_order"`, which ones it
+        overrode. Choosing needs both copies decoded, so it is done
+        once, when the blob index is first built.
 
     Raises
     ------
@@ -108,7 +203,8 @@ class GDB:
         expected `.gdb` magic -- unlike the module-level functions in
         `gdb_reader`/`registry` (which warn and return empty results),
         since a `GDB` object that isn't backed by a real `.gdb` file
-        can't usefully do anything at all.
+        can't usefully do anything at all. Also if `duplicate_blobs` is
+        not one of the values above.
 
     Examples
     --------
@@ -137,10 +233,32 @@ class GDB:
     the Rust-plan's M4 notes). Close it (`db.close()`, or use `GDB` as a
     context manager) when done with it, or just let it get
     garbage-collected -- `__del__` closes it too, as a safety net.
+
+    `duplicate_blobs="row_order"` only ever chooses between two copies
+    of one channel that are a pure reordering of each other, and only on
+    a line with an *order-defining channel*: a single-copy numeric
+    channel of the same length stored in monotone order (typically an
+    ID, date or time). A revised copy (different values) is never
+    second-guessed, and neither is a pair with a copy that is itself
+    perfectly monotone (a re-sort by a channel's own value leaves a
+    smooth ramp that looks like the best copy but is the stale one).
+    Among a qualifying pair, the copy at least 3x
+    rougher along the rows -- median row-to-row step over the 5-95%
+    spread of the values -- is treated as the stale one, since data in
+    acquisition order varies smoothly and a re-sort scrambles that. This
+    was validated against independent spreadsheet exports of the one
+    real file known to have such copies (it never contradicted them),
+    but it cannot decide a channel that is smooth in both orders, and no
+    on-disk marker has been found that would make it unnecessary.
     """
 
-    def __init__(self, path: str):
+    def __init__(self, path: str, duplicate_blobs: str = "last"):
+        if duplicate_blobs not in ("last", "row_order"):
+            raise ValueError(
+                f"duplicate_blobs must be 'last' or 'row_order', got {duplicate_blobs!r}"
+            )
         self.path = path
+        self.duplicate_blobs = duplicate_blobs
         self._file = open(path, "rb")
         header = self._file.read(4096)
         if not check_magic(header):
@@ -435,16 +553,207 @@ class GDB:
             anything beyond an occasional one-off lookup -- this class
             always wants line/channel listings and random-access
             reads, so it always builds the index.
+
+        Warns
+        -----
+        GDBParseWarning
+            If a real line's channel has more than one blob in the
+            blob chain. By default the **last** one in chain order is
+            used, but that is not always the current copy (issue #2):
+            an older copy with the same values in a different row order
+            can sit either before or after the current one, and nothing
+            decoded so far says which is which. With
+            `duplicate_blobs="row_order"` the warning also says which
+            pairs were switched to an earlier copy. Duplicates in the
+            administrative slots past the last real line (the REG/IPJ
+            registry, whose stale copies are expected and handled by
+            `pygdb.registry`) are not reported.
         """
         if self._blob_index is None:
             chans_max = self.chans_max
-            index: Dict[Tuple[int, int], BlobHeader] = {}
+            copies: Dict[Tuple[int, int], List[BlobHeader]] = {}
             for blob in iter_blobs(self.path):
-                index[blob.line_channel(chans_max)] = blob
+                copies.setdefault(blob.line_channel(chans_max), []).append(blob)
+            index: Dict[Tuple[int, int], BlobHeader] = {k: v[-1] for k, v in copies.items()}
+            duplicated = [k for k, v in copies.items() if len(v) > 1]
             self._blob_index = index
             self._calibrate_line_indices()
+            if duplicated:
+                overridden: List[Tuple[int, int]] = []
+                if self.duplicate_blobs == "row_order":
+                    overridden = self._prefer_row_order_copies(duplicated, copies, index)
+                self._warn_duplicate_blobs(duplicated, overridden)
         return self._blob_index
 
+    def _read_copy(self, blob: BlobHeader, channel: ChannelRecord) -> np.ndarray:
+        return read_blob_values(
+            self.path, blob, channel,
+            comp_level=self.comp_level or 0, page_size=self.page_size, file=self._file,
+        )
+
+    def _prefer_row_order_copies(
+        self,
+        duplicated: List[Tuple[int, int]],
+        copies: Dict[Tuple[int, int], List[BlobHeader]],
+        index: Dict[Tuple[int, int], BlobHeader],
+    ) -> List[Tuple[int, int]]:
+        """
+        Apply `duplicate_blobs="row_order"` to the duplicated pairs.
+
+        Parameters
+        ----------
+        duplicated : list of (int, int)
+            `(line_slot, channel_slot)` keys with more than one blob.
+        copies : dict of {(int, int) : list of BlobHeader}
+            Every blob for each key, in chain order.
+        index : dict of {(int, int) : BlobHeader}
+            The blob index being built; updated in place with the
+            earlier copy wherever one is preferred.
+
+        Returns
+        -------
+        list of (int, int)
+            The keys switched from the last copy to an earlier one.
+
+        Notes
+        -----
+        See the class docstring for exactly when a pair qualifies. A pair
+        that doesn't (a string or array channel, more than two copies,
+        different values, no order-defining channel on the line, or no
+        clear winner) simply keeps the last copy.
+        """
+        real_lines = {line.index for line in self.lines}
+        channels = {c.index: c for c in self.channels}
+        duplicated_keys = set(duplicated)
+        reference_cache: Dict[Tuple[int, int], bool] = {}
+        overridden: List[Tuple[int, int]] = []
+        for key in duplicated:
+            line_slot, channel_slot = key
+            channel = channels.get(channel_slot)
+            blobs = copies[key]
+            if (
+                line_slot not in real_lines or channel is None or len(blobs) != 2
+                or channel.is_string or channel.is_array
+            ):
+                continue
+            first, last = self._read_copy(blobs[0], channel), self._read_copy(blobs[1], channel)
+            if len(first) != len(last):
+                continue
+            ref_key = (line_slot, len(first))
+            if ref_key not in reference_cache:
+                reference_cache[ref_key] = self._line_has_order_reference(
+                    line_slot, len(first), duplicated_keys, channels
+                )
+            if reference_cache[ref_key] and _row_order_pick(first, last) == 0:
+                index[key] = blobs[0]
+                overridden.append(key)
+        return overridden
+
+    def _line_has_order_reference(
+        self,
+        line_slot: int,
+        n_rows: int,
+        duplicated_keys: set,
+        channels: Dict[int, ChannelRecord],
+    ) -> bool:
+        """
+        Whether a line has a channel proving its rows are in some fixed order.
+
+        Parameters
+        ----------
+        line_slot : int
+            The line's slot.
+        n_rows : int
+            The row count of the duplicated copies being judged.
+        duplicated_keys : set of (int, int)
+            Keys with more than one blob -- excluded, since their own
+            order is what is in question.
+        channels : dict of {int : ChannelRecord}
+            Channel records by slot.
+
+        Returns
+        -------
+        bool
+            True if some single-copy, numeric, non-array channel of
+            `n_rows` values on this line is stored in monotone
+            (non-decreasing) order and isn't constant -- typically an
+            ID, date or time channel.
+        """
+        for (ls, cs), blob in sorted(self._ensure_blob_index().items()):
+            channel = channels.get(cs)
+            if (
+                ls != line_slot or (ls, cs) in duplicated_keys or channel is None
+                or channel.is_string or channel.is_array
+            ):
+                continue
+            values = self._read_copy(blob, channel)
+            if len(values) == n_rows and _is_monotone_reference(values):
+                return True
+        return False
+
+    def _warn_duplicate_blobs(
+        self,
+        duplicated: List[Tuple[int, int]],
+        overridden: Sequence[Tuple[int, int]] = (),
+    ) -> None:
+        """
+        Warn about (line, channel) pairs that have more than one blob.
+
+        Parameters
+        ----------
+        duplicated : list of (int, int)
+            `(line_slot, channel_slot)` keys seen more than once in the
+            blob chain.
+        overridden : sequence of (int, int), optional
+            The keys `duplicate_blobs="row_order"` switched to an
+            earlier copy.
+
+        Warns
+        -----
+        GDBParseWarning
+            Once, naming how many real (line, channel) pairs are
+            affected and a few examples (and, for `"row_order"`, which
+            ones were switched). Nothing is emitted if every duplicate
+            is in an administrative slot.
+        """
+        line_names = {line.index: line.name for line in self.lines}
+        channel_names = {c.index: c.name for c in self.channels}
+
+        def describe(keys, limit=3):
+            named = [
+                f"line {line_names[ls]!r} channel {channel_names.get(cs, f'#{cs}')!r}"
+                for ls, cs in keys if ls in line_names
+            ]
+            more = f" and {len(named) - limit} more" if len(named) > limit else ""
+            return ", ".join(named[:limit]) + more
+
+        n_real = sum(1 for ls, _ in duplicated if ls in line_names)
+        if not n_real:
+            return
+        head = (
+            f"{self.path}: {n_real} (line, channel) pair(s) have more than one "
+            f"blob in the blob chain ({describe(duplicated)})"
+        )
+        if self.duplicate_blobs == "row_order":
+            tail = (
+                f" -- duplicate_blobs='row_order': switched {len(overridden)} pair(s) to "
+                f"an earlier copy because the last copy was the same values in a "
+                f"markedly rougher row order ({describe(overridden)}); kept the last "
+                f"copy in chain order for the other {n_real - len(overridden)}. This "
+                f"is a heuristic, not a decoded field (see issue #2)"
+                if overridden else
+                f" -- duplicate_blobs='row_order': no pair needed switching, so the "
+                f"last copy in chain order was kept for all of them (see issue #2)"
+            )
+        else:
+            tail = (
+                " -- using the last one in chain order, which is not always the "
+                "current copy. If a channel's rows look scrambled against the line's "
+                "other channels, this is the likely cause; duplicate_blobs='row_order' "
+                "can pick the acquisition-order copy (see issue #2)"
+            )
+        warnings.warn(head + tail, GDBParseWarning, stacklevel=4)
+
     def _calibrate_line_indices(self) -> None:
         """
         Correct a possible small, fixed off-by-N in every line's index.

{python_gdb-0.2.0 → python_gdb-0.2.1}/pygdb/gdb_reader.py

@@ -1682,11 +1682,16 @@ def read_blob_values(path: str, blob: BlobHeader, channel: ChannelRecord,
     established ground truth exactly) and for both single- and
     multi-page DB_COMP_SPEED (a real 2-page `Northing_AMGz55` blob
     decodes to sane real coordinates with real `rDUMMY` sentinels).
-    See docs/provenance/notes.md section 6.6d: a multi-page blob is
-    simply one continuous compressed stream spanning the whole
-    `n_pages*page_size` span, not one independently-framed chunk per
-    page -- no special multi-page logic was actually needed once this
-    was verified, just reading the full span instead of one page.
+    See docs/provenance/notes.md section 6.6d: a multi-page blob has no
+    per-page re-framing -- just read the full `n_pages*page_size` span
+    instead of one page. **A DB_COMP_SPEED blob is, however, a chain of
+    chunks of at most 16368 decompressed bytes each, not one chunk**
+    (docs/provenance/notes.md section 6.6e): only the first carries the
+    16-byte magic, later ones are a bare 12-byte sub-header plus
+    payload, and the blob header's `+24` field is the total
+    decompressed size across the chain. Decoding only the first chunk
+    -- as this function once did -- silently truncated any channel
+    longer than 2046 float64 values on a line.
 
     **A real third on-disk variant, auto-detected here rather than
     assumed away (docs/provenance/notes.md section 6.6b):** even
@@ -1769,11 +1774,11 @@ def read_blob_values(path: str, blob: BlobHeader, channel: ChannelRecord,
     # to be a SINGLE continuous compressed stream spanning the whole
     # n_pages*page_size span -- NOT one independently-framed chunk per
     # page. There is no per-page re-framing to handle: reading the full
-    # span and decompressing it as one stream (zlib.decompressobj()
-    # naturally stops at the real end of stream and reports the rest as
-    # padding; the LZRW1 chunk header's own decompressed_length/
-    # chunk_length fields already span the full compressed length
-    # regardless of how many pages it spilled into) is sufficient.
+    # span is sufficient for zlib (zlib.decompressobj() naturally stops
+    # at the real end of stream and reports the rest as padding, and its
+    # single stream always matches the blob header's `+24` total). LZRW1
+    # is different: the span holds a chain of chunks, not one -- see the
+    # DB_COMP_SPEED branch below.
     if page_size is None:
         with _file_handle(path, file) as f:
             header = f.read(128)
@@ -1838,9 +1843,19 @@ def read_blob_values(path: str, blob: BlobHeader, channel: ChannelRecord,
                 )
                 return np.array([])
     elif subtype == _lzrw1.DB_COMP_SPEED:
+        # A blob is a chain of chunks of at most 16368 decompressed bytes
+        # each (docs/spec.md section 7.3); the blob header's `+24` field
+        # is the total across all of them, and is what tells the decoder
+        # where the chain ends (the bytes after the last chunk are page
+        # padding, not a reliable terminator).
+        with _file_handle(path, file) as f:
+            f.seek(blob.offset + 24)
+            total_field = f.read(4)
+        total_decompressed = (
+            struct.unpack("<i", total_field)[0] if len(total_field) == 4 else 0
+        )
         try:
-            chunk = _lzrw1.parse_chunk_header(raw_span, 0)
-            decompressed = _lzrw1.decode_speed_chunk(raw_span, chunk)
+            decompressed = _lzrw1.decode_speed_blob(raw_span, total_decompressed)
         except _lzrw1.LZRW1DecodeError as e:
             _warn(
                 f"blob_index={blob.blob_index}: LZRW1 chunk decode failed ({e}) "

{python_gdb-0.2.0 → python_gdb-0.2.1}/pygdb/lzrw1.py

@@ -15,9 +15,10 @@ reference C wrapper:
   1. No 4-byte FLAG_BYTES prefix (the reference C code's own
      FLAG_COMPRESS/FLAG_COPY byte + 3 padding bytes) -- the control word
      starts immediately for a compressed chunk.
-  2. Each chunk (which may span several of the file's physical
-     1024-byte pages) is preceded by a 28-byte Geosoft-specific wrapper,
-     not part of LZRW1 itself:
+  2. The first chunk of a blob (which may span several of the file's
+     physical 1024-byte pages) is preceded by a 28-byte Geosoft-specific
+     wrapper, not part of LZRW1 itself (see point 4 for what follows
+     it):
        - 16 bytes: the magic sub-header shared with the `.grd` sibling
          format and with `.gdb`'s DB_COMP_SIZE (zlib) mode:
          `0f 0e ff fe  12 34 56 78  <subtype int32>  <reserved int32>`
@@ -49,6 +50,21 @@ reference C wrapper:
          values). Every real marker value found across all 10 real
          Speed files was one of these two constants -- zero exceptions,
          zero unrecognized third values.
+  4. **A blob is a chain of chunks, not one chunk** ([CONFIRMED] on
+     every Speed blob checked -- see docs/spec.md section 7.3/7.4). A
+     chunk decompresses to at most 16368 bytes (2046 float64 values);
+     a channel holding more data than that on one line is split across
+     several chunks stored back to back. Only the *first* chunk of a
+     blob carries the 16-byte magic; each later one is just its own
+     bare 12-byte `<decompressed_length> <chunk_length> <marker>`
+     sub-header immediately followed by its payload, starting
+     `chunk_length` bytes after the previous sub-header began. The
+     blob header (docs/spec.md section 7.4) records the total
+     decompressed size at `+24`, which is how a reader knows when to
+     stop -- the bytes after the last chunk are page padding, not
+     zeros, so they can't be relied on as a terminator. Every chunk
+     decoded independently (LZRW1 back-references never reach across
+     a chunk boundary).  See `decode_speed_blob`.
 
 Validated exactly (not just "plausibly") against **all 10 real**
 DB_COMP_SPEED files now in this project's sample set (the original 4
@@ -72,6 +88,7 @@ from __future__ import annotations
 import struct
 import warnings
 from dataclasses import dataclass
+from typing import List, Optional
 
 try:
     from . import _native as _native_ext
@@ -208,7 +225,8 @@ def _lzrw1_decompress_py(data: bytes, start: int, decompressed_length: int) -> b
 
 @dataclass
 class SpeedChunk:
-    magic_offset: int          # file offset of the 16-byte magic sub-header
+    magic_offset: Optional[int]  # offset of the 16-byte magic sub-header; None for a
+                                 # continuation chunk, which has no magic of its own
     subtype: int                # 1 = DB_COMP_SPEED, 2 = DB_COMP_SIZE
     decompressed_length: int
     chunk_length: int          # includes the 12-byte length sub-header
@@ -361,11 +379,135 @@ def decode_speed_chunk(data: bytes, chunk: SpeedChunk):
         ) from e
 
 
+def decode_speed_blob(data: bytes, total_decompressed_length: int = 0):
+    """
+    Decode every chunk of a DB_COMP_SPEED blob, in order.
+
+    Parameters
+    ----------
+    data : bytes or bytearray
+        The blob's compressed span, starting at the 16-byte magic of its
+        first chunk (i.e. everything after the blob header,
+        docs/spec.md section 7.4).
+    total_decompressed_length : int, optional
+        The blob's total decompressed size in bytes, from its header
+        (`+24`, docs/spec.md section 7.4). Decoding continues chunk by
+        chunk until this many bytes have been produced. If not positive
+        (a header that doesn't carry it, as in some hand-built
+        fixtures), only the first chunk is decoded.
+
+    Returns
+    -------
+    bytearray or bytes
+        The concatenated output of every chunk. A single-chunk blob
+        returns exactly what `decode_speed_chunk` does for it (no extra
+        copy); a multi-chunk one is a new, writable `bytearray`.
+
+    Raises
+    ------
+    LZRW1DecodeError
+        If any chunk fails to decode (see `decode_speed_chunk`), the
+        chain runs off the end of `data`, or the chunks don't add up
+        to exactly `total_decompressed_length`.
+
+    Notes
+    -----
+    A blob is a chain of chunks of at most 16368 decompressed bytes
+    each, not a single chunk (module docstring point 4): only the first
+    carries the 16-byte magic, and every later one is a bare 12-byte
+    sub-header plus payload starting `chunk_length` bytes after the
+    previous sub-header began. This reader used to decode only the first
+    chunk, silently truncating any channel longer than 2046 float64
+    values on a line to exactly that length.
+
+    Dispatches to the compiled `pygdb._native` extension when it's
+    available and `data` is `bytes` (same algorithm, ported to Rust --
+    see `rust/src/lib.rs`), falling back to the pure-Python
+    `_decode_speed_blob_py` below otherwise. The native version raises
+    `ValueError`/`IndexError` for the same conditions; both are turned
+    into `LZRW1DecodeError` here, so callers never see which backend
+    produced a failure.
+    """
+    if _native_ext is not None and isinstance(data, bytes):
+        try:
+            return _native_ext.decode_speed_blob(data, total_decompressed_length)
+        except (ValueError, IndexError) as e:
+            raise LZRW1DecodeError(str(e)) from e
+    return _decode_speed_blob_py(data, total_decompressed_length)
+
+
+def _decode_speed_blob_py(data: bytes, total_decompressed_length: int = 0):
+    """
+    Pure-Python reference implementation of `decode_speed_blob`.
+
+    Parameters
+    ----------
+    data : bytes or bytearray
+        See `decode_speed_blob`.
+    total_decompressed_length : int, optional
+        See `decode_speed_blob`.
+
+    Returns
+    -------
+    bytearray or bytes
+        See `decode_speed_blob`.
+
+    Raises
+    ------
+    LZRW1DecodeError
+        See `decode_speed_blob`.
+    """
+    first = parse_chunk_header(data, 0)
+    out = decode_speed_chunk(data, first)
+    if total_decompressed_length <= len(out):
+        return out
+
+    parts: List[bytes] = [out]
+    produced = len(out)
+    header_start = first.payload_offset - 12  # where this chunk's sub-header began
+    chunk_length = first.chunk_length
+    while produced < total_decompressed_length:
+        if chunk_length < 12:
+            raise LZRW1DecodeError(
+                f"implausible chunk_length={chunk_length} -- corrupt chunk chain"
+            )
+        header_start += chunk_length
+        try:
+            decompressed_length, chunk_length, marker = struct.unpack_from(
+                "<iii", data, header_start
+            )
+        except struct.error as e:
+            raise LZRW1DecodeError(
+                f"chunk chain runs off the end of the data after {produced} of "
+                f"{total_decompressed_length} byte(s) -- truncated data"
+            ) from e
+        chunk = SpeedChunk(
+            magic_offset=None,
+            subtype=DB_COMP_SPEED,
+            decompressed_length=decompressed_length,
+            chunk_length=chunk_length,
+            marker=marker,
+            payload_offset=header_start + 12,
+        )
+        parts.append(decode_speed_chunk(data, chunk))
+        produced += decompressed_length
+
+    if produced != total_decompressed_length:
+        raise LZRW1DecodeError(
+            f"chunks decode to {produced} byte(s) but the blob header declares "
+            f"{total_decompressed_length}"
+        )
+    return bytearray().join(parts)
+
+
 def find_speed_chunks(data: bytes):
     """
     Yield every DB_COMP_SPEED (subtype==1) chunk found in `data`.
 
-    Scans for the shared 16-byte magic byte-by-byte.
+    Scans for the shared 16-byte magic byte-by-byte. Since only the
+    *first* chunk of a blob carries that magic (see `decode_speed_blob`),
+    this finds one chunk per blob, not every chunk -- it's a scanning
+    helper for locating blobs, not a way to decode them.
 
     Parameters
     ----------

{python_gdb-0.2.0 → python_gdb-0.2.1}/rust/Cargo.lock

@@ -46,7 +46,7 @@ dependencies = [
 
 [[package]]
 name = "pygdb-native"
-version = "0.2.0"
+version = "0.2.1"
 dependencies = [
  "flate2",
  "pyo3",

{python_gdb-0.2.0 → python_gdb-0.2.1}/rust/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "pygdb-native"
-version = "0.2.0"
+version = "0.2.1"
 edition = "2021"
 description = "Optional Rust-accelerated backend for python-gdb (pygdb._native)"
 license = "MIT"

{python_gdb-0.2.0 → python_gdb-0.2.1}/rust/src/lib.rs

@@ -148,6 +148,175 @@ fn lzrw1_decompress_impl(data: &[u8], start: usize, out: &mut [u8]) -> PyResult<
     Ok(())
 }
 
+const DB_COMP_SPEED: i32 = 1;
+const MARKER_COMPRESSED: i32 = -186263865; // 0xF4E5D6C7 -- payload is real LZRW1
+const MARKER_STORED_RAW: i32 = -253635901; // 0xF0E1D2C3 -- payload is stored verbatim
+
+/// A chunk's 12-byte `<decompressed_length> <chunk_length> <marker>`
+/// sub-header, read from `data[header_start..]`.
+fn read_chunk_subheader(data: &[u8], header_start: usize) -> PyResult<(i32, i32, i32)> {
+    let bytes = header_start
+        .checked_add(12)
+        .and_then(|end| data.get(header_start..end))
+        .ok_or_else(|| {
+            PyIndexError::new_err(
+                "decode_speed_blob: chunk chain runs off the end of the data -- truncated",
+            )
+        })?;
+    let field = |i: usize| i32::from_le_bytes([bytes[i], bytes[i + 1], bytes[i + 2], bytes[i + 3]]);
+    Ok((field(0), field(4), field(8)))
+}
+
+/// Decode one chunk's payload into `out` (exactly its `decompressed_length`
+/// bytes) -- the Rust counterpart of `pygdb.lzrw1.decode_speed_chunk`,
+/// with the same checks in the same order.
+fn decode_speed_chunk_into(
+    data: &[u8],
+    header_start: usize,
+    chunk_length: i32,
+    marker: i32,
+    out: &mut [u8],
+) -> PyResult<()> {
+    let payload_offset = header_start + 12;
+    match marker {
+        MARKER_STORED_RAW => {
+            if chunk_length as i64 - 12 != out.len() as i64 {
+                return Err(PyValueError::new_err(format!(
+                    "decode_speed_blob: stored-raw chunk should have chunk_length-12 == \
+                     decompressed_length (got chunk_length-12={}, decompressed_length={})",
+                    chunk_length as i64 - 12,
+                    out.len(),
+                )));
+            }
+            let payload = data
+                .get(payload_offset..payload_offset + out.len())
+                .ok_or_else(|| {
+                    PyIndexError::new_err(
+                        "decode_speed_blob: truncated stored-raw payload -- file cut off mid-chunk?",
+                    )
+                })?;
+            out.copy_from_slice(payload);
+            Ok(())
+        }
+        MARKER_COMPRESSED => lzrw1_decompress_impl(data, payload_offset, out),
+        other => Err(PyValueError::new_err(format!(
+            "decode_speed_blob: unrecognized marker value: {other}"
+        ))),
+    }
+}
+
+/// Walk a whole `DB_COMP_SPEED` blob's chain of chunks into `out`.
+///
+/// `out.len()` is either the first chunk's `decompressed_length` (a
+/// single-chunk blob, or no usable total) or the blob's declared total; the
+/// loop stops as soon as `out` is full, and errors if a chunk would
+/// overshoot it -- matching `pygdb.lzrw1._decode_speed_blob_py`.
+fn decode_speed_chain_into(data: &[u8], out: &mut [u8]) -> PyResult<()> {
+    let mut written = 0usize;
+    let mut header_start = 16usize; // just past the first chunk's 16-byte magic
+    loop {
+        let (decompressed_length, chunk_length, marker) = read_chunk_subheader(data, header_start)?;
+        if !(0 < decompressed_length && decompressed_length < 200_000_000) {
+            return Err(PyValueError::new_err(format!(
+                "decode_speed_blob: implausible decompressed_length={decompressed_length} -- \
+                 likely a misaligned or corrupt chunk header"
+            )));
+        }
+        let end = written + decompressed_length as usize;
+        if end > out.len() {
+            return Err(PyValueError::new_err(
+                "decode_speed_blob: chunks decode to more bytes than the blob header declares",
+            ));
+        }
+        decode_speed_chunk_into(
+            data,
+            header_start,
+            chunk_length,
+            marker,
+            &mut out[written..end],
+        )?;
+        written = end;
+        if written == out.len() {
+            return Ok(());
+        }
+        if chunk_length < 12 {
+            return Err(PyValueError::new_err(format!(
+                "decode_speed_blob: implausible chunk_length={chunk_length} -- corrupt chunk chain"
+            )));
+        }
+        header_start += chunk_length as usize;
+    }
+}
+
+/// Decode every chunk of a `DB_COMP_SPEED` blob into one writable Python
+/// `bytearray`.
+///
+/// Rust port of `pygdb.lzrw1.decode_speed_blob` -- see that module's
+/// docstring (point 4) and docs/spec.md section 7.3/7.4 for the format:
+/// a blob is a chain of chunks of at most 16368 decompressed bytes each,
+/// only the first preceded by the 16-byte magic; `data` starts at that
+/// magic, and `total_decompressed_length` is the blob header's `+24`
+/// field (how the decoder knows the chain has ended -- the bytes after
+/// the last chunk are page padding, not zeros). A total that isn't
+/// larger than the first chunk (including 0 or negative, "no usable
+/// total") decodes just the first chunk.
+///
+/// Raises `ValueError` for a malformed chunk or chain (bad marker,
+/// implausible lengths, a total the chain doesn't add up to) and
+/// `IndexError` for truncated data -- `pygdb.lzrw1.decode_speed_blob`
+/// turns both into `LZRW1DecodeError`, so callers never see which
+/// backend produced the failure.
+///
+/// The output buffer is sized once, up front, from the first chunk's
+/// header and the declared total, then filled in place via
+/// `PyByteArray::new_with` under `Python::detach` -- the same single-
+/// allocation technique and the same GIL-release argument as
+/// `lzrw1_decompress` (`data` is `&[u8]`, so an immutable `bytes`; the
+/// target `bytearray` isn't Python-visible until this returns). A
+/// declared total larger than any real chain could produce from `data`
+/// (LZRW1 expands at most 8x, plus the 12-byte sub-headers) is rejected
+/// before allocating, so a corrupt header can't request a huge buffer.
+#[pyfunction]
+fn decode_speed_blob<'py>(
+    py: Python<'py>,
+    data: &[u8],
+    total_decompressed_length: i64,
+) -> PyResult<Bound<'py, PyByteArray>> {
+    let subtype = data
+        .get(8..12)
+        .map(|b| i32::from_le_bytes([b[0], b[1], b[2], b[3]]))
+        .ok_or_else(|| PyIndexError::new_err("decode_speed_blob: truncated -- no chunk header"))?;
+    if subtype != DB_COMP_SPEED {
+        return Err(PyValueError::new_err(format!(
+            "decode_speed_blob: not a Speed chunk (subtype={subtype})"
+        )));
+    }
+    let (first_length, _, _) = read_chunk_subheader(data, 16)?;
+    if !(0 < first_length && first_length < 200_000_000) {
+        return Err(PyValueError::new_err(format!(
+            "decode_speed_blob: implausible decompressed_length={first_length} -- \
+             likely a misaligned or corrupt chunk header"
+        )));
+    }
+    let first_length = first_length as usize;
+    let out_len = if total_decompressed_length > first_length as i64 {
+        let total = total_decompressed_length as u64;
+        if total > (data.len() as u64).saturating_mul(9) {
+            return Err(PyValueError::new_err(format!(
+                "decode_speed_blob: blob header declares {total} decompressed byte(s), \
+                 more than {} byte(s) of chunk data could possibly produce",
+                data.len(),
+            )));
+        }
+        total as usize
+    } else {
+        first_length
+    };
+    PyByteArray::new_with(py, out_len, |buf| {
+        py.detach(|| decode_speed_chain_into(data, buf))
+    })
+}
+
 /// Decompress a `DB_COMP_SIZE` blob's raw zlib/DEFLATE stream into a
 /// writable Python `bytearray`.
 ///
@@ -483,6 +652,7 @@ fn decode_fixed_width_strings_ucs4<'py>(
 fn _native(m: &Bound<'_, PyModule>) -> PyResult<()> {
     m.add_function(wrap_pyfunction!(ping, m)?)?;
     m.add_function(wrap_pyfunction!(lzrw1_decompress, m)?)?;
+    m.add_function(wrap_pyfunction!(decode_speed_blob, m)?)?;
     m.add_function(wrap_pyfunction!(zlib_decompress, m)?)?;
     m.add_function(wrap_pyfunction!(decompress_grd_blocks, m)?)?;
     m.add_function(wrap_pyfunction!(decode_fixed_width_strings_ucs4, m)?)?;