python-gdb 0.2.0__tar.gz → 0.3.0__tar.gz

{python_gdb-0.2.0 → python_gdb-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python-gdb
-Version: 0.2.0
+Version: 0.3.0
 Classifier: Development Status :: 3 - Alpha
 Classifier: Intended Audience :: Science/Research
 Classifier: Topic :: Scientific/Engineering :: GIS
@@ -190,6 +190,7 @@ zensical serve
   boundary on reverse engineering.
 - [`docs/provenance/`](docs/provenance/index.md) — the original
   research log and write-up this implementation was derived from.
+- [`CHANGELOG.md`](CHANGELOG.md) — what changed in each release.
 
 ## Testing
 

{python_gdb-0.2.0 → python_gdb-0.3.0}/README.md

@@ -152,6 +152,7 @@ zensical serve
   boundary on reverse engineering.
 - [`docs/provenance/`](docs/provenance/index.md) — the original
   research log and write-up this implementation was derived from.
+- [`CHANGELOG.md`](CHANGELOG.md) — what changed in each release.
 
 ## Testing
 

{python_gdb-0.2.0 → python_gdb-0.3.0}/pygdb/__init__.py

@@ -16,16 +16,19 @@ Reading only: writing/mutating `.gdb` or `.grd` files is out of scope.
 
 from .gdb import GDB, CompressionInfo
 from .gdb_reader import (
+    BlobDirectory,
     BlobHeader,
     ChannelRecord,
     GDBParseWarning,
     LineRecord,
     check_magic,
+    exact_line_table_start,
     find_blob,
     find_channel_table,
     find_line_table,
     header_fields,
     iter_blobs,
+    read_blob_directory,
     read_blob_values,
     read_channels,
     read_lines,
@@ -36,6 +39,7 @@ from .registry import find_coordinate_systems
 
 __all__ = [
     "GDB",
+    "BlobDirectory",
     "BlobHeader",
     "ChannelRecord",
     "CompressionInfo",
@@ -45,12 +49,14 @@ __all__ = [
     "LZRW1DecodeError",
     "LineRecord",
     "check_magic",
+    "exact_line_table_start",
     "find_blob",
     "find_channel_table",
     "find_coordinate_systems",
     "find_line_table",
     "header_fields",
     "iter_blobs",
+    "read_blob_directory",
     "read_blob_values",
     "read_channels",
     "read_grd",

{python_gdb-0.2.0 → python_gdb-0.3.0}/pygdb/gdb.py

@@ -16,7 +16,7 @@ from __future__ import annotations
 import warnings
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Dict, Iterator, List, Optional, Tuple, Union
+from typing import Dict, Iterator, List, Optional, Sequence, Tuple, Union
 
 import numpy as np
 
@@ -30,9 +30,13 @@ from .gdb_reader import (
     check_magic,
     header_fields,
     iter_blobs,
+    read_blob_directory,
     read_blob_values,
     read_channels,
-    read_lines,
+    _read_lines,
+    DIRECTORY_ABSENT,
+    DIRECTORY_INVALID,
+    DIRECTORY_LIVE,
 )
 from .registry import find_channel_roles, find_coordinate_systems
 
@@ -100,6 +104,15 @@ class GDB:
     ----------
     path : str
         Path to the `.gdb` file.
+    include_unlisted_blobs : bool, optional
+        A file that carries a blob directory (see Notes) lists exactly
+        one live blob per (line, channel). By default a blob the
+        directory does not list -- an all-zero slot -- is **skipped**, as
+        if it were absent, and one `GDBParseWarning` says how many
+        (line, channel) blobs in which channels were skipped. Pass
+        `True` to read them anyway (the last copy in chain order). A
+        file with no directory, or an all-zero one, is unaffected: every
+        blob in the chain is served.
 
     Raises
     ------
@@ -137,10 +150,23 @@ class GDB:
     the Rust-plan's M4 notes). Close it (`db.close()`, or use `GDB` as a
     context manager) when done with it, or just let it get
     garbage-collected -- `__del__` closes it too, as a safety net.
+
+    **Which copy of a blob is read.** The blob chain is append-only, so
+    a (line, channel) can have several blobs, an older stale one and
+    the current one, in either order (issue #2). The file's persisted
+    *blob directory* (docs/spec.md section 2.2) says which is current,
+    and it is the only thing consulted. A directory entry is used only
+    if its start page lands on a walked blob header whose `blob_index`
+    equals the slot and whose page count matches; if a non-zero entry
+    fails that check, the last copy in chain order is used and a
+    `GDBParseWarning` says so. Without a usable directory the last copy
+    in chain order is used, with a warning if a pair is duplicated: that
+    is a guess, and not always the current copy.
     """
 
-    def __init__(self, path: str):
+    def __init__(self, path: str, include_unlisted_blobs: bool = False):
         self.path = path
+        self.include_unlisted_blobs = include_unlisted_blobs
         self._file = open(path, "rb")
         header = self._file.read(4096)
         if not check_magic(header):
@@ -152,6 +178,7 @@ class GDB:
         self._fields = header_fields(header)
         self._channels: Optional[List[ChannelRecord]] = None
         self._lines: Optional[List[LineRecord]] = None
+        self._lines_exact = False
         self._channels_by_name: Optional[Dict[str, List[ChannelRecord]]] = None
         self._lines_by_name: Optional[Dict[str, List[LineRecord]]] = None
         self._blob_index: Optional[Dict[Tuple[int, int], BlobHeader]] = None
@@ -267,7 +294,7 @@ class GDB:
     def lines(self) -> List[LineRecord]:
         """list of LineRecord: This file's line table, read once and cached."""
         if self._lines is None:
-            self._lines = read_lines(self.path)
+            self._lines, self._lines_exact = _read_lines(self.path)
             self._lines_by_name = {}
             for l in self._lines:
                 self._lines_by_name.setdefault(l.name, []).append(l)
@@ -429,28 +456,144 @@ class GDB:
         Returns
         -------
         dict of {(int, int) : BlobHeader}
-            Maps `(line_slot, channel_slot)` to `BlobHeader`, built
-            with one blob-chain walk and cached from then on.
-            `iter_blobs`/`find_blob` themselves recommend this for
-            anything beyond an occasional one-off lookup -- this class
-            always wants line/channel listings and random-access
+            Maps `(line_slot, channel_slot)` to the `BlobHeader` of its
+            live blob, built with one blob-chain walk and cached from
+            then on. `iter_blobs`/`find_blob` themselves recommend this
+            for anything beyond an occasional one-off lookup -- this
+            class always wants line/channel listings and random-access
             reads, so it always builds the index.
+
+        Warns
+        -----
+        GDBParseWarning
+            See `_select_live_blobs`: when blobs are skipped because the
+            file's blob directory does not list them, when a directory
+            entry fails validation, or when a pair is duplicated and
+            the file has no directory to say which copy is current.
         """
         if self._blob_index is None:
             chans_max = self.chans_max
-            index: Dict[Tuple[int, int], BlobHeader] = {}
+            copies: Dict[Tuple[int, int], List[BlobHeader]] = {}
+            by_offset: Dict[int, BlobHeader] = {}
             for blob in iter_blobs(self.path):
-                index[blob.line_channel(chans_max)] = blob
-            self._blob_index = index
-            self._calibrate_line_indices()
+                copies.setdefault(blob.line_channel(chans_max), []).append(blob)
+                by_offset[blob.offset] = blob
+            self._blob_index = {k: v[-1] for k, v in copies.items()}
+            if not self.lines:
+                return self._blob_index
+            if not self._lines_exact:
+                self._calibrate_line_indices()
+            self._select_live_blobs(copies, by_offset)
         return self._blob_index
 
+    def _select_live_blobs(
+        self,
+        copies: Dict[Tuple[int, int], List[BlobHeader]],
+        by_offset: Dict[int, BlobHeader],
+    ) -> None:
+        """
+        Choose each real (line, channel)'s live blob using the blob directory.
+
+        Parameters
+        ----------
+        copies : dict of {(int, int) : list of BlobHeader}
+            Every blob for each key, in chain order.
+        by_offset : dict of {int : BlobHeader}
+            Every blob of the chain, keyed by absolute offset.
+
+        Warns
+        -----
+        GDBParseWarning
+            Once per situation, naming counts and examples: (1) pairs
+            skipped because the directory has no live entry for them;
+            (2) pairs whose non-zero directory entry failed validation
+            (the last copy in chain order is used); (3) with no usable
+            directory at all, pairs that have more than one blob.
+            Administrative slots past the last real line (the REG/IPJ
+            registry, whose stale copies `pygdb.registry` handles
+            itself) are never judged or reported.
+
+        Notes
+        -----
+        Updates `self._blob_index` in place. See the class docstring for
+        the rules.
+        """
+        index = self._blob_index
+        real_lines = {line.index for line in self.lines}
+        directory = read_blob_directory(self.path)
+        if directory is None:
+            duplicated = [k for k, v in copies.items() if len(v) > 1 and k[0] in real_lines]
+            if duplicated:
+                self._warn_pairs(
+                    f"{len(duplicated)} (line, channel) pair(s) have more than one blob in the "
+                    f"blob chain ({self._describe_pairs(duplicated)}) and the file has no blob "
+                    f"directory to say which is current -- using the last one in chain order, "
+                    f"which is not always the current copy. If a channel's rows look scrambled "
+                    f"against the line's other channels, this is the likely cause (see issue #2)"
+                )
+            return
+        first_offset = min(by_offset)
+        skipped: List[Tuple[int, int]] = []
+        invalid: List[Tuple[int, int]] = []
+        for key in copies:
+            line_slot, channel_slot = key
+            if line_slot not in real_lines:
+                continue
+            status, blob = directory.resolve(
+                line_slot * self.chans_max + channel_slot, by_offset, first_offset, self.page_size,
+            )
+            if status == DIRECTORY_LIVE:
+                index[key] = blob
+            elif status == DIRECTORY_ABSENT:
+                skipped.append(key)
+            elif status == DIRECTORY_INVALID:
+                invalid.append(key)
+        if skipped and not self.include_unlisted_blobs:
+            for key in skipped:
+                del index[key]
+            channel_names = {c.index: c.name for c in self.channels}
+            by_channel: Dict[int, int] = {}
+            for _, channel_slot in skipped:
+                by_channel[channel_slot] = by_channel.get(channel_slot, 0) + 1
+            listing = ", ".join(
+                f"{channel_names.get(cs, f'#{cs}')!r} ({n})" for cs, n in sorted(by_channel.items())[:5]
+            )
+            more = f" and {len(by_channel) - 5} more channel(s)" if len(by_channel) > 5 else ""
+            self._warn_pairs(
+                f"skipped {len(skipped)} (line, channel) blob(s) in {len(by_channel)} channel(s) "
+                f"[{listing}{more}] that the file's blob directory does not list as live. Pass "
+                f"include_unlisted_blobs=True to read them anyway"
+            )
+        if invalid:
+            self._warn_pairs(
+                f"{len(invalid)} (line, channel) pair(s) have a blob-directory entry that does not "
+                f"point at a blob with the right index and size ({self._describe_pairs(invalid)}) "
+                f"-- using the last copy in chain order for them"
+            )
+
+    def _describe_pairs(self, keys: Sequence[Tuple[int, int]], limit: int = 3) -> str:
+        line_names = {line.index: line.name for line in self.lines}
+        channel_names = {c.index: c.name for c in self.channels}
+        named = [
+            f"line {line_names[ls]!r} channel {channel_names.get(cs, f'#{cs}')!r}"
+            for ls, cs in keys if ls in line_names
+        ]
+        more = f" and {len(named) - limit} more" if len(named) > limit else ""
+        return ", ".join(named[:limit]) + more
+
+    def _warn_pairs(self, message: str) -> None:
+        # stacklevel: here -> _select_live_blobs -> _ensure_blob_index -> public method -> caller
+        warnings.warn(f"{self.path}: {message}", GDBParseWarning, stacklevel=5)
+
     def _calibrate_line_indices(self) -> None:
         """
         Correct a possible small, fixed off-by-N in every line's index.
 
-        See `find_line_table`'s and `read_lines`'s docstrings in
-        `gdb_reader.py`. Checks, for a handful of small integer shifts,
+        Only used when `read_lines` had to fall back to the
+        `find_line_table` heuristic (`self._lines_exact` is False): a
+        file whose line table is located exactly already has true
+        indices. See `find_line_table`'s and `read_lines`'s docstrings
+        in `gdb_reader.py`. Checks, for a handful of small integer shifts,
         which one makes the most already-found lines actually have at
         least one real data blob on disk for *some* channel -- then
         applies the winning shift to every `LineRecord.index` in

{python_gdb-0.2.0 → python_gdb-0.3.0}/pygdb/gdb_reader.py

@@ -443,11 +443,15 @@ def header_fields(data: bytes) -> dict:
     Returns
     -------
     dict
-        Maps each of `"chans_max"`, `"users_max"`, `"page_size"`, and
-        `"comp_level"` to its decoded int32 value. See
-        docs/provenance/notes.md section 6.1 for the full table
-        including the still-unknown offsets, and for why each
-        confidence label was assigned.
+        Maps each of `"chans_max"` (word 24), `"blobs_max"` (28),
+        `"lines_max"` (36), `"users_max"` (40), `"index_slots"` (44,
+        the total number of blob-directory slots), `"data_slots"` (48,
+        the number of those that address (line, channel) data blobs),
+        `"page_size"` (100), and `"comp_level"` (120) to its decoded
+        int32 value. See docs/spec.md section 2 for the full table
+        with a confidence rating per word, and
+        docs/provenance/notes.md section 6.1/6.1b for the derivations
+        and the still-unknown offsets.
 
     Warns
     -----
@@ -466,7 +470,8 @@ def header_fields(data: bytes) -> dict:
     (`DB_COMP_SIZE`) IS confirmed real zlib.
     """
     result = {}
-    for name, offset in (("chans_max", 24), ("users_max", 40),
+    for name, offset in (("chans_max", 24), ("blobs_max", 28), ("lines_max", 36),
+                          ("users_max", 40), ("index_slots", 44), ("data_slots", 48),
                           ("page_size", 100), ("comp_level", 120)):
         try:
             result[name] = struct.unpack_from("<i", data, offset)[0]
@@ -713,10 +718,10 @@ class LineRecord:
     -----
     **[LIKELY]/[UNKNOWN]** -- much less firmly established than
     `ChannelRecord`: only the name (relative +32) and category code
-    (relative +108) fields are decoded, and locating the table itself
-    (`find_line_table` below) is a heuristic scan rather than the
-    structurally-proven SUPER-anchor technique used for the channel
-    table. See docs/spec.md section 3.2 and docs/provenance/notes.md
+    (relative +108) fields are decoded. The table is located exactly
+    (`exact_line_table_start`, **[CONFIRMED]** on the corpus) and only
+    falls back to the `find_line_table` heuristic scan when that
+    cannot be validated. See docs/spec.md section 3.2 and docs/provenance/notes.md
     section 6.3.
 
     `eq=False` keeps the default identity-based `__eq__`/`__hash__`
@@ -785,23 +790,25 @@ def find_line_table(data: bytes, search_window: Tuple[int, Optional[int]] = (128
 
     Notes
     -----
+    **Prefer `exact_line_table_start`** (via `read_lines`): the line
+    table's start follows exactly from `lines_max` and the channel
+    table's position (docs/spec.md section 2.1), which this heuristic
+    predates. This scan is now only the fallback for a file where that
+    arithmetic cannot be validated.
+
     Unlike `find_channel_table`, there's no known default-name anchor
     (the line table has nothing analogous to the channel table's
-    "SUPER" user record immediately after it) and no confirmed header
-    field gives its start offset directly -- reconciling one with the
-    header's capacity fields was tried and didn't cleanly round-trip
-    (docs/provenance/log.md Session 1 section 1.16,
-    docs/provenance/notes.md section 6.3). This is therefore a
-    heuristic **[LIKELY]** scan, not the structurally-proven technique
-    used for the channel table: it looks for a run of 128-byte records
+    "SUPER" user record immediately after it), so this is a
+    heuristic **[LIKELY]** scan, not a structurally-proven technique:
+    it looks for a run of 128-byte records
     whose relative +32 field looks like a clean, NUL-terminated,
     printable line name and whose relative +108 category field
     matches one of the two confirmed real values (100=NORMAL/FLIGHT,
     200=GROUP), then returns the earliest such record in the run with
     the most hits at a consistent 128-byte phase.
 
-    **Known limitation, found by real-file testing, not yet fixed
-    here:** if a table's true first slot(s) don't carry a category
+    **Known limitation, found by real-file testing (fixed by using
+    `exact_line_table_start` instead, not here):** if a table's true first slot(s) don't carry a category
     code in {100, 200}, this returns a start that's one or more slots
     too late -- every subsequent `LineRecord.index` is then off by
     that same fixed amount, which breaks blob_index lookups by line
@@ -847,6 +854,60 @@ def find_line_table(data: bytes, search_window: Tuple[int, Optional[int]] = (128
     return min(phase_hits[best_phase])
 
 
+_LINE_TABLE_GAP = 24  # [CONFIRMED] bytes between the end of the line table and the
+                      # channel table -- docs/spec.md section 2.1
+
+
+def exact_line_table_start(data: bytes, lines_max: Optional[int]) -> Optional[int]:
+    """
+    Compute the line table's start from the header and the channel table.
+
+    Parameters
+    ----------
+    data : bytes
+        The file's bytes up to at least the end of the channel table
+        (`read_lines` passes everything before the blob region).
+    lines_max : int or None
+        The line-table capacity (header word 36); `None` or non-positive
+        means unknown.
+
+    Returns
+    -------
+    int or None
+        Byte offset of the line table's first record, or `None` if the
+        arithmetic cannot be validated (unknown `lines_max`, no
+        locatable channel table, a start before the header ends, or no
+        line-shaped record anywhere in the computed table) -- the
+        caller then falls back to `find_line_table`.
+
+    Notes
+    -----
+    The line table is `lines_max` 128-byte slots followed by a 24-byte
+    gap and then the channel table, so its start is
+    `find_channel_table(data) - 24 - lines_max * 128`. **[CONFIRMED]**
+    on all 23 real files examined (docs/spec.md section 2.1;
+    docs/provenance/notes.md section 6.1b): the computed start is
+    always a slot boundary that matches the heuristic's start, or is an
+    earlier slot the heuristic missed. The at-least-one-line-record
+    check only guards a file whose layout does not follow this
+    arithmetic; it is not needed for any real file seen.
+    """
+    if not lines_max or lines_max <= 0:
+        return None
+    try:
+        channel_start = find_channel_table(data)
+    except ValueError:
+        return None
+    start = channel_start - _LINE_TABLE_GAP - lines_max * SYMBOL_RECORD_SIZE
+    if start < 256:
+        return None
+    for i in range(lines_max):
+        rec = _parse_line_record(data, start + i * SYMBOL_RECORD_SIZE, i)
+        if rec.name and rec.name_is_clean and rec.category_code in DB_CATEGORY_LINE_NAMES:
+            return start
+    return None
+
+
 def read_lines(path: str) -> List[LineRecord]:
     """
     Decode the line symbol table.
@@ -869,30 +930,59 @@ def read_lines(path: str) -> List[LineRecord]:
 
     Notes
     -----
-    Heuristic (see `find_line_table`) -- less firmly established than
-    `read_channels`. Since no confirmed header field gives the line
-    table's slot capacity (the way `chans_max` does for the channel
-    table), this reads forward from the located start until 8
+    The table is located **exactly** whenever possible (see
+    `exact_line_table_start`): it holds `lines_max` (header word 36)
+    128-byte slots and ends 24 bytes before the channel table, so its
+    start is `channel_table_start - 24 - lines_max * 128`
+    (**[CONFIRMED]** on every real file in the corpus, docs/spec.md
+    section 2.1). Every one of the `lines_max` slots is then examined,
+    and `LineRecord.index` is the true slot number, so
+    `LineRecord.index` is exact and blob_index lookups keyed on it
+    are right.
+
+    Only when that arithmetic cannot be validated (a header without
+    `lines_max`, or no line-shaped record at the computed position) does
+    this fall back to the older heuristic (see `find_line_table`):
+    **[LIKELY]**, less firmly established, reading forward until 8
     consecutive records fail to look like either a populated line
-    record or clean unused capacity -- a tolerance against one-off
-    corruption/false-positive records, not a precisely-known table
-    boundary.
-
-    **`LineRecord.index` can be off by a small, fixed amount** on a
-    file where `find_line_table`'s heuristic starts one or more slots
-    late -- see that function's docstring. This makes `.name` still
-    correct but `.index` (and therefore any blob_index lookup keyed on
-    it) wrong. `GDB` (in `gdb.py`) corrects this against the actual
-    blob chain before exposing lines by name; call it instead of this
-    function directly when you need working (line, channel) data
-    access, not just a list of names.
+    record or clean unused capacity. In that fallback
+    **`LineRecord.index` can be off by a small, fixed amount** when the
+    scan starts one or more slots late -- `.name` is still correct but
+    `.index` is wrong. `GDB` (in `gdb.py`) corrects this against the
+    actual blob chain only in that case.
+
+    A populated slot whose category is neither `100` nor `200` (for
+    example the `65636` slot 0 of one 1991 file) is not returned, as
+    before -- but, unlike the fallback, it no longer shifts the
+    indices of the lines after it.
+    """
+    return _read_lines(path)[0]
+
+
+def _read_lines(path: str) -> Tuple[List[LineRecord], bool]:
+    """
+    Decode the line symbol table, also reporting how it was located.
+
+    Parameters
+    ----------
+    path : str
+        Path to the `.gdb` file.
+
+    Returns
+    -------
+    lines : list of LineRecord
+        As `read_lines`.
+    exact : bool
+        True if the table was located by `exact_line_table_start`
+        (indices are exact); False if the heuristic fallback was used
+        (indices may need `GDB`'s blob-chain calibration).
     """
     with open(path, "rb") as f:
         header = f.read(4096)
         if not check_magic(header):
             _warn(f"{path}: does not start with the expected '!CBD' magic -- "
                   f"not a recognized .gdb file, returning no lines")
-            return []
+            return [], False
         blob_start = blob_region_start(header)
         f.seek(0, 2)
         size = f.tell()
@@ -900,14 +990,24 @@ def read_lines(path: str) -> List[LineRecord]:
         read_size = blob_start if (blob_start is not None and 0 < blob_start <= size) else min(size, 20_000_000)
         data = f.read(read_size)
 
+    lines_max = header_fields(header).get("lines_max")
+    exact_start = exact_line_table_start(data, lines_max)
+    if exact_start is not None:
+        lines = []
+        for i in range(lines_max):
+            rec = _parse_line_record(data, exact_start + i * SYMBOL_RECORD_SIZE, i)
+            if rec.name and rec.name_is_clean and rec.category_code in DB_CATEGORY_LINE_NAMES:
+                lines.append(rec)
+        return lines, True
+
     try:
         table_start = find_line_table(data, search_window=(128, len(data)))
     except ValueError as e:
         _warn(f"{path}: could not locate the line symbol table ({e}) -- "
               f"returning no lines")
-        return []
+        return [], False
 
-    lines: List[LineRecord] = []
+    lines = []
     consecutive_bad = 0
     i = 0
     while True:
@@ -929,7 +1029,7 @@ def read_lines(path: str) -> List[LineRecord]:
             consecutive_bad += 1
             if consecutive_bad >= 8:
                 break
-    return lines
+    return lines, False
 
 
 BLOB_MAGIC = b"\xcc\xcc\x00\xff"
@@ -1301,6 +1401,145 @@ def find_blob(path: str, line_slot: int, channel_slot: int, chans_max: Optional[
     return None
 
 
+DIRECTORY_OFFSET = 280  # [CONFIRMED] first blob-directory slot -- docs/spec.md section 2.2
+_DIRECTORY_SLOT_DTYPE = np.dtype([("word", "<u4"), ("n_pages", "<u2")])  # 6 bytes, unaligned
+_DIRECTORY_LIVE_FLAG = 0x8  # top nibble of a live (line, channel) entry's 32-bit word
+
+DIRECTORY_LIVE = "live"
+DIRECTORY_ABSENT = "absent"
+DIRECTORY_INVALID = "invalid"
+DIRECTORY_OUTSIDE = "outside"
+
+
+@dataclass(eq=False)
+class BlobDirectory:
+    """
+    The persisted blob directory: which blob is the live one for each slot.
+
+    Attributes
+    ----------
+    data_slots : int
+        Number of directory slots that address (line, channel) data
+        blobs (header word 48, `lines_max * chans_max`); slot `i` is
+        the blob whose `blob_index` is `i`.
+    entries : dict of {int : (int, int)}
+        Every **non-zero** data slot as `(32-bit word, n_pages)`, keyed
+        by slot. A zero slot is simply absent from this dict.
+
+    Notes
+    -----
+    **[CONFIRMED]** layout, **[LIKELY]** interpretation
+    (docs/spec.md section 2.2; docs/provenance/notes.md section 6.1c).
+    The directory is an array of 6-byte slots starting at file offset
+    280. A live entry is `(0x80000000 | start page, n_pages)` where
+    the start page is relative to the first blob (`(offset - blob
+    region start) / page_size`). Across the real corpus it addressed
+    100% of the real (line, channel) blobs of 20 of 22 files, and for
+    every duplicated pair with an independent oracle (13 of 13) it
+    pointed at the correct copy. A slot that is all-zero belongs to a
+    blob the file does not list as live.
+    """
+
+    data_slots: int
+    entries: dict
+
+    def resolve(self, blob_index: int, blobs_by_offset: dict, first_offset: int, page_size: int):
+        """
+        Look up the live blob for `blob_index`.
+
+        Parameters
+        ----------
+        blob_index : int
+            The slot, `line_slot * chans_max + channel_slot`.
+        blobs_by_offset : dict of {int : BlobHeader}
+            Every blob of the chain walk, keyed by absolute `offset`.
+        first_offset : int
+            Absolute offset of the first blob (`blob_region_start`).
+        page_size : int
+            The file's page size.
+
+        Returns
+        -------
+        status : str
+            `"live"`: the entry's start page lands on a walked blob
+            header whose `blob_index` equals `blob_index` and whose
+            `n_pages` equals the entry's page count (the strict
+            check). `"absent"`: the slot is all-zero. `"invalid"`: the
+            slot is non-zero but fails the strict check. `"outside"`:
+            `blob_index` is not a data slot at all (an administrative
+            slot), so the directory says nothing about it.
+        blob : BlobHeader or None
+            The live blob for `"live"`, otherwise `None`.
+        """
+        if not 0 <= blob_index < self.data_slots:
+            return DIRECTORY_OUTSIDE, None
+        entry = self.entries.get(blob_index)
+        if entry is None:
+            return DIRECTORY_ABSENT, None
+        word, n_pages = entry
+        if word >> 28 != _DIRECTORY_LIVE_FLAG:
+            return DIRECTORY_INVALID, None
+        blob = blobs_by_offset.get(first_offset + (word & 0x7FFFFFFF) * page_size)
+        if blob is None or blob.blob_index != blob_index or blob.n_pages != n_pages:
+            return DIRECTORY_INVALID, None
+        return DIRECTORY_LIVE, blob
+
+
+def read_blob_directory(path: str) -> Optional[BlobDirectory]:
+    """
+    Read the persisted blob directory.
+
+    Parameters
+    ----------
+    path : str
+        Path to the `.gdb` file.
+
+    Returns
+    -------
+    BlobDirectory or None
+        The directory, or `None` when the file has none to trust: a bad
+        magic or truncated header, header words that are inconsistent
+        with the layout (`data_slots != lines_max * chans_max`, or the
+        data slots would run past the blob region), or a directory
+        whose every data slot is zero. Absence is not an anomaly, so
+        nothing is warned.
+
+    Notes
+    -----
+    See `BlobDirectory`. Only the data slots (`0 <= slot <
+    data_slots`) are read; the registry blob-symbol slots and the
+    cache slots after them are not used by the reader.
+    """
+    with open(path, "rb") as f:
+        header = f.read(4096)
+        if not check_magic(header):
+            return None
+        fields = header_fields(header)
+        chans_max, lines_max, data_slots, page_size = (
+            fields["chans_max"], fields["lines_max"], fields["data_slots"], fields["page_size"],
+        )
+        if None in (chans_max, lines_max, data_slots, page_size):
+            return None
+        blob_start = blob_region_start(header)
+        if (
+            blob_start is None or data_slots <= 0 or data_slots != lines_max * chans_max
+            or DIRECTORY_OFFSET + data_slots * _DIRECTORY_SLOT_DTYPE.itemsize > blob_start
+        ):
+            return None
+        f.seek(DIRECTORY_OFFSET)
+        raw = f.read(data_slots * _DIRECTORY_SLOT_DTYPE.itemsize)
+    if len(raw) != data_slots * _DIRECTORY_SLOT_DTYPE.itemsize:
+        return None
+    slots = np.frombuffer(raw, dtype=_DIRECTORY_SLOT_DTYPE)
+    nonzero = np.nonzero((slots["word"] != 0) | (slots["n_pages"] != 0))[0]
+    if len(nonzero) == 0:
+        return None
+    entries = {
+        int(i): (int(slots["word"][i]), int(slots["n_pages"][i])) for i in nonzero
+    }
+    return BlobDirectory(data_slots=data_slots, entries=entries)
+
+
 def _element_width(channel: ChannelRecord) -> Optional[int]:
     """
     Byte width of one element of `channel`'s data.
@@ -1682,11 +1921,16 @@ def read_blob_values(path: str, blob: BlobHeader, channel: ChannelRecord,
     established ground truth exactly) and for both single- and
     multi-page DB_COMP_SPEED (a real 2-page `Northing_AMGz55` blob
     decodes to sane real coordinates with real `rDUMMY` sentinels).
-    See docs/provenance/notes.md section 6.6d: a multi-page blob is
-    simply one continuous compressed stream spanning the whole
-    `n_pages*page_size` span, not one independently-framed chunk per
-    page -- no special multi-page logic was actually needed once this
-    was verified, just reading the full span instead of one page.
+    See docs/provenance/notes.md section 6.6d: a multi-page blob has no
+    per-page re-framing -- just read the full `n_pages*page_size` span
+    instead of one page. **A DB_COMP_SPEED blob is, however, a chain of
+    chunks of at most 16368 decompressed bytes each, not one chunk**
+    (docs/provenance/notes.md section 6.6e): only the first carries the
+    16-byte magic, later ones are a bare 12-byte sub-header plus
+    payload, and the blob header's `+24` field is the total
+    decompressed size across the chain. Decoding only the first chunk
+    -- as this function once did -- silently truncated any channel
+    longer than 2046 float64 values on a line.
 
     **A real third on-disk variant, auto-detected here rather than
     assumed away (docs/provenance/notes.md section 6.6b):** even
@@ -1769,11 +2013,11 @@ def read_blob_values(path: str, blob: BlobHeader, channel: ChannelRecord,
     # to be a SINGLE continuous compressed stream spanning the whole
     # n_pages*page_size span -- NOT one independently-framed chunk per
     # page. There is no per-page re-framing to handle: reading the full
-    # span and decompressing it as one stream (zlib.decompressobj()
-    # naturally stops at the real end of stream and reports the rest as
-    # padding; the LZRW1 chunk header's own decompressed_length/
-    # chunk_length fields already span the full compressed length
-    # regardless of how many pages it spilled into) is sufficient.
+    # span is sufficient for zlib (zlib.decompressobj() naturally stops
+    # at the real end of stream and reports the rest as padding, and its
+    # single stream always matches the blob header's `+24` total). LZRW1
+    # is different: the span holds a chain of chunks, not one -- see the
+    # DB_COMP_SPEED branch below.
     if page_size is None:
         with _file_handle(path, file) as f:
             header = f.read(128)
@@ -1838,9 +2082,19 @@ def read_blob_values(path: str, blob: BlobHeader, channel: ChannelRecord,
                 )
                 return np.array([])
     elif subtype == _lzrw1.DB_COMP_SPEED:
+        # A blob is a chain of chunks of at most 16368 decompressed bytes
+        # each (docs/spec.md section 7.3); the blob header's `+24` field
+        # is the total across all of them, and is what tells the decoder
+        # where the chain ends (the bytes after the last chunk are page
+        # padding, not a reliable terminator).
+        with _file_handle(path, file) as f:
+            f.seek(blob.offset + 24)
+            total_field = f.read(4)
+        total_decompressed = (
+            struct.unpack("<i", total_field)[0] if len(total_field) == 4 else 0
+        )
         try:
-            chunk = _lzrw1.parse_chunk_header(raw_span, 0)
-            decompressed = _lzrw1.decode_speed_chunk(raw_span, chunk)
+            decompressed = _lzrw1.decode_speed_blob(raw_span, total_decompressed)
         except _lzrw1.LZRW1DecodeError as e:
             _warn(
                 f"blob_index={blob.blob_index}: LZRW1 chunk decode failed ({e}) "

{python_gdb-0.2.0 → python_gdb-0.3.0}/pygdb/lzrw1.py

@@ -15,9 +15,10 @@ reference C wrapper:
   1. No 4-byte FLAG_BYTES prefix (the reference C code's own
      FLAG_COMPRESS/FLAG_COPY byte + 3 padding bytes) -- the control word
      starts immediately for a compressed chunk.
-  2. Each chunk (which may span several of the file's physical
-     1024-byte pages) is preceded by a 28-byte Geosoft-specific wrapper,
-     not part of LZRW1 itself:
+  2. The first chunk of a blob (which may span several of the file's
+     physical 1024-byte pages) is preceded by a 28-byte Geosoft-specific
+     wrapper, not part of LZRW1 itself (see point 4 for what follows
+     it):
        - 16 bytes: the magic sub-header shared with the `.grd` sibling
          format and with `.gdb`'s DB_COMP_SIZE (zlib) mode:
          `0f 0e ff fe  12 34 56 78  <subtype int32>  <reserved int32>`
@@ -49,6 +50,21 @@ reference C wrapper:
          values). Every real marker value found across all 10 real
          Speed files was one of these two constants -- zero exceptions,
          zero unrecognized third values.
+  4. **A blob is a chain of chunks, not one chunk** ([CONFIRMED] on
+     every Speed blob checked -- see docs/spec.md section 7.3/7.4). A
+     chunk decompresses to at most 16368 bytes (2046 float64 values);
+     a channel holding more data than that on one line is split across
+     several chunks stored back to back. Only the *first* chunk of a
+     blob carries the 16-byte magic; each later one is just its own
+     bare 12-byte `<decompressed_length> <chunk_length> <marker>`
+     sub-header immediately followed by its payload, starting
+     `chunk_length` bytes after the previous sub-header began. The
+     blob header (docs/spec.md section 7.4) records the total
+     decompressed size at `+24`, which is how a reader knows when to
+     stop -- the bytes after the last chunk are page padding, not
+     zeros, so they can't be relied on as a terminator. Every chunk
+     decoded independently (LZRW1 back-references never reach across
+     a chunk boundary).  See `decode_speed_blob`.
 
 Validated exactly (not just "plausibly") against **all 10 real**
 DB_COMP_SPEED files now in this project's sample set (the original 4
@@ -72,6 +88,7 @@ from __future__ import annotations
 import struct
 import warnings
 from dataclasses import dataclass
+from typing import List, Optional
 
 try:
     from . import _native as _native_ext
@@ -208,7 +225,8 @@ def _lzrw1_decompress_py(data: bytes, start: int, decompressed_length: int) -> b
 
 @dataclass
 class SpeedChunk:
-    magic_offset: int          # file offset of the 16-byte magic sub-header
+    magic_offset: Optional[int]  # offset of the 16-byte magic sub-header; None for a
+                                 # continuation chunk, which has no magic of its own
     subtype: int                # 1 = DB_COMP_SPEED, 2 = DB_COMP_SIZE
     decompressed_length: int
     chunk_length: int          # includes the 12-byte length sub-header
@@ -361,11 +379,135 @@ def decode_speed_chunk(data: bytes, chunk: SpeedChunk):
         ) from e
 
 
+def decode_speed_blob(data: bytes, total_decompressed_length: int = 0):
+    """
+    Decode every chunk of a DB_COMP_SPEED blob, in order.
+
+    Parameters
+    ----------
+    data : bytes or bytearray
+        The blob's compressed span, starting at the 16-byte magic of its
+        first chunk (i.e. everything after the blob header,
+        docs/spec.md section 7.4).
+    total_decompressed_length : int, optional
+        The blob's total decompressed size in bytes, from its header
+        (`+24`, docs/spec.md section 7.4). Decoding continues chunk by
+        chunk until this many bytes have been produced. If not positive
+        (a header that doesn't carry it, as in some hand-built
+        fixtures), only the first chunk is decoded.
+
+    Returns
+    -------
+    bytearray or bytes
+        The concatenated output of every chunk. A single-chunk blob
+        returns exactly what `decode_speed_chunk` does for it (no extra
+        copy); a multi-chunk one is a new, writable `bytearray`.
+
+    Raises
+    ------
+    LZRW1DecodeError
+        If any chunk fails to decode (see `decode_speed_chunk`), the
+        chain runs off the end of `data`, or the chunks don't add up
+        to exactly `total_decompressed_length`.
+
+    Notes
+    -----
+    A blob is a chain of chunks of at most 16368 decompressed bytes
+    each, not a single chunk (module docstring point 4): only the first
+    carries the 16-byte magic, and every later one is a bare 12-byte
+    sub-header plus payload starting `chunk_length` bytes after the
+    previous sub-header began. This reader used to decode only the first
+    chunk, silently truncating any channel longer than 2046 float64
+    values on a line to exactly that length.
+
+    Dispatches to the compiled `pygdb._native` extension when it's
+    available and `data` is `bytes` (same algorithm, ported to Rust --
+    see `rust/src/lib.rs`), falling back to the pure-Python
+    `_decode_speed_blob_py` below otherwise. The native version raises
+    `ValueError`/`IndexError` for the same conditions; both are turned
+    into `LZRW1DecodeError` here, so callers never see which backend
+    produced a failure.
+    """
+    if _native_ext is not None and isinstance(data, bytes):
+        try:
+            return _native_ext.decode_speed_blob(data, total_decompressed_length)
+        except (ValueError, IndexError) as e:
+            raise LZRW1DecodeError(str(e)) from e
+    return _decode_speed_blob_py(data, total_decompressed_length)
+
+
+def _decode_speed_blob_py(data: bytes, total_decompressed_length: int = 0):
+    """
+    Pure-Python reference implementation of `decode_speed_blob`.
+
+    Parameters
+    ----------
+    data : bytes or bytearray
+        See `decode_speed_blob`.
+    total_decompressed_length : int, optional
+        See `decode_speed_blob`.
+
+    Returns
+    -------
+    bytearray or bytes
+        See `decode_speed_blob`.
+
+    Raises
+    ------
+    LZRW1DecodeError
+        See `decode_speed_blob`.
+    """
+    first = parse_chunk_header(data, 0)
+    out = decode_speed_chunk(data, first)
+    if total_decompressed_length <= len(out):
+        return out
+
+    parts: List[bytes] = [out]
+    produced = len(out)
+    header_start = first.payload_offset - 12  # where this chunk's sub-header began
+    chunk_length = first.chunk_length
+    while produced < total_decompressed_length:
+        if chunk_length < 12:
+            raise LZRW1DecodeError(
+                f"implausible chunk_length={chunk_length} -- corrupt chunk chain"
+            )
+        header_start += chunk_length
+        try:
+            decompressed_length, chunk_length, marker = struct.unpack_from(
+                "<iii", data, header_start
+            )
+        except struct.error as e:
+            raise LZRW1DecodeError(
+                f"chunk chain runs off the end of the data after {produced} of "
+                f"{total_decompressed_length} byte(s) -- truncated data"
+            ) from e
+        chunk = SpeedChunk(
+            magic_offset=None,
+            subtype=DB_COMP_SPEED,
+            decompressed_length=decompressed_length,
+            chunk_length=chunk_length,
+            marker=marker,
+            payload_offset=header_start + 12,
+        )
+        parts.append(decode_speed_chunk(data, chunk))
+        produced += decompressed_length
+
+    if produced != total_decompressed_length:
+        raise LZRW1DecodeError(
+            f"chunks decode to {produced} byte(s) but the blob header declares "
+            f"{total_decompressed_length}"
+        )
+    return bytearray().join(parts)
+
+
 def find_speed_chunks(data: bytes):
     """
     Yield every DB_COMP_SPEED (subtype==1) chunk found in `data`.
 
-    Scans for the shared 16-byte magic byte-by-byte.
+    Scans for the shared 16-byte magic byte-by-byte. Since only the
+    *first* chunk of a blob carries that magic (see `decode_speed_blob`),
+    this finds one chunk per blob, not every chunk -- it's a scanning
+    helper for locating blobs, not a way to decode them.
 
     Parameters
     ----------

{python_gdb-0.2.0 → python_gdb-0.3.0}/rust/Cargo.lock

@@ -46,7 +46,7 @@ dependencies = [
 
 [[package]]
 name = "pygdb-native"
-version = "0.2.0"
+version = "0.3.0"
 dependencies = [
  "flate2",
  "pyo3",

{python_gdb-0.2.0 → python_gdb-0.3.0}/rust/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "pygdb-native"
-version = "0.2.0"
+version = "0.3.0"
 edition = "2021"
 description = "Optional Rust-accelerated backend for python-gdb (pygdb._native)"
 license = "MIT"

{python_gdb-0.2.0 → python_gdb-0.3.0}/rust/src/lib.rs

@@ -148,6 +148,175 @@ fn lzrw1_decompress_impl(data: &[u8], start: usize, out: &mut [u8]) -> PyResult<
     Ok(())
 }
 
+const DB_COMP_SPEED: i32 = 1;
+const MARKER_COMPRESSED: i32 = -186263865; // 0xF4E5D6C7 -- payload is real LZRW1
+const MARKER_STORED_RAW: i32 = -253635901; // 0xF0E1D2C3 -- payload is stored verbatim
+
+/// A chunk's 12-byte `<decompressed_length> <chunk_length> <marker>`
+/// sub-header, read from `data[header_start..]`.
+fn read_chunk_subheader(data: &[u8], header_start: usize) -> PyResult<(i32, i32, i32)> {
+    let bytes = header_start
+        .checked_add(12)
+        .and_then(|end| data.get(header_start..end))
+        .ok_or_else(|| {
+            PyIndexError::new_err(
+                "decode_speed_blob: chunk chain runs off the end of the data -- truncated",
+            )
+        })?;
+    let field = |i: usize| i32::from_le_bytes([bytes[i], bytes[i + 1], bytes[i + 2], bytes[i + 3]]);
+    Ok((field(0), field(4), field(8)))
+}
+
+/// Decode one chunk's payload into `out` (exactly its `decompressed_length`
+/// bytes) -- the Rust counterpart of `pygdb.lzrw1.decode_speed_chunk`,
+/// with the same checks in the same order.
+fn decode_speed_chunk_into(
+    data: &[u8],
+    header_start: usize,
+    chunk_length: i32,
+    marker: i32,
+    out: &mut [u8],
+) -> PyResult<()> {
+    let payload_offset = header_start + 12;
+    match marker {
+        MARKER_STORED_RAW => {
+            if chunk_length as i64 - 12 != out.len() as i64 {
+                return Err(PyValueError::new_err(format!(
+                    "decode_speed_blob: stored-raw chunk should have chunk_length-12 == \
+                     decompressed_length (got chunk_length-12={}, decompressed_length={})",
+                    chunk_length as i64 - 12,
+                    out.len(),
+                )));
+            }
+            let payload = data
+                .get(payload_offset..payload_offset + out.len())
+                .ok_or_else(|| {
+                    PyIndexError::new_err(
+                        "decode_speed_blob: truncated stored-raw payload -- file cut off mid-chunk?",
+                    )
+                })?;
+            out.copy_from_slice(payload);
+            Ok(())
+        }
+        MARKER_COMPRESSED => lzrw1_decompress_impl(data, payload_offset, out),
+        other => Err(PyValueError::new_err(format!(
+            "decode_speed_blob: unrecognized marker value: {other}"
+        ))),
+    }
+}
+
+/// Walk a whole `DB_COMP_SPEED` blob's chain of chunks into `out`.
+///
+/// `out.len()` is either the first chunk's `decompressed_length` (a
+/// single-chunk blob, or no usable total) or the blob's declared total; the
+/// loop stops as soon as `out` is full, and errors if a chunk would
+/// overshoot it -- matching `pygdb.lzrw1._decode_speed_blob_py`.
+fn decode_speed_chain_into(data: &[u8], out: &mut [u8]) -> PyResult<()> {
+    let mut written = 0usize;
+    let mut header_start = 16usize; // just past the first chunk's 16-byte magic
+    loop {
+        let (decompressed_length, chunk_length, marker) = read_chunk_subheader(data, header_start)?;
+        if !(0 < decompressed_length && decompressed_length < 200_000_000) {
+            return Err(PyValueError::new_err(format!(
+                "decode_speed_blob: implausible decompressed_length={decompressed_length} -- \
+                 likely a misaligned or corrupt chunk header"
+            )));
+        }
+        let end = written + decompressed_length as usize;
+        if end > out.len() {
+            return Err(PyValueError::new_err(
+                "decode_speed_blob: chunks decode to more bytes than the blob header declares",
+            ));
+        }
+        decode_speed_chunk_into(
+            data,
+            header_start,
+            chunk_length,
+            marker,
+            &mut out[written..end],
+        )?;
+        written = end;
+        if written == out.len() {
+            return Ok(());
+        }
+        if chunk_length < 12 {
+            return Err(PyValueError::new_err(format!(
+                "decode_speed_blob: implausible chunk_length={chunk_length} -- corrupt chunk chain"
+            )));
+        }
+        header_start += chunk_length as usize;
+    }
+}
+
+/// Decode every chunk of a `DB_COMP_SPEED` blob into one writable Python
+/// `bytearray`.
+///
+/// Rust port of `pygdb.lzrw1.decode_speed_blob` -- see that module's
+/// docstring (point 4) and docs/spec.md section 7.3/7.4 for the format:
+/// a blob is a chain of chunks of at most 16368 decompressed bytes each,
+/// only the first preceded by the 16-byte magic; `data` starts at that
+/// magic, and `total_decompressed_length` is the blob header's `+24`
+/// field (how the decoder knows the chain has ended -- the bytes after
+/// the last chunk are page padding, not zeros). A total that isn't
+/// larger than the first chunk (including 0 or negative, "no usable
+/// total") decodes just the first chunk.
+///
+/// Raises `ValueError` for a malformed chunk or chain (bad marker,
+/// implausible lengths, a total the chain doesn't add up to) and
+/// `IndexError` for truncated data -- `pygdb.lzrw1.decode_speed_blob`
+/// turns both into `LZRW1DecodeError`, so callers never see which
+/// backend produced the failure.
+///
+/// The output buffer is sized once, up front, from the first chunk's
+/// header and the declared total, then filled in place via
+/// `PyByteArray::new_with` under `Python::detach` -- the same single-
+/// allocation technique and the same GIL-release argument as
+/// `lzrw1_decompress` (`data` is `&[u8]`, so an immutable `bytes`; the
+/// target `bytearray` isn't Python-visible until this returns). A
+/// declared total larger than any real chain could produce from `data`
+/// (LZRW1 expands at most 8x, plus the 12-byte sub-headers) is rejected
+/// before allocating, so a corrupt header can't request a huge buffer.
+#[pyfunction]
+fn decode_speed_blob<'py>(
+    py: Python<'py>,
+    data: &[u8],
+    total_decompressed_length: i64,
+) -> PyResult<Bound<'py, PyByteArray>> {
+    let subtype = data
+        .get(8..12)
+        .map(|b| i32::from_le_bytes([b[0], b[1], b[2], b[3]]))
+        .ok_or_else(|| PyIndexError::new_err("decode_speed_blob: truncated -- no chunk header"))?;
+    if subtype != DB_COMP_SPEED {
+        return Err(PyValueError::new_err(format!(
+            "decode_speed_blob: not a Speed chunk (subtype={subtype})"
+        )));
+    }
+    let (first_length, _, _) = read_chunk_subheader(data, 16)?;
+    if !(0 < first_length && first_length < 200_000_000) {
+        return Err(PyValueError::new_err(format!(
+            "decode_speed_blob: implausible decompressed_length={first_length} -- \
+             likely a misaligned or corrupt chunk header"
+        )));
+    }
+    let first_length = first_length as usize;
+    let out_len = if total_decompressed_length > first_length as i64 {
+        let total = total_decompressed_length as u64;
+        if total > (data.len() as u64).saturating_mul(9) {
+            return Err(PyValueError::new_err(format!(
+                "decode_speed_blob: blob header declares {total} decompressed byte(s), \
+                 more than {} byte(s) of chunk data could possibly produce",
+                data.len(),
+            )));
+        }
+        total as usize
+    } else {
+        first_length
+    };
+    PyByteArray::new_with(py, out_len, |buf| {
+        py.detach(|| decode_speed_chain_into(data, buf))
+    })
+}
+
 /// Decompress a `DB_COMP_SIZE` blob's raw zlib/DEFLATE stream into a
 /// writable Python `bytearray`.
 ///
@@ -483,6 +652,7 @@ fn decode_fixed_width_strings_ucs4<'py>(
 fn _native(m: &Bound<'_, PyModule>) -> PyResult<()> {
     m.add_function(wrap_pyfunction!(ping, m)?)?;
     m.add_function(wrap_pyfunction!(lzrw1_decompress, m)?)?;
+    m.add_function(wrap_pyfunction!(decode_speed_blob, m)?)?;
     m.add_function(wrap_pyfunction!(zlib_decompress, m)?)?;
     m.add_function(wrap_pyfunction!(decompress_grd_blocks, m)?)?;
     m.add_function(wrap_pyfunction!(decode_fixed_width_strings_ucs4, m)?)?;