python-gdb 0.2.1__tar.gz → 0.3.0__tar.gz

{python_gdb-0.2.1 → python_gdb-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python-gdb
-Version: 0.2.1
+Version: 0.3.0
 Classifier: Development Status :: 3 - Alpha
 Classifier: Intended Audience :: Science/Research
 Classifier: Topic :: Scientific/Engineering :: GIS

{python_gdb-0.2.1 → python_gdb-0.3.0}/pygdb/__init__.py

@@ -16,16 +16,19 @@ Reading only: writing/mutating `.gdb` or `.grd` files is out of scope.
 
 from .gdb import GDB, CompressionInfo
 from .gdb_reader import (
+    BlobDirectory,
     BlobHeader,
     ChannelRecord,
     GDBParseWarning,
     LineRecord,
     check_magic,
+    exact_line_table_start,
     find_blob,
     find_channel_table,
     find_line_table,
     header_fields,
     iter_blobs,
+    read_blob_directory,
     read_blob_values,
     read_channels,
     read_lines,
@@ -36,6 +39,7 @@ from .registry import find_coordinate_systems
 
 __all__ = [
     "GDB",
+    "BlobDirectory",
     "BlobHeader",
     "ChannelRecord",
     "CompressionInfo",
@@ -45,12 +49,14 @@ __all__ = [
     "LZRW1DecodeError",
     "LineRecord",
     "check_magic",
+    "exact_line_table_start",
     "find_blob",
     "find_channel_table",
     "find_coordinate_systems",
     "find_line_table",
     "header_fields",
     "iter_blobs",
+    "read_blob_directory",
     "read_blob_values",
     "read_channels",
     "read_grd",

{python_gdb-0.2.1 → python_gdb-0.3.0}/pygdb/gdb.py

@@ -30,9 +30,13 @@ from .gdb_reader import (
     check_magic,
     header_fields,
     iter_blobs,
+    read_blob_directory,
     read_blob_values,
     read_channels,
-    read_lines,
+    _read_lines,
+    DIRECTORY_ABSENT,
+    DIRECTORY_INVALID,
+    DIRECTORY_LIVE,
 )
 from .registry import find_channel_roles, find_coordinate_systems
 
@@ -60,87 +64,6 @@ LineRef = Union[str, Tuple[str, int], "LineRecord"]
 ChannelRef = Union[str, Tuple[str, int], "ChannelRecord"]
 
 
-def _finite_values(values: np.ndarray) -> np.ndarray:
-    v = np.asarray(values, dtype=float).ravel()
-    return v[np.isfinite(v) & (np.abs(v) < 1e30)]
-
-
-def _roughness(values: np.ndarray) -> Optional[float]:
-    """
-    Median row-to-row step divided by the 5-95% spread of the values.
-
-    Returns
-    -------
-    float or None
-        Small for data that varies smoothly along its rows, large for
-        the same values in a scrambled order. None if there are too few
-        finite values or they are constant.
-    """
-    v = _finite_values(values)
-    if len(v) < 50:
-        return None
-    spread = float(np.percentile(v, 95) - np.percentile(v, 5))
-    if spread == 0.0:
-        return None
-    return float(np.median(np.abs(np.diff(v))) / spread)
-
-
-def _row_order_pick(first: np.ndarray, last: np.ndarray, factor: float = 3.0) -> Optional[int]:
-    """
-    Decide which of two copies of one channel is in acquisition order.
-
-    Parameters
-    ----------
-    first, last : numpy.ndarray
-        The earlier and later copy in the blob chain.
-    factor : float, optional
-        How many times rougher one copy must be than the other.
-
-    Returns
-    -------
-    int or None
-        0 if the *first* copy is the smooth one and the last is a
-        markedly rougher reordering of it; 1 if the last is the smooth
-        one; None if the copies are not a pure reordering of each other
-        (different values or length) or neither is clearly smoother.
-
-    Notes
-    -----
-    A copy that is *itself* perfectly monotone is never judged: a sorted
-    ramp is smoother than any real signal, and a re-sort by a channel's
-    own value (a coordinate re-sorted by itself) leaves exactly that. It
-    could equally be a genuine ID or time channel, and nothing here can
-    tell the two apart, so such a pair is undecided.
-    """
-    if len(first) != len(last):
-        return None
-    a, b = np.asarray(first).ravel(), np.asarray(last).ravel()
-    try:
-        if not np.array_equal(np.sort(a), np.sort(b), equal_nan=True):
-            return None
-    except TypeError:  # dtype without a NaN notion (integers)
-        if not np.array_equal(np.sort(a), np.sort(b)):
-            return None
-    if _is_monotone_reference(a) or _is_monotone_reference(b):
-        return None
-    ra, rb = _roughness(a), _roughness(b)
-    if ra is None or rb is None:
-        return None
-    if rb > 0 and rb >= factor * ra:
-        return 0
-    if ra > 0 and ra >= factor * rb:
-        return 1
-    return None
-
-
-def _is_monotone_reference(values: np.ndarray) -> bool:
-    """A non-constant channel stored in non-decreasing order (an ID, date or time)."""
-    v = _finite_values(values)
-    if len(v) < 50 or v[0] == v[-1]:
-        return False
-    return bool(np.mean(np.diff(v) >= 0) >= 0.999)
-
-
 @dataclass
 class CompressionInfo:
     """
@@ -181,20 +104,15 @@ class GDB:
     ----------
     path : str
         Path to the `.gdb` file.
-    duplicate_blobs : {"last", "row_order"}, optional
-        What to do when a (line, channel) has more than one blob in the
-        blob chain (issue #2). `"last"` (the default) uses the last one
-        in chain order. `"row_order"` is an opt-in **heuristic**, not a
-        decoded field: for a duplicated numeric channel whose two
-        copies hold exactly the same values in a different row order
-        (a stale re-sorted copy), on a line that has an order-defining
-        channel (see Notes), it prefers the copy whose values vary
-        smoothly along the rows -- acquisition order, the order the
-        line's ID/time channels are stored in -- and keeps the last
-        copy in every other case. Either way one `GDBParseWarning`
-        names the affected pairs and, for `"row_order"`, which ones it
-        overrode. Choosing needs both copies decoded, so it is done
-        once, when the blob index is first built.
+    include_unlisted_blobs : bool, optional
+        A file that carries a blob directory (see Notes) lists exactly
+        one live blob per (line, channel). By default a blob the
+        directory does not list -- an all-zero slot -- is **skipped**, as
+        if it were absent, and one `GDBParseWarning` says how many
+        (line, channel) blobs in which channels were skipped. Pass
+        `True` to read them anyway (the last copy in chain order). A
+        file with no directory, or an all-zero one, is unaffected: every
+        blob in the chain is served.
 
     Raises
     ------
@@ -203,8 +121,7 @@ class GDB:
         expected `.gdb` magic -- unlike the module-level functions in
         `gdb_reader`/`registry` (which warn and return empty results),
         since a `GDB` object that isn't backed by a real `.gdb` file
-        can't usefully do anything at all. Also if `duplicate_blobs` is
-        not one of the values above.
+        can't usefully do anything at all.
 
     Examples
     --------
@@ -234,31 +151,22 @@ class GDB:
     context manager) when done with it, or just let it get
     garbage-collected -- `__del__` closes it too, as a safety net.
 
-    `duplicate_blobs="row_order"` only ever chooses between two copies
-    of one channel that are a pure reordering of each other, and only on
-    a line with an *order-defining channel*: a single-copy numeric
-    channel of the same length stored in monotone order (typically an
-    ID, date or time). A revised copy (different values) is never
-    second-guessed, and neither is a pair with a copy that is itself
-    perfectly monotone (a re-sort by a channel's own value leaves a
-    smooth ramp that looks like the best copy but is the stale one).
-    Among a qualifying pair, the copy at least 3x
-    rougher along the rows -- median row-to-row step over the 5-95%
-    spread of the values -- is treated as the stale one, since data in
-    acquisition order varies smoothly and a re-sort scrambles that. This
-    was validated against independent spreadsheet exports of the one
-    real file known to have such copies (it never contradicted them),
-    but it cannot decide a channel that is smooth in both orders, and no
-    on-disk marker has been found that would make it unnecessary.
+    **Which copy of a blob is read.** The blob chain is append-only, so
+    a (line, channel) can have several blobs, an older stale one and
+    the current one, in either order (issue #2). The file's persisted
+    *blob directory* (docs/spec.md section 2.2) says which is current,
+    and it is the only thing consulted. A directory entry is used only
+    if its start page lands on a walked blob header whose `blob_index`
+    equals the slot and whose page count matches; if a non-zero entry
+    fails that check, the last copy in chain order is used and a
+    `GDBParseWarning` says so. Without a usable directory the last copy
+    in chain order is used, with a warning if a pair is duplicated: that
+    is a guess, and not always the current copy.
     """
 
-    def __init__(self, path: str, duplicate_blobs: str = "last"):
-        if duplicate_blobs not in ("last", "row_order"):
-            raise ValueError(
-                f"duplicate_blobs must be 'last' or 'row_order', got {duplicate_blobs!r}"
-            )
+    def __init__(self, path: str, include_unlisted_blobs: bool = False):
         self.path = path
-        self.duplicate_blobs = duplicate_blobs
+        self.include_unlisted_blobs = include_unlisted_blobs
         self._file = open(path, "rb")
         header = self._file.read(4096)
         if not check_magic(header):
@@ -270,6 +178,7 @@ class GDB:
         self._fields = header_fields(header)
         self._channels: Optional[List[ChannelRecord]] = None
         self._lines: Optional[List[LineRecord]] = None
+        self._lines_exact = False
         self._channels_by_name: Optional[Dict[str, List[ChannelRecord]]] = None
         self._lines_by_name: Optional[Dict[str, List[LineRecord]]] = None
         self._blob_index: Optional[Dict[Tuple[int, int], BlobHeader]] = None
@@ -385,7 +294,7 @@ class GDB:
     def lines(self) -> List[LineRecord]:
         """list of LineRecord: This file's line table, read once and cached."""
         if self._lines is None:
-            self._lines = read_lines(self.path)
+            self._lines, self._lines_exact = _read_lines(self.path)
             self._lines_by_name = {}
             for l in self._lines:
                 self._lines_by_name.setdefault(l.name, []).append(l)
@@ -547,219 +456,144 @@ class GDB:
         Returns
         -------
         dict of {(int, int) : BlobHeader}
-            Maps `(line_slot, channel_slot)` to `BlobHeader`, built
-            with one blob-chain walk and cached from then on.
-            `iter_blobs`/`find_blob` themselves recommend this for
-            anything beyond an occasional one-off lookup -- this class
-            always wants line/channel listings and random-access
+            Maps `(line_slot, channel_slot)` to the `BlobHeader` of its
+            live blob, built with one blob-chain walk and cached from
+            then on. `iter_blobs`/`find_blob` themselves recommend this
+            for anything beyond an occasional one-off lookup -- this
+            class always wants line/channel listings and random-access
             reads, so it always builds the index.
 
         Warns
         -----
         GDBParseWarning
-            If a real line's channel has more than one blob in the
-            blob chain. By default the **last** one in chain order is
-            used, but that is not always the current copy (issue #2):
-            an older copy with the same values in a different row order
-            can sit either before or after the current one, and nothing
-            decoded so far says which is which. With
-            `duplicate_blobs="row_order"` the warning also says which
-            pairs were switched to an earlier copy. Duplicates in the
-            administrative slots past the last real line (the REG/IPJ
-            registry, whose stale copies are expected and handled by
-            `pygdb.registry`) are not reported.
+            See `_select_live_blobs`: when blobs are skipped because the
+            file's blob directory does not list them, when a directory
+            entry fails validation, or when a pair is duplicated and
+            the file has no directory to say which copy is current.
         """
         if self._blob_index is None:
             chans_max = self.chans_max
             copies: Dict[Tuple[int, int], List[BlobHeader]] = {}
+            by_offset: Dict[int, BlobHeader] = {}
             for blob in iter_blobs(self.path):
                 copies.setdefault(blob.line_channel(chans_max), []).append(blob)
-            index: Dict[Tuple[int, int], BlobHeader] = {k: v[-1] for k, v in copies.items()}
-            duplicated = [k for k, v in copies.items() if len(v) > 1]
-            self._blob_index = index
-            self._calibrate_line_indices()
-            if duplicated:
-                overridden: List[Tuple[int, int]] = []
-                if self.duplicate_blobs == "row_order":
-                    overridden = self._prefer_row_order_copies(duplicated, copies, index)
-                self._warn_duplicate_blobs(duplicated, overridden)
+                by_offset[blob.offset] = blob
+            self._blob_index = {k: v[-1] for k, v in copies.items()}
+            if not self.lines:
+                return self._blob_index
+            if not self._lines_exact:
+                self._calibrate_line_indices()
+            self._select_live_blobs(copies, by_offset)
         return self._blob_index
 
-    def _read_copy(self, blob: BlobHeader, channel: ChannelRecord) -> np.ndarray:
-        return read_blob_values(
-            self.path, blob, channel,
-            comp_level=self.comp_level or 0, page_size=self.page_size, file=self._file,
-        )
-
-    def _prefer_row_order_copies(
+    def _select_live_blobs(
         self,
-        duplicated: List[Tuple[int, int]],
         copies: Dict[Tuple[int, int], List[BlobHeader]],
-        index: Dict[Tuple[int, int], BlobHeader],
-    ) -> List[Tuple[int, int]]:
+        by_offset: Dict[int, BlobHeader],
+    ) -> None:
         """
-        Apply `duplicate_blobs="row_order"` to the duplicated pairs.
+        Choose each real (line, channel)'s live blob using the blob directory.
 
         Parameters
         ----------
-        duplicated : list of (int, int)
-            `(line_slot, channel_slot)` keys with more than one blob.
         copies : dict of {(int, int) : list of BlobHeader}
             Every blob for each key, in chain order.
-        index : dict of {(int, int) : BlobHeader}
-            The blob index being built; updated in place with the
-            earlier copy wherever one is preferred.
+        by_offset : dict of {int : BlobHeader}
+            Every blob of the chain, keyed by absolute offset.
 
-        Returns
-        -------
-        list of (int, int)
-            The keys switched from the last copy to an earlier one.
+        Warns
+        -----
+        GDBParseWarning
+            Once per situation, naming counts and examples: (1) pairs
+            skipped because the directory has no live entry for them;
+            (2) pairs whose non-zero directory entry failed validation
+            (the last copy in chain order is used); (3) with no usable
+            directory at all, pairs that have more than one blob.
+            Administrative slots past the last real line (the REG/IPJ
+            registry, whose stale copies `pygdb.registry` handles
+            itself) are never judged or reported.
 
         Notes
         -----
-        See the class docstring for exactly when a pair qualifies. A pair
-        that doesn't (a string or array channel, more than two copies,
-        different values, no order-defining channel on the line, or no
-        clear winner) simply keeps the last copy.
+        Updates `self._blob_index` in place. See the class docstring for
+        the rules.
         """
+        index = self._blob_index
         real_lines = {line.index for line in self.lines}
-        channels = {c.index: c for c in self.channels}
-        duplicated_keys = set(duplicated)
-        reference_cache: Dict[Tuple[int, int], bool] = {}
-        overridden: List[Tuple[int, int]] = []
-        for key in duplicated:
-            line_slot, channel_slot = key
-            channel = channels.get(channel_slot)
-            blobs = copies[key]
-            if (
-                line_slot not in real_lines or channel is None or len(blobs) != 2
-                or channel.is_string or channel.is_array
-            ):
-                continue
-            first, last = self._read_copy(blobs[0], channel), self._read_copy(blobs[1], channel)
-            if len(first) != len(last):
-                continue
-            ref_key = (line_slot, len(first))
-            if ref_key not in reference_cache:
-                reference_cache[ref_key] = self._line_has_order_reference(
-                    line_slot, len(first), duplicated_keys, channels
+        directory = read_blob_directory(self.path)
+        if directory is None:
+            duplicated = [k for k, v in copies.items() if len(v) > 1 and k[0] in real_lines]
+            if duplicated:
+                self._warn_pairs(
+                    f"{len(duplicated)} (line, channel) pair(s) have more than one blob in the "
+                    f"blob chain ({self._describe_pairs(duplicated)}) and the file has no blob "
+                    f"directory to say which is current -- using the last one in chain order, "
+                    f"which is not always the current copy. If a channel's rows look scrambled "
+                    f"against the line's other channels, this is the likely cause (see issue #2)"
                 )
-            if reference_cache[ref_key] and _row_order_pick(first, last) == 0:
-                index[key] = blobs[0]
-                overridden.append(key)
-        return overridden
-
-    def _line_has_order_reference(
-        self,
-        line_slot: int,
-        n_rows: int,
-        duplicated_keys: set,
-        channels: Dict[int, ChannelRecord],
-    ) -> bool:
-        """
-        Whether a line has a channel proving its rows are in some fixed order.
-
-        Parameters
-        ----------
-        line_slot : int
-            The line's slot.
-        n_rows : int
-            The row count of the duplicated copies being judged.
-        duplicated_keys : set of (int, int)
-            Keys with more than one blob -- excluded, since their own
-            order is what is in question.
-        channels : dict of {int : ChannelRecord}
-            Channel records by slot.
-
-        Returns
-        -------
-        bool
-            True if some single-copy, numeric, non-array channel of
-            `n_rows` values on this line is stored in monotone
-            (non-decreasing) order and isn't constant -- typically an
-            ID, date or time channel.
-        """
-        for (ls, cs), blob in sorted(self._ensure_blob_index().items()):
-            channel = channels.get(cs)
-            if (
-                ls != line_slot or (ls, cs) in duplicated_keys or channel is None
-                or channel.is_string or channel.is_array
-            ):
+            return
+        first_offset = min(by_offset)
+        skipped: List[Tuple[int, int]] = []
+        invalid: List[Tuple[int, int]] = []
+        for key in copies:
+            line_slot, channel_slot = key
+            if line_slot not in real_lines:
                 continue
-            values = self._read_copy(blob, channel)
-            if len(values) == n_rows and _is_monotone_reference(values):
-                return True
-        return False
-
-    def _warn_duplicate_blobs(
-        self,
-        duplicated: List[Tuple[int, int]],
-        overridden: Sequence[Tuple[int, int]] = (),
-    ) -> None:
-        """
-        Warn about (line, channel) pairs that have more than one blob.
-
-        Parameters
-        ----------
-        duplicated : list of (int, int)
-            `(line_slot, channel_slot)` keys seen more than once in the
-            blob chain.
-        overridden : sequence of (int, int), optional
-            The keys `duplicate_blobs="row_order"` switched to an
-            earlier copy.
+            status, blob = directory.resolve(
+                line_slot * self.chans_max + channel_slot, by_offset, first_offset, self.page_size,
+            )
+            if status == DIRECTORY_LIVE:
+                index[key] = blob
+            elif status == DIRECTORY_ABSENT:
+                skipped.append(key)
+            elif status == DIRECTORY_INVALID:
+                invalid.append(key)
+        if skipped and not self.include_unlisted_blobs:
+            for key in skipped:
+                del index[key]
+            channel_names = {c.index: c.name for c in self.channels}
+            by_channel: Dict[int, int] = {}
+            for _, channel_slot in skipped:
+                by_channel[channel_slot] = by_channel.get(channel_slot, 0) + 1
+            listing = ", ".join(
+                f"{channel_names.get(cs, f'#{cs}')!r} ({n})" for cs, n in sorted(by_channel.items())[:5]
+            )
+            more = f" and {len(by_channel) - 5} more channel(s)" if len(by_channel) > 5 else ""
+            self._warn_pairs(
+                f"skipped {len(skipped)} (line, channel) blob(s) in {len(by_channel)} channel(s) "
+                f"[{listing}{more}] that the file's blob directory does not list as live. Pass "
+                f"include_unlisted_blobs=True to read them anyway"
+            )
+        if invalid:
+            self._warn_pairs(
+                f"{len(invalid)} (line, channel) pair(s) have a blob-directory entry that does not "
+                f"point at a blob with the right index and size ({self._describe_pairs(invalid)}) "
+                f"-- using the last copy in chain order for them"
+            )
 
-        Warns
-        -----
-        GDBParseWarning
-            Once, naming how many real (line, channel) pairs are
-            affected and a few examples (and, for `"row_order"`, which
-            ones were switched). Nothing is emitted if every duplicate
-            is in an administrative slot.
-        """
+    def _describe_pairs(self, keys: Sequence[Tuple[int, int]], limit: int = 3) -> str:
         line_names = {line.index: line.name for line in self.lines}
         channel_names = {c.index: c.name for c in self.channels}
+        named = [
+            f"line {line_names[ls]!r} channel {channel_names.get(cs, f'#{cs}')!r}"
+            for ls, cs in keys if ls in line_names
+        ]
+        more = f" and {len(named) - limit} more" if len(named) > limit else ""
+        return ", ".join(named[:limit]) + more
 
-        def describe(keys, limit=3):
-            named = [
-                f"line {line_names[ls]!r} channel {channel_names.get(cs, f'#{cs}')!r}"
-                for ls, cs in keys if ls in line_names
-            ]
-            more = f" and {len(named) - limit} more" if len(named) > limit else ""
-            return ", ".join(named[:limit]) + more
-
-        n_real = sum(1 for ls, _ in duplicated if ls in line_names)
-        if not n_real:
-            return
-        head = (
-            f"{self.path}: {n_real} (line, channel) pair(s) have more than one "
-            f"blob in the blob chain ({describe(duplicated)})"
-        )
-        if self.duplicate_blobs == "row_order":
-            tail = (
-                f" -- duplicate_blobs='row_order': switched {len(overridden)} pair(s) to "
-                f"an earlier copy because the last copy was the same values in a "
-                f"markedly rougher row order ({describe(overridden)}); kept the last "
-                f"copy in chain order for the other {n_real - len(overridden)}. This "
-                f"is a heuristic, not a decoded field (see issue #2)"
-                if overridden else
-                f" -- duplicate_blobs='row_order': no pair needed switching, so the "
-                f"last copy in chain order was kept for all of them (see issue #2)"
-            )
-        else:
-            tail = (
-                " -- using the last one in chain order, which is not always the "
-                "current copy. If a channel's rows look scrambled against the line's "
-                "other channels, this is the likely cause; duplicate_blobs='row_order' "
-                "can pick the acquisition-order copy (see issue #2)"
-            )
-        warnings.warn(head + tail, GDBParseWarning, stacklevel=4)
+    def _warn_pairs(self, message: str) -> None:
+        # stacklevel: here -> _select_live_blobs -> _ensure_blob_index -> public method -> caller
+        warnings.warn(f"{self.path}: {message}", GDBParseWarning, stacklevel=5)
 
     def _calibrate_line_indices(self) -> None:
         """
         Correct a possible small, fixed off-by-N in every line's index.
 
-        See `find_line_table`'s and `read_lines`'s docstrings in
-        `gdb_reader.py`. Checks, for a handful of small integer shifts,
+        Only used when `read_lines` had to fall back to the
+        `find_line_table` heuristic (`self._lines_exact` is False): a
+        file whose line table is located exactly already has true
+        indices. See `find_line_table`'s and `read_lines`'s docstrings
+        in `gdb_reader.py`. Checks, for a handful of small integer shifts,
         which one makes the most already-found lines actually have at
         least one real data blob on disk for *some* channel -- then
         applies the winning shift to every `LineRecord.index` in

{python_gdb-0.2.1 → python_gdb-0.3.0}/pygdb/gdb_reader.py

@@ -443,11 +443,15 @@ def header_fields(data: bytes) -> dict:
     Returns
     -------
     dict
-        Maps each of `"chans_max"`, `"users_max"`, `"page_size"`, and
-        `"comp_level"` to its decoded int32 value. See
-        docs/provenance/notes.md section 6.1 for the full table
-        including the still-unknown offsets, and for why each
-        confidence label was assigned.
+        Maps each of `"chans_max"` (word 24), `"blobs_max"` (28),
+        `"lines_max"` (36), `"users_max"` (40), `"index_slots"` (44,
+        the total number of blob-directory slots), `"data_slots"` (48,
+        the number of those that address (line, channel) data blobs),
+        `"page_size"` (100), and `"comp_level"` (120) to its decoded
+        int32 value. See docs/spec.md section 2 for the full table
+        with a confidence rating per word, and
+        docs/provenance/notes.md section 6.1/6.1b for the derivations
+        and the still-unknown offsets.
 
     Warns
     -----
@@ -466,7 +470,8 @@ def header_fields(data: bytes) -> dict:
     (`DB_COMP_SIZE`) IS confirmed real zlib.
     """
     result = {}
-    for name, offset in (("chans_max", 24), ("users_max", 40),
+    for name, offset in (("chans_max", 24), ("blobs_max", 28), ("lines_max", 36),
+                          ("users_max", 40), ("index_slots", 44), ("data_slots", 48),
                           ("page_size", 100), ("comp_level", 120)):
         try:
             result[name] = struct.unpack_from("<i", data, offset)[0]
@@ -713,10 +718,10 @@ class LineRecord:
     -----
     **[LIKELY]/[UNKNOWN]** -- much less firmly established than
     `ChannelRecord`: only the name (relative +32) and category code
-    (relative +108) fields are decoded, and locating the table itself
-    (`find_line_table` below) is a heuristic scan rather than the
-    structurally-proven SUPER-anchor technique used for the channel
-    table. See docs/spec.md section 3.2 and docs/provenance/notes.md
+    (relative +108) fields are decoded. The table is located exactly
+    (`exact_line_table_start`, **[CONFIRMED]** on the corpus) and only
+    falls back to the `find_line_table` heuristic scan when that
+    cannot be validated. See docs/spec.md section 3.2 and docs/provenance/notes.md
     section 6.3.
 
     `eq=False` keeps the default identity-based `__eq__`/`__hash__`
@@ -785,23 +790,25 @@ def find_line_table(data: bytes, search_window: Tuple[int, Optional[int]] = (128
 
     Notes
     -----
+    **Prefer `exact_line_table_start`** (via `read_lines`): the line
+    table's start follows exactly from `lines_max` and the channel
+    table's position (docs/spec.md section 2.1), which this heuristic
+    predates. This scan is now only the fallback for a file where that
+    arithmetic cannot be validated.
+
     Unlike `find_channel_table`, there's no known default-name anchor
     (the line table has nothing analogous to the channel table's
-    "SUPER" user record immediately after it) and no confirmed header
-    field gives its start offset directly -- reconciling one with the
-    header's capacity fields was tried and didn't cleanly round-trip
-    (docs/provenance/log.md Session 1 section 1.16,
-    docs/provenance/notes.md section 6.3). This is therefore a
-    heuristic **[LIKELY]** scan, not the structurally-proven technique
-    used for the channel table: it looks for a run of 128-byte records
+    "SUPER" user record immediately after it), so this is a
+    heuristic **[LIKELY]** scan, not a structurally-proven technique:
+    it looks for a run of 128-byte records
     whose relative +32 field looks like a clean, NUL-terminated,
     printable line name and whose relative +108 category field
     matches one of the two confirmed real values (100=NORMAL/FLIGHT,
     200=GROUP), then returns the earliest such record in the run with
     the most hits at a consistent 128-byte phase.
 
-    **Known limitation, found by real-file testing, not yet fixed
-    here:** if a table's true first slot(s) don't carry a category
+    **Known limitation, found by real-file testing (fixed by using
+    `exact_line_table_start` instead, not here):** if a table's true first slot(s) don't carry a category
     code in {100, 200}, this returns a start that's one or more slots
     too late -- every subsequent `LineRecord.index` is then off by
     that same fixed amount, which breaks blob_index lookups by line
@@ -847,6 +854,60 @@ def find_line_table(data: bytes, search_window: Tuple[int, Optional[int]] = (128
     return min(phase_hits[best_phase])
 
 
+_LINE_TABLE_GAP = 24  # [CONFIRMED] bytes between the end of the line table and the
+                      # channel table -- docs/spec.md section 2.1
+
+
+def exact_line_table_start(data: bytes, lines_max: Optional[int]) -> Optional[int]:
+    """
+    Compute the line table's start from the header and the channel table.
+
+    Parameters
+    ----------
+    data : bytes
+        The file's bytes up to at least the end of the channel table
+        (`read_lines` passes everything before the blob region).
+    lines_max : int or None
+        The line-table capacity (header word 36); `None` or non-positive
+        means unknown.
+
+    Returns
+    -------
+    int or None
+        Byte offset of the line table's first record, or `None` if the
+        arithmetic cannot be validated (unknown `lines_max`, no
+        locatable channel table, a start before the header ends, or no
+        line-shaped record anywhere in the computed table) -- the
+        caller then falls back to `find_line_table`.
+
+    Notes
+    -----
+    The line table is `lines_max` 128-byte slots followed by a 24-byte
+    gap and then the channel table, so its start is
+    `find_channel_table(data) - 24 - lines_max * 128`. **[CONFIRMED]**
+    on all 23 real files examined (docs/spec.md section 2.1;
+    docs/provenance/notes.md section 6.1b): the computed start is
+    always a slot boundary that matches the heuristic's start, or is an
+    earlier slot the heuristic missed. The at-least-one-line-record
+    check only guards a file whose layout does not follow this
+    arithmetic; it is not needed for any real file seen.
+    """
+    if not lines_max or lines_max <= 0:
+        return None
+    try:
+        channel_start = find_channel_table(data)
+    except ValueError:
+        return None
+    start = channel_start - _LINE_TABLE_GAP - lines_max * SYMBOL_RECORD_SIZE
+    if start < 256:
+        return None
+    for i in range(lines_max):
+        rec = _parse_line_record(data, start + i * SYMBOL_RECORD_SIZE, i)
+        if rec.name and rec.name_is_clean and rec.category_code in DB_CATEGORY_LINE_NAMES:
+            return start
+    return None
+
+
 def read_lines(path: str) -> List[LineRecord]:
     """
     Decode the line symbol table.
@@ -869,30 +930,59 @@ def read_lines(path: str) -> List[LineRecord]:
 
     Notes
     -----
-    Heuristic (see `find_line_table`) -- less firmly established than
-    `read_channels`. Since no confirmed header field gives the line
-    table's slot capacity (the way `chans_max` does for the channel
-    table), this reads forward from the located start until 8
+    The table is located **exactly** whenever possible (see
+    `exact_line_table_start`): it holds `lines_max` (header word 36)
+    128-byte slots and ends 24 bytes before the channel table, so its
+    start is `channel_table_start - 24 - lines_max * 128`
+    (**[CONFIRMED]** on every real file in the corpus, docs/spec.md
+    section 2.1). Every one of the `lines_max` slots is then examined,
+    and `LineRecord.index` is the true slot number, so
+    `LineRecord.index` is exact and blob_index lookups keyed on it
+    are right.
+
+    Only when that arithmetic cannot be validated (a header without
+    `lines_max`, or no line-shaped record at the computed position) does
+    this fall back to the older heuristic (see `find_line_table`):
+    **[LIKELY]**, less firmly established, reading forward until 8
     consecutive records fail to look like either a populated line
-    record or clean unused capacity -- a tolerance against one-off
-    corruption/false-positive records, not a precisely-known table
-    boundary.
-
-    **`LineRecord.index` can be off by a small, fixed amount** on a
-    file where `find_line_table`'s heuristic starts one or more slots
-    late -- see that function's docstring. This makes `.name` still
-    correct but `.index` (and therefore any blob_index lookup keyed on
-    it) wrong. `GDB` (in `gdb.py`) corrects this against the actual
-    blob chain before exposing lines by name; call it instead of this
-    function directly when you need working (line, channel) data
-    access, not just a list of names.
+    record or clean unused capacity. In that fallback
+    **`LineRecord.index` can be off by a small, fixed amount** when the
+    scan starts one or more slots late -- `.name` is still correct but
+    `.index` is wrong. `GDB` (in `gdb.py`) corrects this against the
+    actual blob chain only in that case.
+
+    A populated slot whose category is neither `100` nor `200` (for
+    example the `65636` slot 0 of one 1991 file) is not returned, as
+    before -- but, unlike the fallback, it no longer shifts the
+    indices of the lines after it.
+    """
+    return _read_lines(path)[0]
+
+
+def _read_lines(path: str) -> Tuple[List[LineRecord], bool]:
+    """
+    Decode the line symbol table, also reporting how it was located.
+
+    Parameters
+    ----------
+    path : str
+        Path to the `.gdb` file.
+
+    Returns
+    -------
+    lines : list of LineRecord
+        As `read_lines`.
+    exact : bool
+        True if the table was located by `exact_line_table_start`
+        (indices are exact); False if the heuristic fallback was used
+        (indices may need `GDB`'s blob-chain calibration).
     """
     with open(path, "rb") as f:
         header = f.read(4096)
         if not check_magic(header):
             _warn(f"{path}: does not start with the expected '!CBD' magic -- "
                   f"not a recognized .gdb file, returning no lines")
-            return []
+            return [], False
         blob_start = blob_region_start(header)
         f.seek(0, 2)
         size = f.tell()
@@ -900,14 +990,24 @@ def read_lines(path: str) -> List[LineRecord]:
         read_size = blob_start if (blob_start is not None and 0 < blob_start <= size) else min(size, 20_000_000)
         data = f.read(read_size)
 
+    lines_max = header_fields(header).get("lines_max")
+    exact_start = exact_line_table_start(data, lines_max)
+    if exact_start is not None:
+        lines = []
+        for i in range(lines_max):
+            rec = _parse_line_record(data, exact_start + i * SYMBOL_RECORD_SIZE, i)
+            if rec.name and rec.name_is_clean and rec.category_code in DB_CATEGORY_LINE_NAMES:
+                lines.append(rec)
+        return lines, True
+
     try:
         table_start = find_line_table(data, search_window=(128, len(data)))
     except ValueError as e:
         _warn(f"{path}: could not locate the line symbol table ({e}) -- "
               f"returning no lines")
-        return []
+        return [], False
 
-    lines: List[LineRecord] = []
+    lines = []
     consecutive_bad = 0
     i = 0
     while True:
@@ -929,7 +1029,7 @@ def read_lines(path: str) -> List[LineRecord]:
             consecutive_bad += 1
             if consecutive_bad >= 8:
                 break
-    return lines
+    return lines, False
 
 
 BLOB_MAGIC = b"\xcc\xcc\x00\xff"
@@ -1301,6 +1401,145 @@ def find_blob(path: str, line_slot: int, channel_slot: int, chans_max: Optional[
     return None
 
 
+DIRECTORY_OFFSET = 280  # [CONFIRMED] first blob-directory slot -- docs/spec.md section 2.2
+_DIRECTORY_SLOT_DTYPE = np.dtype([("word", "<u4"), ("n_pages", "<u2")])  # 6 bytes, unaligned
+_DIRECTORY_LIVE_FLAG = 0x8  # top nibble of a live (line, channel) entry's 32-bit word
+
+DIRECTORY_LIVE = "live"
+DIRECTORY_ABSENT = "absent"
+DIRECTORY_INVALID = "invalid"
+DIRECTORY_OUTSIDE = "outside"
+
+
+@dataclass(eq=False)
+class BlobDirectory:
+    """
+    The persisted blob directory: which blob is the live one for each slot.
+
+    Attributes
+    ----------
+    data_slots : int
+        Number of directory slots that address (line, channel) data
+        blobs (header word 48, `lines_max * chans_max`); slot `i` is
+        the blob whose `blob_index` is `i`.
+    entries : dict of {int : (int, int)}
+        Every **non-zero** data slot as `(32-bit word, n_pages)`, keyed
+        by slot. A zero slot is simply absent from this dict.
+
+    Notes
+    -----
+    **[CONFIRMED]** layout, **[LIKELY]** interpretation
+    (docs/spec.md section 2.2; docs/provenance/notes.md section 6.1c).
+    The directory is an array of 6-byte slots starting at file offset
+    280. A live entry is `(0x80000000 | start page, n_pages)` where
+    the start page is relative to the first blob (`(offset - blob
+    region start) / page_size`). Across the real corpus it addressed
+    100% of the real (line, channel) blobs of 20 of 22 files, and for
+    every duplicated pair with an independent oracle (13 of 13) it
+    pointed at the correct copy. A slot that is all-zero belongs to a
+    blob the file does not list as live.
+    """
+
+    data_slots: int
+    entries: dict
+
+    def resolve(self, blob_index: int, blobs_by_offset: dict, first_offset: int, page_size: int):
+        """
+        Look up the live blob for `blob_index`.
+
+        Parameters
+        ----------
+        blob_index : int
+            The slot, `line_slot * chans_max + channel_slot`.
+        blobs_by_offset : dict of {int : BlobHeader}
+            Every blob of the chain walk, keyed by absolute `offset`.
+        first_offset : int
+            Absolute offset of the first blob (`blob_region_start`).
+        page_size : int
+            The file's page size.
+
+        Returns
+        -------
+        status : str
+            `"live"`: the entry's start page lands on a walked blob
+            header whose `blob_index` equals `blob_index` and whose
+            `n_pages` equals the entry's page count (the strict
+            check). `"absent"`: the slot is all-zero. `"invalid"`: the
+            slot is non-zero but fails the strict check. `"outside"`:
+            `blob_index` is not a data slot at all (an administrative
+            slot), so the directory says nothing about it.
+        blob : BlobHeader or None
+            The live blob for `"live"`, otherwise `None`.
+        """
+        if not 0 <= blob_index < self.data_slots:
+            return DIRECTORY_OUTSIDE, None
+        entry = self.entries.get(blob_index)
+        if entry is None:
+            return DIRECTORY_ABSENT, None
+        word, n_pages = entry
+        if word >> 28 != _DIRECTORY_LIVE_FLAG:
+            return DIRECTORY_INVALID, None
+        blob = blobs_by_offset.get(first_offset + (word & 0x7FFFFFFF) * page_size)
+        if blob is None or blob.blob_index != blob_index or blob.n_pages != n_pages:
+            return DIRECTORY_INVALID, None
+        return DIRECTORY_LIVE, blob
+
+
+def read_blob_directory(path: str) -> Optional[BlobDirectory]:
+    """
+    Read the persisted blob directory.
+
+    Parameters
+    ----------
+    path : str
+        Path to the `.gdb` file.
+
+    Returns
+    -------
+    BlobDirectory or None
+        The directory, or `None` when the file has none to trust: a bad
+        magic or truncated header, header words that are inconsistent
+        with the layout (`data_slots != lines_max * chans_max`, or the
+        data slots would run past the blob region), or a directory
+        whose every data slot is zero. Absence is not an anomaly, so
+        nothing is warned.
+
+    Notes
+    -----
+    See `BlobDirectory`. Only the data slots (`0 <= slot <
+    data_slots`) are read; the registry blob-symbol slots and the
+    cache slots after them are not used by the reader.
+    """
+    with open(path, "rb") as f:
+        header = f.read(4096)
+        if not check_magic(header):
+            return None
+        fields = header_fields(header)
+        chans_max, lines_max, data_slots, page_size = (
+            fields["chans_max"], fields["lines_max"], fields["data_slots"], fields["page_size"],
+        )
+        if None in (chans_max, lines_max, data_slots, page_size):
+            return None
+        blob_start = blob_region_start(header)
+        if (
+            blob_start is None or data_slots <= 0 or data_slots != lines_max * chans_max
+            or DIRECTORY_OFFSET + data_slots * _DIRECTORY_SLOT_DTYPE.itemsize > blob_start
+        ):
+            return None
+        f.seek(DIRECTORY_OFFSET)
+        raw = f.read(data_slots * _DIRECTORY_SLOT_DTYPE.itemsize)
+    if len(raw) != data_slots * _DIRECTORY_SLOT_DTYPE.itemsize:
+        return None
+    slots = np.frombuffer(raw, dtype=_DIRECTORY_SLOT_DTYPE)
+    nonzero = np.nonzero((slots["word"] != 0) | (slots["n_pages"] != 0))[0]
+    if len(nonzero) == 0:
+        return None
+    entries = {
+        int(i): (int(slots["word"][i]), int(slots["n_pages"][i])) for i in nonzero
+    }
+    return BlobDirectory(data_slots=data_slots, entries=entries)
+
+
 def _element_width(channel: ChannelRecord) -> Optional[int]:
     """
     Byte width of one element of `channel`'s data.

{python_gdb-0.2.1 → python_gdb-0.3.0}/rust/Cargo.lock

@@ -46,7 +46,7 @@ dependencies = [
 
 [[package]]
 name = "pygdb-native"
-version = "0.2.1"
+version = "0.3.0"
 dependencies = [
  "flate2",
  "pyo3",

{python_gdb-0.2.1 → python_gdb-0.3.0}/rust/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "pygdb-native"
-version = "0.2.1"
+version = "0.3.0"
 edition = "2021"
 description = "Optional Rust-accelerated backend for python-gdb (pygdb._native)"
 license = "MIT"