python-gdb 0.3.0__tar.gz → 0.4.0__tar.gz

{python_gdb-0.3.0 → python_gdb-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python-gdb
-Version: 0.3.0
+Version: 0.4.0
 Classifier: Development Status :: 3 - Alpha
 Classifier: Intended Audience :: Science/Research
 Classifier: Topic :: Scientific/Engineering :: GIS

{python_gdb-0.3.0 → python_gdb-0.4.0}/pygdb/__init__.py

@@ -29,38 +29,61 @@ from .gdb_reader import (
     header_fields,
     iter_blobs,
     read_blob_directory,
+    read_blob_symbols,
     read_blob_values,
     read_channels,
     read_lines,
 )
 from .grd_reader import GrdHeader, GRDParseWarning, read_grd
 from .lzrw1 import LZRW1DecodeError
-from .registry import find_coordinate_systems
+from .registry import (
+    ChannelMaker,
+    DisplayListEntry,
+    ProjectionParameters,
+    find_channel_makers,
+    find_channel_roles,
+    find_channel_settings,
+    find_coordinate_systems,
+    find_display_lists,
+    find_projection_parameters,
+)
+from .unseen import GDBUnseenFeatureWarning, unseen_feature_report
 
 __all__ = [
     "GDB",
     "BlobDirectory",
     "BlobHeader",
+    "ChannelMaker",
     "ChannelRecord",
     "CompressionInfo",
+    "DisplayListEntry",
     "GDBParseWarning",
+    "GDBUnseenFeatureWarning",
     "GRDParseWarning",
     "GrdHeader",
     "LZRW1DecodeError",
     "LineRecord",
+    "ProjectionParameters",
     "check_magic",
     "exact_line_table_start",
     "find_blob",
+    "find_channel_makers",
+    "find_channel_roles",
+    "find_channel_settings",
     "find_channel_table",
     "find_coordinate_systems",
+    "find_display_lists",
     "find_line_table",
+    "find_projection_parameters",
     "header_fields",
     "iter_blobs",
     "read_blob_directory",
+    "read_blob_symbols",
     "read_blob_values",
     "read_channels",
     "read_grd",
     "read_lines",
+    "unseen_feature_report",
 ]
 
 from importlib.metadata import PackageNotFoundError, version

{python_gdb-0.3.0 → python_gdb-0.4.0}/pygdb/gdb.py

@@ -27,6 +27,7 @@ from .gdb_reader import (
     GS_TYPE_DUMMY_VALUE,
     GS_TYPE_NUMPY_DTYPE,
     LineRecord,
+    blob_region_start,
     check_magic,
     header_fields,
     iter_blobs,
@@ -38,7 +39,18 @@ from .gdb_reader import (
     DIRECTORY_INVALID,
     DIRECTORY_LIVE,
 )
-from .registry import find_channel_roles, find_coordinate_systems
+from .registry import (
+    ChannelMaker,
+    DisplayListEntry,
+    ProjectionParameters,
+    find_channel_makers,
+    find_channel_roles,
+    find_channel_settings,
+    find_coordinate_systems,
+    find_display_lists,
+    find_projection_parameters,
+)
+from . import unseen
 
 # docs/spec.md section 7
 _DB_COMP_NAMES = {
@@ -176,6 +188,8 @@ class GDB:
                 f"not a recognized .gdb file"
             )
         self._fields = header_fields(header)
+        unseen.check_header(path, header)
+        unseen.check_admin_objects(path)
         self._channels: Optional[List[ChannelRecord]] = None
         self._lines: Optional[List[LineRecord]] = None
         self._lines_exact = False
@@ -184,6 +198,10 @@ class GDB:
         self._blob_index: Optional[Dict[Tuple[int, int], BlobHeader]] = None
         self._coordinate_systems: Optional[List[str]] = None
         self._coordinate_channels: Optional[Dict[str, Optional[str]]] = None
+        self._channel_settings: Optional[Dict[str, Dict[str, str]]] = None
+        self._projection_parameters: Optional[Dict[str, ProjectionParameters]] = None
+        self._channel_makers: Optional[Dict[str, ChannelMaker]] = None
+        self._display_lists: Optional[List[List[DisplayListEntry]]] = None
 
     def __repr__(self) -> str:
         return f"GDB({self.path!r})"
@@ -273,6 +291,68 @@ class GDB:
             )
         return self._coordinate_channels
 
+    @property
+    def channel_settings(self) -> Dict[str, Dict[str, str]]:
+        """
+        dict of {str : dict of {str : str}}: Real per-channel settings
+        recorded in this file's own internal registry
+        (docs/provenance/notes.md section 6.8c), e.g. `{"raw_mag":
+        {"UNITS": "nT"}, ...}`. Only a channel with at least one
+        populated registry key is present; most real files have none for
+        most channels -- see `pygdb.registry.find_channel_settings` for
+        what is and isn't decoded (the registry's key/value entries, not
+        its nested `MAKER` records, which are `channel_makers`).
+        """
+        if self._channel_settings is None:
+            self._channel_settings = find_channel_settings(self.path, channels=self.channels)
+        return self._channel_settings
+
+    @property
+    def channel_makers(self) -> Dict[str, ChannelMaker]:
+        """
+        dict of {str : ChannelMaker}: How each channel was made, from the
+        `MAKER` records in this file's registry (docs/spec.md section 9) --
+        the tool (`"newchan.gx"`, a math expression, `"newxy.gx"`, ...),
+        its label, and the parameters it ran with, e.g. a derived
+        channel's formula. Only channels with a record are present; see
+        `pygdb.registry.find_channel_makers`.
+        """
+        if self._channel_makers is None:
+            self._channel_makers = find_channel_makers(self.path, channels=self.channels)
+        return self._channel_makers
+
+    @property
+    def display_lists(self) -> List[List[DisplayListEntry]]:
+        """
+        list of list of DisplayListEntry: The file's `Display List` objects
+        (docs/spec.md section 9) -- **[LIKELY]** the channels shown in the
+        database's spreadsheet view -- each entry resolved by handle to the
+        channel's current name. See `pygdb.registry.find_display_lists`.
+        """
+        if self._display_lists is None:
+            self._display_lists = find_display_lists(self.path, channels=self.channels)
+        return self._display_lists
+
+    @property
+    def projection_parameters(self) -> Dict[str, ProjectionParameters]:
+        """
+        dict of {str : ProjectionParameters}: Real geodetic parameters for
+        each coordinate system named in `coordinate_systems`, decoded from
+        this file's own internal IPJ registry (docs/spec.md section 8,
+        docs/provenance/notes.md section 6.7b) -- datum, ellipsoid, and
+        (when this coordinate system is a projected one) central
+        meridian, scale factor, and false easting/northing. An empty
+        dict just means no `IPJ` content was found, same as
+        `coordinate_systems`; see `pygdb.registry.find_projection_parameters`
+        for exactly what is and isn't decoded.
+        """
+        if self._projection_parameters is None:
+            max_real_line_slot = max((line.index for line in self.lines), default=-1)
+            self._projection_parameters = find_projection_parameters(
+                self.path, max_real_line_slot=max_real_line_slot,
+            )
+        return self._projection_parameters
+
     # -- channels / lines ----------------------------------------------------
 
     @property
@@ -532,7 +612,13 @@ class GDB:
                     f"against the line's other channels, this is the likely cause (see issue #2)"
                 )
             return
-        first_offset = min(by_offset)
+        # Directory start pages count from the blob region's start (header
+        # word 108), not from the first blob walked: a real file can hold a
+        # page there that is not a blob (docs/spec.md section 6.2).
+        with open(self.path, "rb") as f:
+            first_offset = blob_region_start(f.read(128))
+        if first_offset is None:
+            first_offset = min(by_offset)
         skipped: List[Tuple[int, int]] = []
         invalid: List[Tuple[int, int]] = []
         for key in copies:

{python_gdb-0.3.0 → python_gdb-0.4.0}/pygdb/gdb_reader.py

@@ -50,11 +50,12 @@ import warnings
 import zlib
 from dataclasses import dataclass
 
-from typing import BinaryIO, List, Optional, Tuple
+from typing import BinaryIO, Dict, List, Optional, Tuple
 
 import numpy as np
 
 from . import lzrw1 as _lzrw1
+from . import unseen
 
 try:
     from . import _native as _native_ext
@@ -330,12 +331,24 @@ class ChannelRecord:
             printable name (observed for real in DB_Mag_833.gdb -- see
             docs/provenance/notes.md). Real records seen so far always
             have dtype either a known GS_* code (0-13) or a small
-            negative string width, and a format code in the known
-            DB_CHAN_FORMAT_* range (0-6).
+            negative string width, a format code in the known
+            DB_CHAN_FORMAT_* range (0-6), and an array width of at least
+            1 (elements per fiducial).
+
+        Notes
+        -----
+        The array-width test catches leftover records that pass the
+        other two: 16 records in three real GSQ files (`DB_AGG_1213`,
+        `DB_Mag_1213`, `DB_Mag_1212`) hold clean names -- fill-pattern
+        names, projection-catalog names, second copies of real channel
+        names -- with dtype 0 and array width 0 (or garbage), and own no
+        data blob at all. Every genuine channel in the corpus has width
+        1 or more (docs/provenance/notes.md section 6.2d).
         """
         dtype_ok = self.dtype_code in GS_TYPE_NAMES or -256 <= self.dtype_code < 0
         format_ok = self.format_code in DB_CHAN_FORMAT_NAMES
-        return dtype_ok and format_ok
+        width_ok = self.array_width >= 1
+        return dtype_ok and format_ok and width_ok
 
 
 def _read_name(raw: bytes, offset: int, max_len: int = 64):
@@ -400,7 +413,7 @@ def check_magic(data: bytes) -> bool:
     The full 16-byte `HEADER_SIGNATURE` is only **[LIKELY]** -- it
     matched exactly in 8/9 real files, but one real file
     (DB_Mag_Elaine_1003.gdb, from GSQ's Mount Gordon delivery) has
-    `f0 f0 f0 f0` at bytes 8-11 instead of the usual `00 00 00 00`.
+    `f0 f0 f0 f0` at bytes 4-7 instead of the usual `00 00 00 00`.
     That file is otherwise structurally normal
     (chans_max/users_max/page_size all decode sanely), so this looks
     like a real, if rare, variation in that sub-block rather than a
@@ -679,6 +692,10 @@ def read_channels(path: str) -> List[ChannelRecord]:
             # DB_CHAN_FORMAT_* range -- garbage doesn't. See docs/provenance/notes.md.
             continue
         channels.append(rec)
+    unseen.check_channels(path, channels)
+    if fields["users_max"] is not None:
+        # The user table directly follows the channel table (docs/spec.md section 2.1).
+        unseen.check_users(path, data, table_start + chans_max * SYMBOL_RECORD_SIZE, fields["users_max"])
     return channels
 
 
@@ -998,6 +1015,7 @@ def _read_lines(path: str) -> Tuple[List[LineRecord], bool]:
             rec = _parse_line_record(data, exact_start + i * SYMBOL_RECORD_SIZE, i)
             if rec.name and rec.name_is_clean and rec.category_code in DB_CATEGORY_LINE_NAMES:
                 lines.append(rec)
+        unseen.check_lines(path, data, lines)
         return lines, True
 
     try:
@@ -1189,12 +1207,32 @@ def blob_region_start(data: bytes) -> Optional[int]:
     return start_page * page_size
 
 
+def _next_blob_page(f: BinaryIO, start: int, page_size: int, size: int) -> Optional[int]:
+    """Offset of the first page boundary at or after `start` whose first
+    bytes are the blob magic, or `None` if there is none before `size`
+    minus one header. Reads in large chunks rather than seeking per page."""
+    chunk_pages = max(1, (1 << 20) // page_size)
+    pos = start
+    while pos + BLOB_HEADER_SIZE <= size:
+        f.seek(pos)
+        buf = f.read(chunk_pages * page_size)
+        if not buf:
+            return None
+        for i in range(0, len(buf), page_size):
+            if buf[i:i + 4] == BLOB_MAGIC and pos + i + BLOB_HEADER_SIZE <= size:
+                return pos + i
+        pos += len(buf)
+    return None
+
+
 def iter_blobs(path: str, max_blobs: Optional[int] = None):
     """
     Walk the self-describing blob chain.
 
     Walks from the start of the blob region to end of file (or
-    `max_blobs`, or the first framing anomaly).
+    `max_blobs`, or a framing anomaly it cannot step past). A page that
+    does not start a blob is skipped: the walk resumes at the next page
+    that does.
 
     Parameters
     ----------
@@ -1212,7 +1250,9 @@ def iter_blobs(path: str, max_blobs: Optional[int] = None):
     -----
     GDBParseWarning
         Reaching the file's true end cleanly is silent (the expected,
-        common case), but a magic mismatch, a non-positive `n_pages`,
+        common case). Pages skipped because they do not start a blob get
+        one summary warning (count and offsets). A magic mismatch with no
+        later blob page, a non-positive `n_pages`,
         a file that ends mid-header, or landing short of true EOF by
         less than one full header (i.e. real leftover bytes, not
         enough to be read at all) are all real anomalies and each gets
@@ -1230,7 +1270,12 @@ def iter_blobs(path: str, max_blobs: Optional[int] = None):
     on the true file size) on 20 real files spanning all 3 agencies
     this project has files from and all three `DB_COMP_*` compression
     modes, 2MB to 1.93GB -- see docs/provenance/notes.md section
-    6.6b/6.6d/6.9.
+    6.6b/6.6d/6.9. One later real file (OpenEI BRIDGE
+    `GP_Master_Gravity_11082023.gdb`, 2023) has pages between blobs
+    that are not blobs: one page of leftover data at the region start
+    and a run of 96 all-zero pages. Its blobs are still contiguous
+    around those gaps, and the resynchronizing walk lands exactly on
+    the file's end.
 
     As a generator, this already "returns partial results" in the most
     natural way possible: whatever's been yielded before a problem is
@@ -1261,6 +1306,7 @@ def iter_blobs(path: str, max_blobs: Optional[int] = None):
             return
         f.seek(off)
         n = 0
+        skipped_runs = []  # [(first offset, page count)] of pages that are not blobs
         while off + BLOB_HEADER_SIZE <= size:
             if max_blobs is not None and n >= max_blobs:
                 return
@@ -1275,17 +1321,25 @@ def iter_blobs(path: str, max_blobs: Optional[int] = None):
                         f"walked -- file is likely truncated; returning the "
                         f"{n} blob(s) already yielded"
                     )
-                else:
+                    return
+                # A page that is not a blob header. Real files can hold
+                # such pages between blobs -- leftover data or never-written
+                # zero pages (docs/spec.md section 6.2) -- so resynchronize on
+                # the next page that starts with the blob magic.
+                resume = _next_blob_page(f, off + page_size, page_size, size)
+                if resume is None:
                     _warn(
                         f"{path}: blob magic mismatch at offset {off} "
                         f"(got {raw[:4].hex()}, expected {BLOB_MAGIC.hex()}) "
-                        f"after {n} blob(s) successfully walked -- stopping "
-                        f"the chain walk here and returning the {n} blob(s) "
-                        f"already yielded; this may be a real structural "
-                        f"anomaly or an administrative-blob variant not yet "
-                        f"understood (docs/provenance/notes.md section 6.4/6.9)"
+                        f"after {n} blob(s) successfully walked, and no later "
+                        f"page starts a blob -- stopping the chain walk here "
+                        f"and returning the {n} blob(s) already yielded"
                     )
-                return
+                    return
+                skipped_runs.append((off, (resume - off) // page_size))
+                off = resume
+                f.seek(off)
+                continue
             if blob.n_pages <= 0:
                 _warn(
                     f"{path}: blob at offset {off} (blob_index={blob.blob_index}) "
@@ -1312,6 +1366,15 @@ def iter_blobs(path: str, max_blobs: Optional[int] = None):
             f.seek(skip, 1)
             off += blob.n_pages * page_size
             n += 1
+        if skipped_runs:
+            pages = sum(count for _start, count in skipped_runs)
+            runs = ", ".join(f"{count} at offset {start}" for start, count in skipped_runs[:5])
+            more = f" and {len(skipped_runs) - 5} more run(s)" if len(skipped_runs) > 5 else ""
+            _warn(
+                f"{path}: skipped {pages} page(s) of the blob region that do not "
+                f"start a blob ({runs}{more}) and resumed the chain walk after "
+                f"each -- leftover or never-written pages between blobs"
+            )
         if n > 0 and off > size:
             # The last blob successfully parsed claimed a page count that
             # implies more data than the file actually contains -- off
@@ -1403,7 +1466,11 @@ def find_blob(path: str, line_slot: int, channel_slot: int, chans_max: Optional[
 
 DIRECTORY_OFFSET = 280  # [CONFIRMED] first blob-directory slot -- docs/spec.md section 2.2
 _DIRECTORY_SLOT_DTYPE = np.dtype([("word", "<u4"), ("n_pages", "<u2")])  # 6 bytes, unaligned
-_DIRECTORY_LIVE_FLAG = 0x8  # top nibble of a live (line, channel) entry's 32-bit word
+# A live entry's 32-bit word has the 0x80000000 bit set; the 0x40000000
+# bit flips each time the blob is rewritten (docs/spec.md section 2.2), so
+# the top nibble is 0x8 or 0xC. The remaining 30 bits are the start page.
+_DIRECTORY_LIVE_BIT = 0x80000000
+_DIRECTORY_PAGE_MASK = 0x3FFFFFFF
 
 DIRECTORY_LIVE = "live"
 DIRECTORY_ABSENT = "absent"
@@ -1433,7 +1500,11 @@ class BlobDirectory:
     The directory is an array of 6-byte slots starting at file offset
     280. A live entry is `(0x80000000 | start page, n_pages)` where
     the start page is relative to the first blob (`(offset - blob
-    region start) / page_size`). Across the real corpus it addressed
+    region start) / page_size`), optionally with the `0x40000000` bit,
+    which flips each time the blob is rewritten. (Every data entry in
+    the first 22 corpus files read `0x8`; the first data entry with
+    `0xC`, in USGS OFR 2011-1270 `Kalay_nk.gdb`, is the live copy -- the
+    other copy is on the free list.) Across the real corpus it addressed
     100% of the real (line, channel) blobs of 20 of 22 files, and for
     every duplicated pair with an independent oracle (13 of 13) it
     pointed at the correct copy. A slot that is all-zero belongs to a
@@ -1477,9 +1548,9 @@ class BlobDirectory:
         if entry is None:
             return DIRECTORY_ABSENT, None
         word, n_pages = entry
-        if word >> 28 != _DIRECTORY_LIVE_FLAG:
+        if not word & _DIRECTORY_LIVE_BIT:
             return DIRECTORY_INVALID, None
-        blob = blobs_by_offset.get(first_offset + (word & 0x7FFFFFFF) * page_size)
+        blob = blobs_by_offset.get(first_offset + (word & _DIRECTORY_PAGE_MASK) * page_size)
         if blob is None or blob.blob_index != blob_index or blob.n_pages != n_pages:
             return DIRECTORY_INVALID, None
         return DIRECTORY_LIVE, blob
@@ -1540,6 +1611,77 @@ def read_blob_directory(path: str) -> Optional[BlobDirectory]:
     return BlobDirectory(data_slots=data_slots, entries=entries)
 
 
+# Blob-symbol record fields, measured from the record's first byte (its
+# name) -- docs/spec.md section 2.1, docs/provenance/notes.md section 6.2d.
+_BLOB_SYMBOL_CATEGORY_OFFSET = 76
+_DB_CATEGORY_BLOB_NORMAL = 0  # vendor constant; the live-symbol category
+
+
+def read_blob_symbols(path: str) -> Optional[Dict[int, str]]:
+    """
+    Read the names of the file's live administrative objects.
+
+    Parameters
+    ----------
+    path : str
+        Path to the `.gdb` file.
+
+    Returns
+    -------
+    dict of {int : str} or None
+        `{symbol_slot: name}` for every live blob symbol. The object
+        named by slot `k` is the administrative blob whose `blob_index`
+        is `data_slots + k` (`header_fields()["data_slots"]`). `None`
+        when the file has no table to trust: a bad magic or truncated
+        header, or header words that would put the table outside the
+        region before the first blob. Absence is not an anomaly, so
+        nothing is warned.
+
+    Notes
+    -----
+    **[CONFIRMED]** layout (docs/spec.md section 2.1,
+    docs/provenance/notes.md section 6.2d): `blobs_max` 128-byte records
+    starting right after the blob directory, at `280 + 6 * index_slots`,
+    each beginning with its NUL-terminated name. A record is live when
+    its category (record `+76`) is `DB_CATEGORY_BLOB_NORMAL` (0), which
+    held on exactly the 1,267 corpus records that own an administrative
+    blob. A freed slot has bit `0x10000` set and may still hold its old
+    name, so the category, not the name, decides.
+
+    Names seen on real files: four fixed objects (`"__dbreg"`,
+    `"Display List"`, `"Line Selection"`, `"Database Extension
+    Objects"`), `"?|IPJ_<X>:<Y>"` projection objects, and `"__<n>"` REG
+    objects where `n` is a global symbol handle (`blobs_max + lines_max
+    + channel_slot` for a channel).
+    """
+    with open(path, "rb") as f:
+        header = f.read(4096)
+        if not check_magic(header):
+            return None
+        fields = header_fields(header)
+        blobs_max, index_slots = fields["blobs_max"], fields["index_slots"]
+        if blobs_max is None or index_slots is None or blobs_max <= 0 or index_slots <= 0:
+            return None
+        blob_start = blob_region_start(header)
+        start = DIRECTORY_OFFSET + index_slots * _DIRECTORY_SLOT_DTYPE.itemsize
+        size = blobs_max * SYMBOL_RECORD_SIZE
+        if blob_start is None or start + size > blob_start:
+            return None
+        f.seek(start)
+        raw = f.read(size)
+    if len(raw) != size:
+        return None
+    symbols = {}
+    for slot in range(blobs_max):
+        rec = raw[slot * SYMBOL_RECORD_SIZE:(slot + 1) * SYMBOL_RECORD_SIZE]
+        if struct.unpack_from("<i", rec, _BLOB_SYMBOL_CATEGORY_OFFSET)[0] != _DB_CATEGORY_BLOB_NORMAL:
+            continue
+        name, is_clean = _read_name(rec, 0)
+        if name and is_clean:
+            symbols[slot] = name
+    return symbols
+
+
 def _element_width(channel: ChannelRecord) -> Optional[int]:
     """
     Byte width of one element of `channel`'s data.
@@ -1737,7 +1879,13 @@ def _decode_numeric_or_string(raw: bytes, channel: ChannelRecord, row_count: Opt
     if channel.is_string:
         if _native_ext is not None:
             max_len, ucs4 = _native_ext.decode_fixed_width_strings_ucs4(raw, width, n)
-            arr = np.frombuffer(ucs4, dtype=f"<U{max_len}")
+            if max_len == 0:
+                # Every record is empty: there is no zero-width numpy string
+                # dtype to view the (empty) buffer as. `np.array(..., "<U0")`
+                # in the fallback below becomes `<U1` the same way.
+                arr = np.zeros(n, dtype="<U1")
+            else:
+                arr = np.frombuffer(ucs4, dtype=f"<U{max_len}")
         else:
             values = [
                 raw[i * width : (i + 1) * width].split(b"\x00")[0].decode("ascii", errors="replace")