dissect.database 0.1.dev2__py3-none-any.whl

dissect/database/__init__.py

@@ -0,0 +1,5 @@
+from __future__ import annotations
+
+from dissect.database.bsd.db import DB
+
+__all__ = ["DB"]

dissect/database/bsd/__init__.py

@@ -0,0 +1,5 @@
+from __future__ import annotations
+
+from dissect.database.bsd.db import DB
+
+__all__ = ["DB"]

dissect/database/bsd/c_db.py

@@ -0,0 +1,598 @@
+# References:
+# - libdb
+from __future__ import annotations
+
+from dissect.cstruct import cstruct
+
+db_def = """
+typedef uint8 u_int8_t;
+typedef uint16 u_int16_t;
+typedef uint32 u_int32_t;
+
+typedef u_int32_t db_pgno_t;                    /* Page number type. */
+typedef u_int16_t db_indx_t;                    /* Page offset type. */
+#define DB_MAX_PAGES            0xffffffff      /* >= # of pages in a file */
+
+typedef u_int32_t db_recno_t;                   /* Record number type. */
+#define DB_MAX_RECORDS          0xffffffff      /* >= # of records in a tree */
+
+typedef struct _db_lsn {
+    uint32      file;
+    uint32      offset;
+} DB_LSN;
+
+#define DB_FILE_ID_LEN          20              /* Unique file ID length. */
+
+/*******************************************************
+ * Crypto.
+ *******************************************************/
+#define DB_IV_BYTES             16              /* Bytes per IV */
+#define DB_MAC_KEY              20              /* Bytes per MAC checksum */
+
+#define DB_RENAMEMAGIC          0x030800        /* File has been renamed. */
+
+#define DB_BTREEVERSION         9               /* Current btree version. */
+#define DB_BTREEOLDVER          8               /* Oldest btree version supported. */
+#define DB_BTREEMAGIC           0x053162
+
+#define DB_HASHVERSION          9               /* Current hash version. */
+#define DB_HASHOLDVER           7               /* Oldest hash version supported. */
+#define DB_HASHMAGIC            0x061561
+
+#define DB_HEAPVERSION          1               /* Current heap version. */
+#define DB_HEAPOLDVER           1               /* Oldest heap version supported. */
+#define DB_HEAPMAGIC            0x074582
+
+#define DB_QAMVERSION           4               /* Current queue version. */
+#define DB_QAMOLDVER            3               /* Oldest queue version supported. */
+#define DB_QAMMAGIC             0x042253
+
+#define DB_SEQUENCE_VERSION     2               /* Current sequence version. */
+#define DB_SEQUENCE_OLDVER      1               /* Oldest sequence version supported. */
+
+#define DB_AM_CHKSUM            0x00000001      /* Checksumming */
+#define DB_AM_COMPENSATE        0x00000002      /* Created by compensating txn */
+#define DB_AM_COMPRESS          0x00000004      /* Compressed BTree */
+#define DB_AM_CREATED           0x00000008      /* Database was created upon open */
+#define DB_AM_CREATED_MSTR      0x00000010      /* Encompassing file was created */
+#define DB_AM_DBM_ERROR         0x00000020      /* Error in DBM/NDBM database */
+#define DB_AM_DELIMITER         0x00000040      /* Variable length delimiter set */
+#define DB_AM_DISCARD           0x00000080      /* Discard any cached pages */
+#define DB_AM_DUP               0x00000100      /* DB_DUP */
+#define DB_AM_DUPSORT           0x00000200      /* DB_DUPSORT */
+#define DB_AM_ENCRYPT           0x00000400      /* Encryption */
+#define DB_AM_FIXEDLEN          0x00000800      /* Fixed-length records */
+#define DB_AM_INMEM             0x00001000      /* In-memory; no sync on close */
+#define DB_AM_INORDER           0x00002000      /* DB_INORDER */
+#define DB_AM_IN_RENAME         0x00004000      /* File is being renamed */
+#define DB_AM_NOT_DURABLE       0x00008000      /* Do not log changes */
+#define DB_AM_OPEN_CALLED       0x00010000      /* DB->open called */
+#define DB_AM_PAD               0x00020000      /* Fixed-length record pad */
+#define DB_AM_PARTDB            0x00040000      /* Handle for a database partition */
+#define DB_AM_PGDEF             0x00080000      /* Page size was defaulted */
+#define DB_AM_RDONLY            0x00100000      /* Database is readonly */
+#define DB_AM_READ_UNCOMMITTED  0x00200000      /* Support degree 1 isolation */
+#define DB_AM_RECNUM            0x00400000      /* DB_RECNUM */
+#define DB_AM_RECOVER           0x00800000      /* DB opened by recovery routine */
+#define DB_AM_RENUMBER          0x01000000      /* DB_RENUMBER */
+#define DB_AM_REVSPLITOFF       0x02000000      /* DB_REVSPLITOFF */
+#define DB_AM_SECONDARY         0x04000000      /* Database is a secondary index */
+#define DB_AM_SNAPSHOT          0x08000000      /* DB_SNAPSHOT */
+#define DB_AM_SUBDB             0x10000000      /* Subdatabases supported */
+#define DB_AM_SWAP              0x20000000      /* Pages need to be byte-swapped */
+#define DB_AM_TXN               0x40000000      /* Opened in a transaction */
+#define DB_AM_VERIFYING         0x80000000      /* DB handle is in the verifier */
+
+/*
+ * DB page formats.
+ *
+ * !!!
+ * This implementation requires that values within the following structures
+ * NOT be padded -- note, ANSI C permits random padding within structures.
+ * If your compiler pads randomly you can just forget ever making DB run on
+ * your system.  In addition, no data type can require larger alignment than
+ * its own size, e.g., a 4-byte data element may not require 8-byte alignment.
+ *
+ * Note that key/data lengths are often stored in db_indx_t's -- this is
+ * not accidental, nor does it limit the key/data size.  If the key/data
+ * item fits on a page, it's guaranteed to be small enough to fit into a
+ * db_indx_t, and storing it in one saves space.
+ */
+
+#define PGNO_INVALID            0               /* Invalid page number in any database. */
+#define PGNO_BASE_MD            0               /* Base database: metadata page number. */
+
+/* Page types. */
+#define P_INVALID               0               /* Invalid page type. */
+#define __P_DUPLICATE           1               /* Duplicate. DEPRECATED in 3.1 */
+#define P_HASH_UNSORTED         2               /* Hash pages created pre 4.6. DEPRECATED */
+#define P_IBTREE                3               /* Btree internal. */
+#define P_IRECNO                4               /* Recno internal. */
+#define P_LBTREE                5               /* Btree leaf. */
+#define P_LRECNO                6               /* Recno leaf. */
+#define P_OVERFLOW              7               /* Overflow. */
+#define P_HASHMETA              8               /* Hash metadata page. */
+#define P_BTREEMETA             9               /* Btree metadata page. */
+#define P_QAMMETA               10              /* Queue metadata page. */
+#define P_QAMDATA               11              /* Queue data page. */
+#define P_LDUP                  12              /* Off-page duplicate leaf. */
+#define P_HASH                  13              /* Sorted hash page. */
+#define P_HEAPMETA              14              /* Heap metadata page. */
+#define P_HEAP                  15              /* Heap data page. */
+#define P_IHEAP                 16              /* Heap internal. */
+#define P_PAGETYPE_MAX          17
+
+/*
+ * When we create pages in mpool, we ask mpool to clear some number of bytes
+ * in the header.  This number must be at least as big as the regular page
+ * headers and cover enough of the btree and hash meta-data pages to obliterate
+ * the page type.
+ */
+#define DB_PAGE_DB_LEN          32
+#define DB_PAGE_QUEUE_LEN       0
+
+#define DBMETA_CHKSUM           0x01
+#define DBMETA_PART_RANGE       0x02
+#define DBMETA_PART_CALLBACK    0x04
+
+/************************************************************************
+ GENERIC METADATA PAGE HEADER
+ *
+ * !!!
+ * The magic and version numbers have to be in the same place in all versions
+ * of the metadata page as the application may not have upgraded the database.
+ ************************************************************************/
+typedef struct _dbmeta33 {
+    DB_LSN      lsn;                            /* 00-07: LSN. */
+    db_pgno_t   pgno;                           /* 08-11: Current page number. */
+    u_int32_t   magic;                          /* 12-15: Magic number. */
+    u_int32_t   version;                        /* 16-19: Version. */
+    u_int32_t   pagesize;                       /* 20-23: Pagesize. */
+    u_int8_t    encrypt_alg;                    /*    24: Encryption algorithm. */
+    u_int8_t    type;                           /*    25: Page type. */
+    u_int8_t    metaflags;                      /* 26: Meta-only flags */
+    u_int8_t    unused1;                        /* 27: Unused. */
+    u_int32_t   free;                           /* 28-31: Free list page number. */
+    db_pgno_t   last_pgno;                      /* 32-35: Page number of last page in db. */
+    u_int32_t   nparts;                         /* 36-39: Number of partitions. */
+    u_int32_t   key_count;                      /* 40-43: Cached key count. */
+    u_int32_t   record_count;                   /* 44-47: Cached record count. */
+    u_int32_t   flags;                          /* 48-51: Flags: unique to each AM. */
+    u_int8_t    uid[DB_FILE_ID_LEN];            /* 52-71: Unique file ID. */
+} DBMETA33, DBMETA;
+
+/************************************************************************
+ BTREE METADATA PAGE LAYOUT
+ ************************************************************************/
+#define BTM_DUP                 0x001           /*   Duplicates. */
+#define BTM_RECNO               0x002           /*   Recno tree. */
+#define BTM_RECNUM              0x004           /*   Btree: maintain record count. */
+#define BTM_FIXEDLEN            0x008           /*   Recno: fixed length records. */
+#define BTM_RENUMBER            0x010           /*   Recno: renumber on insert/delete. */
+#define BTM_SUBDB               0x020           /*   Subdatabases. */
+#define BTM_DUPSORT             0x040           /*   Duplicates are sorted. */
+#define BTM_COMPRESS            0x080           /*   Compressed. */
+#define BTM_MASK                0x0ff
+
+typedef struct _btmeta33 {
+    DBMETA      dbmeta;                         /* 00-71: Generic meta-data header. */
+
+    u_int32_t   unused1;                        /* 72-75: Unused space. */
+    u_int32_t   minkey;                         /* 76-79: Btree: Minkey. */
+    u_int32_t   re_len;                         /* 80-83: Recno: fixed-length record length. */
+    u_int32_t   re_pad;                         /* 84-87: Recno: fixed-length record pad. */
+    u_int32_t   root;                           /* 88-91: Root page. */
+    u_int32_t   unused2[92];                    /* 92-459: Unused space. */
+    u_int32_t   crypto_magic;                   /* 460-463: Crypto magic number */
+    u_int32_t   trash[3];                       /* 464-475: Trash space - Do not use */
+    u_int8_t    iv[DB_IV_BYTES];                /* 476-495: Crypto IV */
+    u_int8_t    chksum[DB_MAC_KEY];             /* 496-511: Page chksum */
+
+    /*
+     * Minimum page size is 512.
+     */
+} BTMETA33, BTMETA;
+
+/************************************************************************
+ HASH METADATA PAGE LAYOUT
+ ************************************************************************/
+#define DB_HASH_DUP             0x01            /*   Duplicates. */
+#define DB_HASH_SUBDB           0x02            /*   Subdatabases. */
+#define DB_HASH_DUPSORT         0x04            /*   Duplicates are sorted. */
+#define NCACHED                 32              /* number of spare points */
+
+typedef struct _hashmeta33 {
+    DBMETA      dbmeta;                         /* 00-71: Generic meta-data page header. */
+
+    u_int32_t   max_bucket;                     /* 72-75: ID of Maximum bucket in use */
+    u_int32_t   high_mask;                      /* 76-79: Modulo mask into table */
+    u_int32_t   low_mask;                       /* 80-83: Modulo mask into table lower half */
+    u_int32_t   ffactor;                        /* 84-87: Fill factor */
+    u_int32_t   nelem;                          /* 88-91: Number of keys in hash table */
+    u_int32_t   h_charkey;                      /* 92-95: Value of hash(CHARKEY) */
+
+    u_int32_t   spares[NCACHED];                /* 96-223: Spare pages for overflow */
+    u_int32_t   unused[59];                     /* 224-459: Unused space */
+    u_int32_t   crypto_magic;                   /* 460-463: Crypto magic number */
+    u_int32_t   trash[3];                       /* 464-475: Trash space - Do not use */
+    u_int8_t    iv[DB_IV_BYTES];                /* 476-495: Crypto IV */
+    u_int8_t    chksum[DB_MAC_KEY];             /* 496-511: Page chksum */
+
+    /*
+     * Minimum page size is 512.
+     */
+} HMETA33, HMETA;
+
+/************************************************************************
+ HEAP METADATA PAGE LAYOUT
+*************************************************************************/
+/*
+ * Heap Meta data page structure
+ *
+ */
+typedef struct _heapmeta {
+    DBMETA      dbmeta;                         /* 00-71: Generic meta-data header. */
+
+    db_pgno_t   curregion;                      /* 72-75: Current region pgno. */
+    u_int32_t   nregions;                       /* 76-79: Number of regions. */
+    u_int32_t   gbytes;                         /* 80-83: GBytes for fixed size heap. */
+    u_int32_t   bytes;                          /* 84-87: Bytes for fixed size heap. */
+    u_int32_t   region_size;                    /* 88-91: Max region size. */
+    u_int32_t   unused2[92];                    /* 92-459: Unused space.*/
+    u_int32_t   crypto_magic;                   /* 460-463: Crypto magic number */
+    u_int32_t   trash[3];                       /* 464-475: Trash space - Do not use */
+    u_int8_t    iv[DB_IV_BYTES];                /* 476-495: Crypto IV */
+    u_int8_t    chksum[DB_MAC_KEY];             /* 496-511: Page chksum */
+
+    /*
+     * Minimum page size is 512.
+     */
+} HEAPMETA;
+
+/************************************************************************
+ QUEUE METADATA PAGE LAYOUT
+ ************************************************************************/
+/*
+ * QAM Meta data page structure
+ *
+ */
+typedef struct _qmeta33 {
+    DBMETA      dbmeta;                         /* 00-71: Generic meta-data header. */
+
+    u_int32_t   first_recno;                    /* 72-75: First not deleted record. */
+    u_int32_t   cur_recno;                      /* 76-79: Next recno to be allocated. */
+    u_int32_t   re_len;                         /* 80-83: Fixed-length record length. */
+    u_int32_t   re_pad;                         /* 84-87: Fixed-length record pad. */
+    u_int32_t   rec_page;                       /* 88-91: Records Per Page. */
+    u_int32_t   page_ext;                       /* 92-95: Pages per extent */
+
+    u_int32_t   unused[91];                     /* 96-459: Unused space */
+    u_int32_t   crypto_magic;                   /* 460-463: Crypto magic number */
+    u_int32_t   trash[3];                       /* 464-475: Trash space - Do not use */
+    u_int8_t    iv[DB_IV_BYTES];                /* 476-495: Crypto IV */
+    u_int8_t    chksum[DB_MAC_KEY];             /* 496-511: Page chksum */
+
+    /*
+     * Minimum page size is 512.
+     */
+} QMETA33, QMETA;
+
+/*
+ * DBMETASIZE is a constant used by __db_file_setup and DB->verify
+ * as a buffer which is guaranteed to be larger than any possible
+ * metadata page size and smaller than any disk sector.
+ */
+#define DBMETASIZE              512
+
+/************************************************************************
+ BTREE/HASH MAIN PAGE LAYOUT
+ ************************************************************************/
+/*
+ *  +-----------------------------------+
+ *  |    lsn    |   pgno    | prev pgno |
+ *  +-----------------------------------+
+ *  | next pgno |  entries  | hf offset |
+ *  +-----------------------------------+
+ *  |   level   |   type    |   chksum  |
+ *  +-----------------------------------+
+ *  |    iv     |   index   | free -->  |
+ *  +-----------+-----------------------+
+ *  |         F R E E A R E A           |
+ *  +-----------------------------------+
+ *  |              <-- free |   item    |
+ *  +-----------------------------------+
+ *  |   item    |   item    |   item    |
+ *  +-----------------------------------+
+ *
+ * sizeof(PAGE) == 26 bytes + possibly 20 bytes of checksum and possibly
+ * 16 bytes of IV (+ 2 bytes for alignment), and the following indices
+ * are guaranteed to be two-byte aligned.  If we aren't doing crypto or
+ * checksumming the bytes are reclaimed for data storage.
+ *
+ * For hash and btree leaf pages, index items are paired, e.g., inp[0] is the
+ * key for inp[1]'s data.  All other types of pages only contain single items.
+ */
+typedef struct __pg_chksum {
+    u_int8_t    unused[2];                      /* 26-27: For alignment */
+    u_int8_t    chksum[4];                      /* 28-31: Checksum */
+} PG_CHKSUM;
+
+typedef struct __pg_crypto {
+    u_int8_t    unused[2];                      /* 26-27: For alignment */
+    u_int8_t    chksum[DB_MAC_KEY];             /* 28-47: Checksum */
+    u_int8_t    iv[DB_IV_BYTES];                /* 48-63: IV */
+    /* !!!
+     * Must be 16-byte aligned for crypto
+     */
+} PG_CRYPTO;
+
+#define LEAFLEVEL   1
+#define MAXBTREELEVEL 255
+
+typedef struct _db_page {
+    DB_LSN      lsn;                            /* 00-07: Log sequence number. */
+    db_pgno_t   pgno;                           /* 08-11: Current page number. */
+    db_pgno_t   prev_pgno;                      /* 12-15: Previous page number. */
+    db_pgno_t   next_pgno;                      /* 16-19: Next page number. */
+    db_indx_t   entries;                        /* 20-21: Number of items on the page. */
+    db_indx_t   hf_offset;                      /* 22-23: High free byte page offset. */
+
+    /*
+     * The btree levels are numbered from the leaf to the root, starting
+     * with 1, so the leaf is level 1, its parent is level 2, and so on.
+     * We maintain this level on all btree pages, but the only place that
+     * we actually need it is on the root page.  It would not be difficult
+     * to hide the byte on the root page once it becomes an internal page,
+     * so we could get this byte back if we needed it for something else.
+     */
+    u_int8_t  level;                            /*    24: Btree tree level. */
+    u_int8_t  type;                             /*    25: Page type. */
+} PAGE;
+
+/*
+ * With many compilers sizeof(PAGE) == 28, while SIZEOF_PAGE == 26.
+ * We add in other things directly after the page header and need
+ * the SIZEOF_PAGE.  When giving the sizeof(), many compilers will
+ * pad it out to the next 4-byte boundary.
+ */
+#define SIZEOF_PAGE             26
+
+/************************************************************************
+ HEAP PAGE LAYOUT
+ ************************************************************************/
+#define HEAPPG_NORMAL           26
+#define HEAPPG_CHKSUM           48
+#define HEAPPG_SEC              64
+
+/*
+ *  +0-----------2------------4-----------6-----------7+
+ *  |                        lsn                       |
+ *  +-------------------------+------------------------+
+ *  |           pgno          |         unused0        |
+ *  +-------------+-----------+-----------+------------+
+ *  |  high_indx  | free_indx |  entries  |  hf offset |
+ *  +-------+-----+-----------+-----------+------------+
+ *  |unused2|type |  unused3  |      ...chksum...      |
+ *  +-------+-----+-----------+------------------------+
+ *  |  ...iv...   |   offset table / free space map    |
+ *  +-------------+------------------------------------+
+ *  |free->            F R E E A R E A                 |
+ *  +--------------------------------------------------+
+ *  |                <-- free |          item          |
+ *  +-------------------------+------------------------+
+ *  |           item          |          item          |
+ *  +-------------------------+------------------------+
+ *
+ * The page layout of both heap internal and data pages.  If not using
+ * crypto, iv will be overwritten with data.  If not using checksumming,
+ * unused3 and chksum will also be overwritten with data and data will start at
+ * 26.  Note that this layout lets us re-use a lot of the PAGE element macros
+ * defined above.
+ */
+typedef struct _heappg {
+    DB_LSN      lsn;                            /* 00-07: Log sequence number. */
+    db_pgno_t   pgno;                           /* 08-11: Current page number. */
+    u_int32_t   high_pgno;                      /* 12-15: Highest page in region. */
+    u_int16_t   high_indx;                      /* 16-17: Highest index in the offset table. */
+    db_indx_t   free_indx;                      /* 18-19: First available index. */
+    db_indx_t   entries;                        /* 20-21: Number of items on the page. */
+    db_indx_t   hf_offset;                      /* 22-23: High free byte page offset. */
+    u_int8_t    unused2[1];                     /*    24: Unused. */
+    u_int8_t    type;                           /*    25: Page type. */
+    u_int8_t    unused3[2];                     /* 26-27: Never used, just checksum alignment. */
+    u_int8_t    chksum[DB_MAC_KEY];             /* 28-47: Checksum */
+    u_int8_t    iv[DB_IV_BYTES];                /* 48-63: IV */
+} HEAPPG;
+
+/* Define first possible data page for heap, 0 is metapage, 1 is region page */
+#define FIRST_HEAP_RPAGE        1
+#define FIRST_HEAP_DPAGE        2
+
+#define HEAP_RECSPLIT           0x01            /* Heap data record is split */
+#define HEAP_RECFIRST           0x02            /* First piece of a split record */
+#define HEAP_RECLAST            0x04            /* Last piece of a split record */
+
+typedef struct __heaphdr {
+    u_int8_t    flags;                          /* 00: Flags describing record. */
+    u_int8_t    unused;                         /* 01: Padding. */
+    u_int16_t   size;                           /* 02-03: The size of the stored data piece. */
+} HEAPHDR;
+
+typedef struct __heaphdrsplt {
+    HEAPHDR     std_hdr;                        /* 00-03: The standard data header */
+    u_int32_t   tsize;                          /* 04-07: Total record size, 1st piece only */
+    db_pgno_t   nextpg;                         /* 08-11: RID.pgno of the next record piece */
+    db_indx_t   nextindx;                       /* 12-13: RID.indx of the next record piece */
+    u_int16_t   unused;                         /* 14-15: Padding. */
+} HEAPSPLITHDR;
+
+/************************************************************************
+ QUEUE MAIN PAGE LAYOUT
+ ************************************************************************/
+/*
+ * Sizes of page below.  Used to reclaim space if not doing
+ * crypto or checksumming.  If you change the QPAGE below you
+ * MUST adjust this too.
+ */
+#define QPAGE_NORMAL            28
+#define QPAGE_CHKSUM            48
+#define QPAGE_SEC               64
+
+typedef struct _qpage {
+    DB_LSN      lsn;                            /* 00-07: Log sequence number. */
+    db_pgno_t   pgno;                           /* 08-11: Current page number. */
+    u_int32_t   unused0[3];                     /* 12-23: Unused. */
+    u_int8_t    unused1[1];                     /*    24: Unused. */
+    u_int8_t    type;                           /*    25: Page type. */
+    u_int8_t    unused2[2];                     /* 26-27: Unused. */
+    u_int8_t    chksum[DB_MAC_KEY];             /* 28-47: Checksum */
+    u_int8_t    iv[DB_IV_BYTES];                /* 48-63: IV */
+} QPAGE;
+
+/************************************************************************
+ OVERFLOW PAGE LAYOUT
+ ************************************************************************/
+
+/*
+ * Overflow items are referenced by HOFFPAGE and BOVERFLOW structures, which
+ * store a page number (the first page of the overflow item) and a length
+ * (the total length of the overflow item).  The overflow item consists of
+ * some number of overflow pages, linked by the next_pgno field of the page.
+ * A next_pgno field of PGNO_INVALID flags the end of the overflow item.
+ *
+ * Overflow page overloads:
+ * The amount of overflow data stored on each page is stored in the
+ * hf_offset field.
+ *
+ * The implementation reference counts overflow items as it's possible
+ * for them to be promoted onto btree internal pages.  The reference
+ * count is stored in the entries field.
+ */
+
+/************************************************************************
+ HASH PAGE LAYOUT
+ ************************************************************************/
+
+/* Each index references a group of bytes on the page. */
+#define H_KEYDATA               1               /* Key/data item. */
+#define H_DUPLICATE             2               /* Duplicate key/data item. */
+#define H_OFFPAGE               3               /* Overflow key/data item. */
+#define H_OFFDUP                4               /* Overflow page of duplicates. */
+
+/*
+ * The first and second types are H_KEYDATA and H_DUPLICATE, represented
+ * by the HKEYDATA structure:
+ *
+ *  +-----------------------------------+
+ *  |    type   | key/data ...          |
+ *  +-----------------------------------+
+ *
+ * For duplicates, the data field encodes duplicate elements in the data
+ * field:
+ *
+ *  +---------------------------------------------------------------+
+ *  |    type   | len1 | element1 | len1 | len2 | element2 | len2   |
+ *  +---------------------------------------------------------------+
+ *
+ * Thus, by keeping track of the offset in the element, we can do both
+ * backward and forward traversal.
+ */
+typedef struct _hkeydata {
+    u_int8_t    type;                           /*    00: Page type. */
+    u_int8_t    data[0];                        /* Variable length key/data item. */
+} HKEYDATA;
+
+/*
+ * The third type is the H_OFFPAGE, represented by the HOFFPAGE structure:
+ */
+typedef struct _hoffpage {
+    u_int8_t    type;                           /*    00: Page type and delete flag. */
+    u_int8_t    unused[3];                      /* 01-03: Padding, unused. */
+    db_pgno_t   pgno;                           /* 04-07: Offpage page number. */
+    u_int32_t   tlen;                           /* 08-11: Total length of item. */
+} HOFFPAGE;
+
+/*
+ * The fourth type is H_OFFDUP represented by the HOFFDUP structure:
+ */
+typedef struct _hoffdup {
+    u_int8_t    type;                           /*    00: Page type and delete flag. */
+    u_int8_t    unused[3];                      /* 01-03: Padding, unused. */
+    db_pgno_t   pgno;                           /* 04-07: Offpage page number. */
+} HOFFDUP;
+
+/************************************************************************
+ BTREE PAGE LAYOUT
+ ************************************************************************/
+
+/* Each index references a group of bytes on the page. */
+#define B_KEYDATA               1               /* Key/data item. */
+#define B_DUPLICATE             2               /* Duplicate key/data item. */
+#define B_OVERFLOW              3               /* Overflow key/data item. */
+
+/*
+ * We have to store a deleted entry flag in the page.   The reason is complex,
+ * but the simple version is that we can't delete on-page items referenced by
+ * a cursor -- the return order of subsequent insertions might be wrong.  The
+ * delete flag is an overload of the top bit of the type byte.
+ */
+#define	B_DELETE	            (0x80)
+
+/*
+ * The first type is B_KEYDATA, represented by the BKEYDATA structure:
+ */
+typedef struct _bkeydata {
+    db_indx_t   len;                            /* 00-01: Key/data item length. */
+    u_int8_t    type;                           /*    02: Page type AND DELETE FLAG. */
+    u_int8_t    data[0];                        /* Variable length key/data item. */
+} BKEYDATA;
+
+/*
+ * The second and third types are B_DUPLICATE and B_OVERFLOW, represented
+ * by the BOVERFLOW structure.
+ */
+typedef struct _boverflow {
+    db_indx_t   unused1;                        /* 00-01: Padding, unused. */
+    u_int8_t    type;                           /*    02: Page type AND DELETE FLAG. */
+    u_int8_t    unused2;                        /*    03: Padding, unused. */
+    db_pgno_t   pgno;                           /* 04-07: Next page number. */
+    u_int32_t   tlen;                           /* 08-11: Total length of item. */
+} BOVERFLOW;
+
+/*
+ * Btree leaf and hash page layouts group indices in sets of two, one for the
+ * key and one for the data.  Everything else does it in sets of one to save
+ * space.  Use the following macros so that it's real obvious what's going on.
+ */
+#define O_INDX                  1
+#define P_INDX                  2
+
+/************************************************************************
+ BTREE INTERNAL PAGE LAYOUT
+ ************************************************************************/
+
+/*
+ * Btree internal entry.
+ */
+typedef struct _binternal {
+    db_indx_t   len;                            /* 00-01: Key/data item length. */
+    u_int8_t    type;                           /*    02: Page type AND DELETE FLAG. */
+    u_int8_t    unused;                         /*    03: Padding, unused. */
+    db_pgno_t   pgno;                           /* 04-07: Page number of referenced page. */
+    db_recno_t  nrecs;                          /* 08-11: Subtree record count. */
+    u_int8_t    data[0];                        /* Variable length key item. */
+} BINTERNAL;
+
+/************************************************************************
+ RECNO INTERNAL PAGE LAYOUT
+ ************************************************************************/
+
+/*
+ * The recno internal entry.
+ */
+typedef struct _rinternal {
+    db_pgno_t   pgno;                           /* 00-03: Page number of referenced page. */
+    db_recno_t  nrecs;                          /* 04-07: Subtree record count. */
+} RINTERNAL;
+"""
+c_db = cstruct().load(db_def)