sproutboat 0.28.0 → 0.29.0

package/vendor/ltx-restore/ltx/crc64.ts.txt

@@ -0,0 +1,73 @@
+/** CRC-64 ISO, compatible with Go hash/crc64. BigInt is used only at the API boundary. */
+import { updateNativeCrc64 } from '../crc64-native.ts';
+const HI = new Uint32Array(256);
+const LO = new Uint32Array(256);
+// Prototype controls: profiling flag, elapsed milliseconds, byte count, native backend.
+export const CRC_PROFILE = new Float64Array(4);
+for (let i = 0; i < 256; i++) {
+  let hi = 0, lo = i;
+  for (let j = 0; j < 8; j++) {
+    const carry = lo & 1;
+    lo = ((lo >>> 1) | ((hi & 1) << 31)) >>> 0;
+    hi >>>= 1;
+    if (carry) hi = (hi ^ 0xd8000000) >>> 0;
+  }
+  HI[i] = hi;
+  LO[i] = lo;
+}
+
+export class Crc64 {
+  private hi = 0xffffffff;
+  private lo = 0xffffffff;
+  private native = false;
+  private nativeState = new Uint8Array(8);
+
+  constructor() {
+    this.native = CRC_PROFILE[3] === 1;
+    this.nativeState.fill(255);
+  }
+
+  update(buf: Uint8Array): void {
+    const started = CRC_PROFILE[0] ? performance.now() : 0;
+    if (this.native) {
+      updateNativeCrc64(this.nativeState, buf);
+    } else {
+      let hi = this.hi, lo = this.lo;
+      for (let i = 0; i < buf.length; i++) {
+        const t = (lo ^ buf[i]) & 0xff;
+        const nextLo = ((lo >>> 8) | (hi << 24)) >>> 0;
+        hi = (HI[t] ^ (hi >>> 8)) >>> 0;
+        lo = (LO[t] ^ nextLo) >>> 0;
+      }
+      this.hi = hi;
+      this.lo = lo;
+    }
+    if (CRC_PROFILE[0]) {
+      CRC_PROFILE[1] += performance.now() - started;
+      CRC_PROFILE[2] += buf.length;
+    }
+  }
+
+  reset(): void {
+    this.hi = 0xffffffff;
+    this.lo = 0xffffffff;
+    this.nativeState.fill(255);
+  }
+
+  sum64(): bigint {
+    if (this.native) {
+      const s = this.nativeState;
+      const hi = (s[0] << 24 | s[1] << 16 | s[2] << 8 | s[3]) >>> 0;
+      const lo = (s[4] << 24 | s[5] << 16 | s[6] << 8 | s[7]) >>> 0;
+      return (BigInt((hi ^ 0xffffffff) >>> 0) << 32n) | BigInt((lo ^ 0xffffffff) >>> 0);
+    }
+    return (BigInt((this.hi ^ 0xffffffff) >>> 0) << 32n) |
+      BigInt((this.lo ^ 0xffffffff) >>> 0);
+  }
+}
+
+export function crc64ISO(buf: Uint8Array): bigint {
+  const h = new Crc64();
+  h.update(buf);
+  return h.sum64();
+}

package/vendor/ltx-restore/ltx/decoder.ts.txt

@@ -0,0 +1,285 @@
+/**
+ * LTX decoder.
+ *
+ * Mirrors Go's `ltx.Decoder`. Reads header, page frames, page index, and
+ * trailer from an in-memory buffer, verifying the file checksum and (for
+ * checksum-tracking snapshots) the post-apply rolling checksum.
+ */
+
+import {
+  ChecksumFlag,
+  HeaderSize,
+  PageHeaderFlagSize,
+  PageHeaderSize,
+  TrailerChecksumOffset,
+  TrailerSize,
+  lockPgno,
+} from './constants.ts';
+import { Crc64 } from './crc64.ts';
+import { u32BE } from './bytes.ts';
+import { ByteReader } from './byte-reader.ts';
+import type { ByteSink } from './sink.ts';
+import type { Header, PageHeader, PageIndex, Trailer } from './types.ts';
+import { isSnapshot } from './types.ts';
+import { unmarshalHeader, validateHeader } from './header.ts';
+import { unmarshalPageHeader } from './page.ts';
+import { unmarshalTrailer } from './trailer.ts';
+import { parsePageIndex } from './page-index.ts';
+import { compressBlockBound, decompressBlock } from './lz4.ts';
+import { checksumPage } from './checksum.ts';
+import { LTXFormatError } from '../errors.ts';
+
+const stateHeader = 'header';
+const statePage = 'page';
+const stateClose = 'close';
+const stateClosed = 'closed';
+
+function noChecksum(h: Header): boolean {
+  return (h.flags & 0x00000002) !== 0;
+}
+
+/** Decoder reads an LTX file from an in-memory buffer. */
+export class Decoder {
+  private r: ByteReader;
+  private header!: Header;
+  private trailer: Trailer = { postApplyChecksum: 0n, fileChecksum: 0n };
+  private pageIndex: PageIndex | null = null;
+  private state = stateHeader;
+  private hash = new Crc64();
+  private pageSeq = new Crc64();
+  private chksum = 0n;
+  private pageN = 0;
+  private retainPageIndex: boolean;
+
+  constructor(buf: Uint8Array) {
+    this.r = new ByteReader(buf);
+    this.retainPageIndex = true;
+  }
+
+  setReader(reader: ByteReader): void { this.r = reader; }
+
+  /** SetRetainPageIndex controls whether Close materializes the page-index map. */
+  setRetainPageIndex(retain: boolean): void {
+    this.retainPageIndex = retain;
+  }
+
+  /** N returns the number of source bytes read. */
+  N(): number {
+    return this.r.pos;
+  }
+
+  /** PageN returns the number of pages decoded. */
+  PageN(): number {
+    return this.pageN;
+  }
+
+  /** Header returns a copy of the header. */
+  Header(): Header {
+    return this.header;
+  }
+
+  /** Trailer returns the trailer. File checksum valid after Close(). */
+  Trailer(): Trailer {
+    return this.trailer;
+  }
+
+  /** PageIndex returns the page index (after Close), or null if not retained. */
+  PageIndex(): PageIndex | null {
+    return this.pageIndex;
+  }
+
+  /** PostApplyPos returns the replication position after applying this file. */
+  PostApplyPos() {
+    return {
+      txID: this.header.maxTXID,
+      postApplyChecksum: this.trailer.postApplyChecksum,
+    };
+  }
+
+  /** DecodeHeader reads and validates the header frame. */
+  decodeHeader(): void {
+    const b = this.r.readFull(HeaderSize);
+    this.header = unmarshalHeader(b);
+    this.hash.update(b);
+    this.state = statePage;
+    validateHeader(this.header);
+    if (!noChecksum(this.header)) {
+      this.chksum = ChecksumFlag;
+    }
+  }
+
+  /**
+   * DecodePage reads the next page frame into `data`. Returns the page header,
+   * or null when the end-of-pages marker is reached.
+   */
+  decodePage(data: Uint8Array): PageHeader | null {
+    if (this.state === stateClosed) throw new LTXFormatError('ltx decoder closed');
+    if (this.state === stateClose) return null;
+    if (this.state !== statePage) {
+      throw new LTXFormatError(`cannot read page header, expected ${this.state}`);
+    }
+    if (data.length !== this.header.pageSize) {
+      throw new LTXFormatError(`invalid page buffer size: ${data.length}, expecting ${this.header.pageSize}`);
+    }
+
+    const b = this.r.readFull(PageHeaderSize);
+    const hdr = unmarshalPageHeader(b);
+    this.hash.update(b);
+
+    // An empty page header marks the end of the page block.
+    if (hdr.pgno === 0 && hdr.flags === 0) {
+      this.state = stateClose;
+      return null;
+    }
+
+    if (hdr.pgno > this.header.commit) {
+      throw new LTXFormatError(`page number ${hdr.pgno} out-of-bounds for commit size ${this.header.commit}`);
+    }
+
+    if (hdr.flags & PageHeaderFlagSize) {
+      // New block format: read size prefix, then LZ4 block data.
+      const sizeBuf = this.r.readFull(4);
+      this.hash.update(sizeBuf);
+      const dataSize = (sizeBuf[0] << 24 | sizeBuf[1] << 16 | sizeBuf[2] << 8 | sizeBuf[3]) >>> 0;
+      if (dataSize < 1 || dataSize > compressBlockBound(this.header.pageSize)) throw new LTXFormatError('invalid compressed page size');
+      const compressed = this.r.readFull(dataSize);
+      const decompressed = decompressBlock(compressed, this.header.pageSize);
+      data.set(decompressed);
+    } else {
+      throw new LTXFormatError('legacy LZ4 frame format not supported');
+    }
+
+    this.hash.update(data);
+    this.hashPgo(this.pageSeq, hdr.pgno);
+    this.pageN++;
+
+    // Track the rolling DB checksum for checksum-tracking snapshots.
+    if (isSnapshot(this.header) && !noChecksum(this.header)) {
+      if (hdr.pgno !== lockPgno(this.header.pageSize)) {
+        this.chksum = ChecksumFlag | (this.chksum ^ checksumPage(hdr.pgno, data));
+      }
+    }
+
+    return hdr;
+  }
+
+  /** Verify reads the entire file, discarding page data (for validation). */
+  verify(): void {
+    this.decodeHeader();
+    const data = new Uint8Array(this.header.pageSize);
+    for (;;) {
+      if (this.decodePage(data) === null) break;
+    }
+    this.close();
+  }
+
+  /** DecodeDatabaseTo writes the snapshot as a SQLite database to `sink`. */
+  decodeDatabaseTo(sink: ByteSink): void {
+    this.decodeHeader();
+    const hdr = this.header;
+    if (!isSnapshot(hdr)) {
+      throw new LTXFormatError('cannot decode non-snapshot LTX file to SQLite database');
+    }
+    const lockPg = lockPgno(hdr.pageSize);
+    const data = new Uint8Array(hdr.pageSize);
+    for (let pgno = 1; pgno <= hdr.commit; pgno++) {
+      if (pgno === lockPg) {
+        sink.write(new Uint8Array(hdr.pageSize));
+        continue;
+      }
+      const ph = this.decodePage(data);
+      if (ph === null) {
+        throw new LTXFormatError(`unexpected end of pages at pgno ${pgno}`);
+      }
+      if (ph.pgno !== pgno) {
+        throw new LTXFormatError(`unexpected pgno while decoding page: read ${ph.pgno}, expected ${pgno}`);
+      }
+      sink.write(data);
+    }
+
+    // Expect one final read to be the end-of-pages marker.
+    if (this.decodePage(data) !== null) {
+      throw new LTXFormatError('unexpected page after end of database');
+    }
+    this.close();
+  }
+
+  /** Close verifies the page index and trailer, then checks the file checksum. */
+  close(): void {
+    if (this.state === stateClosed) return;
+    if (this.state !== stateClose) {
+      throw new LTXFormatError(`cannot close, expected ${this.state}`);
+    }
+
+    const index: PageIndex = new Map();
+    let n = 0;
+    let nextOffset = HeaderSize;
+    const seq = new Crc64();
+    const self = this;
+
+    parsePageIndex(
+      this.r,
+      (buf) => self.hash.update(buf),
+      (pgno, offset, size) => {
+        if (pgno > self.header.commit) {
+          throw new LTXFormatError(`page index pgno ${pgno} exceeds commit ${self.header.commit}`);
+        }
+        if (offset < nextOffset) {
+          throw new LTXFormatError(
+            `page index pgno ${pgno} offset ${offset} overlaps previous frame ending at ${nextOffset}`,
+          );
+        }
+        if (size <= PageHeaderSize) {
+          throw new LTXFormatError(`page index pgno ${pgno} has invalid frame size ${size}`);
+        }
+        nextOffset = offset + size;
+        n++;
+        self.hashPgo(seq, pgno);
+        if (self.retainPageIndex) {
+          index.set(pgno, {
+            level: 0,
+            minTXID: self.header.minTXID,
+            maxTXID: self.header.maxTXID,
+            offset,
+            size,
+          });
+        }
+      },
+    );
+
+    if (n !== this.pageN) {
+      throw new LTXFormatError(`page index has ${n} entries but ${this.pageN} pages were decoded`);
+    }
+    if (seq.sum64() !== this.pageSeq.sum64()) {
+      throw new LTXFormatError('page index does not match the decoded page numbers');
+    }
+    this.pageIndex = this.retainPageIndex ? index : null;
+
+    // Read the trailer; everything except the file checksum field is hashed.
+    const t = this.r.readFull(TrailerSize);
+    this.hash.update(t.subarray(0, TrailerChecksumOffset));
+    this.trailer = unmarshalTrailer(t);
+    if (this.r.remaining > 0) {
+      throw new LTXFormatError('unexpected data after trailer');
+    }
+
+    const computed = ChecksumFlag | this.hash.sum64();
+    if (computed !== this.trailer.fileChecksum) {
+      throw new LTXFormatError('file checksum mismatch');
+    }
+
+    if (isSnapshot(this.header) && !noChecksum(this.header)) {
+      if (this.trailer.postApplyChecksum !== this.chksum) {
+        throw new LTXFormatError(
+          `post-apply checksum in trailer (${this.trailer.postApplyChecksum.toString(16)}) does not match calculated checksum (${this.chksum.toString(16)})`,
+        );
+      }
+    }
+
+    this.state = stateClosed;
+  }
+
+  private hashPgo(h: Crc64, pgno: number): void {
+    h.update(u32BE(pgno));
+  }
+}

package/vendor/ltx-restore/ltx/header.ts.txt

@@ -0,0 +1,126 @@
+/**
+ * LTX header frame marshalling.
+ */
+
+import { HeaderSize, Magic, Version } from './constants.ts';
+import {
+  ChecksumFlag,
+  isValidHeaderFlags,
+  isValidPageSize,
+} from './constants.ts';
+import { assertFull, readU32BE, readU64BE, writeU32BE, writeU64BE } from './bytes.ts';
+import type { Header } from './types.ts';
+import { isSnapshot } from './types.ts';
+import { lockPgno } from './constants.ts';
+import { LTXFormatError } from '../errors.ts';
+
+/**
+ * MarshalBinary encodes a header to its 100-byte representation.
+ */
+export function marshalHeader(h: Header): Uint8Array {
+  const b = new Uint8Array(HeaderSize);
+  for (let i = 0; i < 4; i++) b[i] = Magic.charCodeAt(i);
+  writeU32BE(b, 4, h.flags);
+  writeU32BE(b, 8, h.pageSize);
+  writeU32BE(b, 12, h.commit);
+  writeU64BE(b, 16, h.minTXID);
+  writeU64BE(b, 24, h.maxTXID);
+  writeU64BE(b, 32, h.timestamp);
+  writeU64BE(b, 40, h.preApplyChecksum);
+  writeU64BE(b, 48, h.walOffset);
+  writeU64BE(b, 56, h.walSize);
+  writeU32BE(b, 64, h.walSalt1);
+  writeU32BE(b, 68, h.walSalt2);
+  writeU64BE(b, 72, h.nodeID);
+  return b;
+}
+
+/**
+ * UnmarshalBinary decodes a header from a byte slice.
+ * Sets `version` from the magic and returns the header.
+ */
+export function unmarshalHeader(b: Uint8Array): Header {
+  assertFull(b, 0, HeaderSize);
+  const magic = String.fromCharCode(b[0], b[1], b[2], b[3]);
+  if (magic !== Magic) {
+    throw new LTXFormatError('invalid LTX file');
+  }
+  return {
+    version: Version,
+    flags: readU32BE(b, 4),
+    pageSize: readU32BE(b, 8),
+    commit: readU32BE(b, 12),
+    minTXID: readU64BE(b, 16),
+    maxTXID: readU64BE(b, 24),
+    timestamp: readU64BE(b, 32),
+    preApplyChecksum: readU64BE(b, 40),
+    walOffset: readU64BE(b, 48),
+    walSize: readU64BE(b, 56),
+    walSalt1: readU32BE(b, 64),
+    walSalt2: readU32BE(b, 68),
+    nodeID: readU64BE(b, 72),
+  };
+}
+
+function noChecksum(flags: number): boolean {
+  return (flags & 0x00000002) !== 0;
+}
+
+/** Validate returns an error (thrown) if h is invalid. */
+export function validateHeader(h: Header): void {
+  if (h.version !== Version) {
+    throw new LTXFormatError('invalid version');
+  }
+  if (!isValidHeaderFlags(h.flags)) {
+    throw new LTXFormatError(`invalid flags: 0x${h.flags.toString(16)}`);
+  }
+  if (!isValidPageSize(h.pageSize)) {
+    throw new LTXFormatError(`invalid page size: ${h.pageSize}`);
+  }
+  if (h.minTXID === 0n) {
+    throw new LTXFormatError('minimum transaction id required');
+  }
+  if (h.maxTXID === 0n) {
+    throw new LTXFormatError('maximum transaction id required');
+  }
+  if (h.minTXID > h.maxTXID) {
+    throw new LTXFormatError(`transaction ids out of order: (${h.minTXID},${h.maxTXID})`);
+  }
+
+  if (h.walOffset < 0n) {
+    throw new LTXFormatError(`wal offset cannot be negative: ${h.walOffset}`);
+  }
+  if (h.walSize < 0n) {
+    throw new LTXFormatError(`wal size cannot be negative: ${h.walSize}`);
+  }
+
+  if ((h.walSalt1 !== 0 || h.walSalt2 !== 0) && h.walOffset === 0n) {
+    throw new LTXFormatError('wal offset required if salt exists');
+  }
+  if (h.walOffset === 0n && h.walSize !== 0n) {
+    throw new LTXFormatError('wal offset required if wal size exists');
+  }
+
+  const snap = isSnapshot(h);
+  const nc = noChecksum(h.flags);
+  if (snap) {
+    if (h.preApplyChecksum !== 0n) {
+      throw new LTXFormatError('pre-apply checksum must be zero on snapshots');
+    }
+  } else if (nc) {
+    if (h.preApplyChecksum !== 0n) {
+      throw new LTXFormatError('pre-apply checksum not allowed');
+    }
+  } else {
+    if (h.preApplyChecksum === 0n) {
+      throw new LTXFormatError('pre-apply checksum required on non-snapshot files');
+    }
+    if ((h.preApplyChecksum & ChecksumFlag) === 0n) {
+      throw new LTXFormatError('invalid pre-apply checksum format');
+    }
+  }
+}
+
+export function headerLockPgno(h: Header): number {
+  return lockPgno(h.pageSize);
+}