socketduct 0.1.18 → 0.1.20

package/src/logging/compressed-log-worker.js

@@ -0,0 +1,430 @@
+/* eslint-disable jsdoc/escape-inline-tags, jsdoc/require-param-type, jsdoc/require-returns */
+import {createHash, randomUUID} from "node:crypto"
+import {constants} from "node:fs"
+import {lstat, mkdir, open, readdir, stat, unlink} from "node:fs/promises"
+import {join} from "node:path"
+import {isMainThread, parentPort, workerData} from "node:worker_threads"
+
+import {inspectLogBatch} from "./log-batch.js"
+import {readCompressedLogSegment} from "./compressed-log-reader.js"
+import {enforceLogRetention} from "./log-retention.js"
+import {createIndexEntry, createIndexHeader, INDEX_ENTRY_BYTES, INDEX_HEADER_BYTES, parseIndexEntry, parseIndexHeader, uuidBytes} from "./log-segment-index.js"
+import {compressLogFrame, decompressLogFrame} from "./zstd-frame-codec.js"
+
+const ROLE = /^[a-z][a-z0-9-]{0,63}$/
+
+/** Dedicated storage worker. Compression and sync never execute on a protocol event loop. */
+export class CompressedLogWorkerSink {
+  /** @type {string} */ #directory
+  /** @type {string} */ #role
+  /** @type {Buffer} */ #writerEpoch
+  /** @type {string} */ #writerEpochText
+  /** @type {number} */ #maxSegmentBytes
+  /** @type {number} */ #compressionLevel
+  /** @type {number} */ #retentionAgeMs
+  /** @type {number} */ #retentionBytes
+  /** @type {import('node:fs/promises').FileHandle | undefined} */ #dataHandle
+  /** @type {import('node:fs/promises').FileHandle | undefined} */ #indexHandle
+  /** @type {string | undefined} */ #dataPath
+  /** @type {string | undefined} */ #indexPath
+  /** @type {Buffer | undefined} */ #segmentUuid
+  /** @type {bigint} */ #dataBytes = 0n
+  /** @type {bigint} */ #durableHighWater = 0n
+  /** @type {{batchId: string, firstSequence: bigint, lastSequence: bigint} | undefined} */ #lastDurableBatch
+
+  /** @param {{directory: string, role: string, writerEpoch: string, maxSegmentBytes: number, compressionLevel: number, retentionAgeMs?: number, retentionBytes?: number}} options - Validated producer settings. */
+  constructor(options) {
+    if (typeof options.directory !== "string" || options.directory.length === 0) throw new TypeError("worker log directory is required")
+    if (typeof options.role !== "string" || !ROLE.test(options.role)) throw new TypeError("worker log role is invalid")
+    const writerEpoch = Buffer.from(options.writerEpoch, "base64url")
+    if (writerEpoch.length !== 16 || writerEpoch.toString("base64url") !== options.writerEpoch) throw new TypeError("worker epoch is invalid")
+    if (!Number.isSafeInteger(options.maxSegmentBytes) || options.maxSegmentBytes < 1024) throw new RangeError("maxSegmentBytes must be at least 1024")
+    this.#directory = options.directory
+    this.#role = options.role
+    this.#writerEpoch = writerEpoch
+    this.#writerEpochText = options.writerEpoch
+    this.#maxSegmentBytes = options.maxSegmentBytes
+    this.#compressionLevel = options.compressionLevel
+    this.#retentionAgeMs = options.retentionAgeMs ?? 30 * 24 * 60 * 60 * 1000
+    this.#retentionBytes = options.retentionBytes ?? 4 * 1024 * 1024 * 1024
+  }
+
+  /** Recovers the producer epoch and opens exactly one active segment. */
+  async start() {
+    await mkdir(this.#directory, {recursive: true, mode: 0o700})
+    const directoryStats = await lstat(this.#directory)
+    if (!directoryStats.isDirectory() || (directoryStats.mode & 0o777) !== 0o700) throw new Error("Unsafe compressed log directory")
+    const prefix = `${this.#role}-${this.#writerEpochText}-`
+    const candidates = (await readdir(this.#directory)).filter((name) => name.startsWith(prefix) && name.endsWith(".jsonl.zst"))
+    /** @type {Array<{path: string, sealed: boolean, firstSequence: bigint | undefined, lastSequence: bigint, lastBatch: {batchId: string, firstSequence: bigint, lastSequence: bigint} | undefined, dataBytes: bigint, segmentUuid: Buffer, orphanRecovered: boolean}>} */
+    const recovered = []
+    for (const name of candidates) {
+      const path = join(this.#directory, name)
+      try {
+        recovered.push(await this.#recoverSegment(path))
+      } catch (error) {
+        await markQuarantined(path, this.#directory)
+        throw error
+      }
+    }
+    recovered.sort((left, right) => left.firstSequence === undefined ? 1 : right.firstSequence === undefined ? -1 : left.firstSequence < right.firstSequence ? -1 : left.firstSequence > right.firstSequence ? 1 : 0)
+    /** @type {bigint | undefined} */
+    let expected
+    for (const segment of recovered) {
+      if (segment.firstSequence !== undefined) {
+        expected ??= segment.firstSequence
+        if (segment.firstSequence !== expected) throw new Error("Compressed log writer epoch contains an interior sequence gap")
+        expected = segment.lastSequence + 1n
+        this.#lastDurableBatch = segment.lastBatch
+      }
+    }
+    this.#durableHighWater = expected === undefined ? 0n : expected - 1n
+    const active = recovered.filter((segment) => !segment.sealed)
+    if (active.length > 1) throw new Error("Compressed log writer epoch has multiple active segments")
+    await this.#enforceRetention()
+    if (active.length === 1) await this.#openRecoveredActive(/** @type {typeof recovered[number]} */ (active[0]))
+    else await this.#createSegment()
+    return {...this.status(), orphanRecoveredCount: recovered.filter((segment) => segment.orphanRecovered).length}
+  }
+
+  /**
+   * Writes and durably indexes one immutable producer-retained batch.
+   * @param message
+   */
+  async appendBatch(/** @type {{batchId: string, firstSequence: string, lastSequence: string, jsonl: Uint8Array}} */ message) {
+    const batchId = Buffer.from(message.batchId, "base64url")
+    const raw = Buffer.from(message.jsonl)
+    if (batchId.length !== 16 || batchId.toString("base64url") !== message.batchId) throw new Error("Invalid compressed log batch ID")
+    const inspected = inspectLogBatch(raw, this.#writerEpoch)
+    if (inspected.firstSequence.toString() !== message.firstSequence || inspected.lastSequence.toString() !== message.lastSequence || !inspected.batchId.equals(batchId)) throw new Error("Compressed log batch metadata mismatch")
+    const existing = this.#lastDurableBatch
+    if (inspected.lastSequence <= this.#durableHighWater) {
+      if (existing?.batchId !== message.batchId || existing.firstSequence !== inspected.firstSequence || existing.lastSequence !== inspected.lastSequence) throw new Error("Compressed log batch is stale or conflicts with durable high-water")
+      return {duplicate: true, ...this.status()}
+    }
+    if (existing?.batchId === message.batchId) throw new Error("Compressed log batch identity conflict")
+    if (inspected.firstSequence !== this.#durableHighWater + 1n) throw new Error("Compressed log batch sequence is not contiguous")
+    const frame = await compressLogFrame(raw, this.#compressionLevel)
+    if (this.#dataBytes > 0n && this.#dataBytes + BigInt(frame.length) > BigInt(this.#maxSegmentBytes)) {
+      await this.#sealActive()
+      await this.#enforceRetention()
+      await this.#createSegment()
+    }
+    if (frame.length > this.#maxSegmentBytes) throw new Error("Compressed log frame exceeds segment byte limit")
+    const dataHandle = required(this.#dataHandle, "data handle")
+    const indexHandle = required(this.#indexHandle, "index handle")
+    const segmentUuid = required(this.#segmentUuid, "segment UUID")
+    const dataOffset = this.#dataBytes
+    await writeAll(dataHandle, frame, Number(dataOffset))
+    await dataHandle.datasync()
+    const entry = createIndexEntry({
+      segmentUuid,
+      writerEpoch: this.#writerEpoch,
+      batchId,
+      firstSequence: inspected.firstSequence,
+      lastSequence: inspected.lastSequence,
+      dataOffset,
+      compressedLength: BigInt(frame.length),
+      uncompressedLength: BigInt(raw.length),
+      recordCount: inspected.recordCount,
+      minTimestamp: inspected.minTimestamp,
+      maxTimestamp: inspected.maxTimestamp,
+      frameSha256: createHash("sha256").update(frame).digest()
+    })
+    const indexStats = await indexHandle.stat()
+    await writeAll(indexHandle, entry, indexStats.size)
+    await indexHandle.datasync()
+    this.#dataBytes += BigInt(frame.length)
+    this.#durableHighWater = inspected.lastSequence
+    this.#lastDurableBatch = {batchId: message.batchId, firstSequence: inspected.firstSequence, lastSequence: inspected.lastSequence}
+    return {duplicate: false, ...this.status()}
+  }
+
+  status() {
+    return {
+      durableHighWater: this.#durableHighWater.toString(),
+      lastDurableBatch: this.#lastDurableBatch === undefined ? undefined : {
+        batchId: this.#lastDurableBatch.batchId,
+        firstSequence: this.#lastDurableBatch.firstSequence.toString(),
+        lastSequence: this.#lastDurableBatch.lastSequence.toString()
+      },
+      segmentBytes: Number(this.#dataBytes),
+      segmentPath: this.#dataPath
+    }
+  }
+
+  /** Syncs, marks the active segment closed, and releases descriptors. */
+  async close() {
+    if (this.#dataHandle === undefined) return this.status()
+    await this.#sealActive()
+    await this.#enforceRetention()
+    return this.status()
+  }
+
+  /** @param {string} path - Candidate data segment. */
+  async #recoverSegment(path) {
+    const name = path.slice(path.lastIndexOf("/") + 1)
+    const uuidText = name.slice(`${this.#role}-${this.#writerEpochText}-`.length, -".jsonl.zst".length)
+    const segmentUuid = uuidBytes(uuidText)
+    const indexPath = `${path}.idx`
+    await Promise.all([assertPrivateFile(path), assertPrivateFile(indexPath)])
+    const sealed = await exists(`${path}.sealed`)
+    const [dataHandle, indexHandle] = await Promise.all([
+      open(path, constants.O_RDWR | constants.O_NOFOLLOW),
+      open(indexPath, constants.O_RDWR | constants.O_NOFOLLOW)
+    ])
+    let orphanRecovered = false
+    try {
+      const indexStats = await indexHandle.stat()
+      if (indexStats.size < INDEX_HEADER_BYTES) throw new Error("Compressed log index header is torn")
+      const completeLength = INDEX_HEADER_BYTES + Math.floor((indexStats.size - INDEX_HEADER_BYTES) / INDEX_ENTRY_BYTES) * INDEX_ENTRY_BYTES
+      if (completeLength !== indexStats.size) {
+        if (sealed) throw new Error("Sealed compressed log index tail is torn")
+        await indexHandle.truncate(completeLength)
+        await indexHandle.datasync()
+      }
+      const headerBytes = Buffer.alloc(INDEX_HEADER_BYTES)
+      await readAll(indexHandle, headerBytes, 0)
+      const header = parseIndexHeader(headerBytes)
+      if (!header.segmentUuid.equals(segmentUuid) || !header.writerEpoch.equals(this.#writerEpoch)) throw new Error("Compressed log segment identity mismatch")
+      let firstSequence
+      let lastSequence = 0n
+      let dataEnd = 0n
+      const entryCount = (completeLength - INDEX_HEADER_BYTES) / INDEX_ENTRY_BYTES
+      for (let index = 0; index < entryCount; index += 1) {
+        const bytes = Buffer.alloc(INDEX_ENTRY_BYTES)
+        await readAll(indexHandle, bytes, INDEX_HEADER_BYTES + index * INDEX_ENTRY_BYTES)
+        const entry = parseIndexEntry(bytes, {segmentUuid, writerEpoch: this.#writerEpoch})
+        firstSequence ??= entry.firstSequence
+        lastSequence = entry.lastSequence
+        dataEnd = entry.dataOffset + entry.compressedLength
+      }
+      const dataStats = await dataHandle.stat()
+      if (BigInt(dataStats.size) < dataEnd) throw new Error("Compressed log data is shorter than its durable index")
+      if (BigInt(dataStats.size) > dataEnd) {
+        if (sealed) throw new Error("Sealed compressed log contains unindexed data")
+        const tailLength = dataStats.size - Number(dataEnd)
+        const tail = Buffer.alloc(tailLength)
+        await readAll(dataHandle, tail, Number(dataEnd))
+        try {
+          const raw = await decompressLogFrame(tail)
+          const inspected = inspectLogBatch(raw, this.#writerEpoch)
+          if (inspected.firstSequence !== lastSequence + 1n) throw new Error("orphan sequence gap")
+          const entry = createIndexEntry({
+            segmentUuid,
+            writerEpoch: this.#writerEpoch,
+            batchId: inspected.batchId,
+            firstSequence: inspected.firstSequence,
+            lastSequence: inspected.lastSequence,
+            dataOffset: dataEnd,
+            compressedLength: BigInt(tail.length),
+            uncompressedLength: BigInt(raw.length),
+            recordCount: inspected.recordCount,
+            minTimestamp: inspected.minTimestamp,
+            maxTimestamp: inspected.maxTimestamp,
+            frameSha256: createHash("sha256").update(tail).digest()
+          })
+          await dataHandle.datasync()
+          await writeAll(indexHandle, entry, completeLength)
+          await indexHandle.datasync()
+          firstSequence ??= inspected.firstSequence
+          lastSequence = inspected.lastSequence
+          dataEnd += BigInt(tail.length)
+          orphanRecovered = true
+        } catch {
+          await dataHandle.truncate(Number(dataEnd))
+          await dataHandle.datasync()
+        }
+      }
+    } finally {
+      await Promise.allSettled([dataHandle.close(), indexHandle.close()])
+    }
+    await readCompressedLogSegment(path)
+    const metadata = await readSegmentMetadata(indexPath, segmentUuid, this.#writerEpoch)
+    return {path, sealed, segmentUuid, orphanRecovered, ...metadata}
+  }
+
+  /** @param {{path: string, segmentUuid: Buffer, dataBytes: bigint}} segment - Recovered active segment. */
+  async #openRecoveredActive(segment) {
+    this.#dataPath = segment.path
+    this.#indexPath = `${segment.path}.idx`
+    this.#segmentUuid = segment.segmentUuid
+    this.#dataBytes = segment.dataBytes
+    this.#dataHandle = await open(segment.path, constants.O_RDWR | constants.O_NOFOLLOW)
+    this.#indexHandle = await open(this.#indexPath, constants.O_RDWR | constants.O_NOFOLLOW)
+  }
+
+  async #createSegment() {
+    const segmentUuidText = randomUUID()
+    const segmentUuid = uuidBytes(segmentUuidText)
+    const name = `${this.#role}-${this.#writerEpochText}-${segmentUuidText}.jsonl.zst`
+    const dataPath = join(this.#directory, name)
+    const indexPath = `${dataPath}.idx`
+    const dataHandle = await open(dataPath, constants.O_RDWR | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600)
+    let indexHandle
+    try {
+      indexHandle = await open(indexPath, constants.O_RDWR | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600)
+      await writeAll(indexHandle, createIndexHeader({segmentUuid, writerEpoch: this.#writerEpoch}), 0)
+      await indexHandle.datasync()
+      await syncDirectory(this.#directory)
+    } catch (error) {
+      await dataHandle.close().catch(() => {})
+      await indexHandle?.close().catch(() => {})
+      await Promise.allSettled([unlink(dataPath), unlink(indexPath)])
+      throw error
+    }
+    this.#dataPath = dataPath
+    this.#indexPath = indexPath
+    this.#segmentUuid = segmentUuid
+    this.#dataBytes = 0n
+    this.#dataHandle = dataHandle
+    this.#indexHandle = indexHandle
+  }
+
+  async #sealActive() {
+    const dataHandle = required(this.#dataHandle, "data handle")
+    const indexHandle = required(this.#indexHandle, "index handle")
+    const dataPath = required(this.#dataPath, "data path")
+    await dataHandle.datasync()
+    await indexHandle.datasync()
+    await Promise.all([dataHandle.close(), indexHandle.close()])
+    const marker = await open(`${dataPath}.sealed`, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600)
+    await marker.sync()
+    await marker.close()
+    await syncDirectory(this.#directory)
+    this.#dataHandle = undefined
+    this.#indexHandle = undefined
+    this.#dataPath = undefined
+    this.#indexPath = undefined
+    this.#segmentUuid = undefined
+    this.#dataBytes = 0n
+  }
+
+  async #enforceRetention() {
+    await enforceLogRetention({directory: this.#directory, maxAgeMs: this.#retentionAgeMs, maxBytes: this.#retentionBytes})
+  }
+}
+
+/**
+ * @param {string} indexPath - Index file. @param {Buffer} segmentUuid - Identity. @param {Buffer} writerEpoch - Identity.
+ * @param segmentUuid
+ * @param writerEpoch
+ */
+async function readSegmentMetadata(indexPath, segmentUuid, writerEpoch) {
+  const handle = await open(indexPath, constants.O_RDONLY | constants.O_NOFOLLOW)
+  try {
+    const stats = await handle.stat()
+    const count = (stats.size - INDEX_HEADER_BYTES) / INDEX_ENTRY_BYTES
+    let firstSequence
+    let lastSequence = 0n
+    /** @type {{batchId: string, firstSequence: bigint, lastSequence: bigint} | undefined} */
+    let lastBatch
+    let dataBytes = 0n
+    for (let index = 0; index < count; index += 1) {
+      const bytes = Buffer.alloc(INDEX_ENTRY_BYTES)
+      await readAll(handle, bytes, INDEX_HEADER_BYTES + index * INDEX_ENTRY_BYTES)
+      const entry = parseIndexEntry(bytes, {segmentUuid, writerEpoch})
+      firstSequence ??= entry.firstSequence
+      lastSequence = entry.lastSequence
+      dataBytes = entry.dataOffset + entry.compressedLength
+      lastBatch = {batchId: entry.batchId.toString("base64url"), firstSequence: entry.firstSequence, lastSequence: entry.lastSequence}
+    }
+    return {firstSequence, lastSequence, lastBatch, dataBytes}
+  } finally { await handle.close() }
+}
+
+/**
+ * Short-write-safe positional writer.
+ * @param handle
+ * @param bytes
+ * @param position
+ */
+export async function writeAll(/** @type {import('node:fs/promises').FileHandle} */ handle, /** @type {Buffer} */ bytes, /** @type {number} */ position) {
+  let offset = 0
+  while (offset < bytes.length) {
+    const {bytesWritten} = await handle.write(bytes, offset, bytes.length - offset, position + offset)
+    if (bytesWritten === 0) throw new Error("Compressed log write made no progress")
+    offset += bytesWritten
+  }
+}
+
+/**
+ *
+ * @param handle
+ * @param bytes
+ * @param position
+ */
+async function readAll(/** @type {import('node:fs/promises').FileHandle} */ handle, /** @type {Buffer} */ bytes, /** @type {number} */ position) {
+  let offset = 0
+  while (offset < bytes.length) {
+    const {bytesRead} = await handle.read(bytes, offset, bytes.length - offset, position + offset)
+    if (bytesRead === 0) throw new Error("Compressed log read made no progress")
+    offset += bytesRead
+  }
+}
+
+/**
+ *
+ * @param directory
+ */
+async function syncDirectory(/** @type {string} */ directory) {
+  const handle = await open(directory, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW)
+  try { await handle.sync() } finally { await handle.close() }
+}
+
+/**
+ *
+ * @param path
+ */
+async function assertPrivateFile(/** @type {string} */ path) {
+  const stats = await lstat(path)
+  if (!stats.isFile() || (stats.mode & 0o777) !== 0o600) throw new Error("Unsafe compressed log file")
+}
+
+/**
+ * @param {string} dataPath - Invalid segment path. @param {string} directory - Private log directory.
+ * @param directory
+ */
+async function markQuarantined(dataPath, directory) {
+  const path = `${dataPath}.quarantine`
+  if (await exists(path)) return
+  const handle = await open(path, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600)
+  try {
+    await writeAll(handle, Buffer.from("LOG_EVIDENCE_INVALID\n"), 0)
+    await handle.sync()
+  } finally { await handle.close() }
+  await syncDirectory(directory)
+}
+
+/**
+ *
+ * @param path
+ */
+async function exists(/** @type {string} */ path) {
+  try { await stat(path); return true } catch (error) { if (error !== null && typeof error === "object" && /** @type {{code?: unknown}} */ (error).code === "ENOENT") return false; throw error }
+}
+
+/**
+ * @param value
+ * @param name
+ * @template T @param {T | undefined} value - Required state. @param {string} name - State label. @returns {T}
+ */
+function required(value, name) {
+  if (value === undefined) throw new Error(`Compressed log ${name} is unavailable`)
+  return value
+}
+
+if (!isMainThread) {
+  const port = /** @type {NonNullable<typeof parentPort>} */ (parentPort)
+  const sink = new CompressedLogWorkerSink(/** @type {ConstructorParameters<typeof CompressedLogWorkerSink>[0]} */ (workerData))
+  let queue = sink.start().then((status) => port.postMessage({type: "ready", ...status}))
+  port.on("message", (message) => {
+    queue = queue.then(async () => {
+      if (message?.type === "batch") port.postMessage({type: "ack", batchId: message.batchId, ...await sink.appendBatch(message)})
+      else if (message?.type === "close") { port.postMessage({type: "closed", ...await sink.close()}); port.close() } else throw new Error("Unsupported compressed log worker message")
+    }).catch((error) => {
+      port.postMessage({type: "failed", code: "LOG_WORKER_FAILED"})
+      setImmediate(() => { throw error })
+    })
+  })
+}