socketduct 0.1.19 → 0.1.21

package/src/logging/compressed-log-reader.js

@@ -0,0 +1,256 @@
+/* eslint-disable jsdoc/escape-inline-tags, jsdoc/require-param-type, jsdoc/require-returns */
+import {createHash} from "node:crypto"
+import {constants} from "node:fs"
+import {lstat, open, readdir, stat} from "node:fs/promises"
+import {join} from "node:path"
+
+import {inspectLogBatch} from "./log-batch.js"
+import {INDEX_ENTRY_BYTES, INDEX_HEADER_BYTES, parseIndexEntry, parseIndexHeader, uuidBytes} from "./log-segment-index.js"
+import {decompressLogFrame} from "./zstd-frame-codec.js"
+
+const SEGMENT_NAME = /^([a-z][a-z0-9-]{0,63})-([A-Za-z0-9_-]{22})-([a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12})\.jsonl\.zst$/
+const DEFAULT_MAX_DECODED_FRAME_BYTES = 16 * 1024 * 1024
+
+/** Incremental durable-index reader for long-running tail consumers. */
+export class CompressedLogTailReader {
+  /** @type {string} */ #directory
+  /** @type {{filters?: Record<string, unknown>, maxDecodedFrameBytes?: number}} */ #options
+  /** @type {Map<string, {nextEntry: number, expectedSequence: bigint | undefined, expectedOffset: bigint, sealed: boolean}>} */ #segments = new Map()
+  /** @type {number} */ #decodedFrames = 0
+
+  /**
+   * @param {string} directory - Private compressed-log directory.
+   * @param {{filters?: Record<string, unknown>, maxDecodedFrameBytes?: number}} [options] - Read policy.
+   */
+  constructor(directory, options = {}) {
+    if (typeof directory !== "string" || directory.length === 0) throw new TypeError("compressed log directory is required")
+    this.#directory = directory
+    this.#options = options
+  }
+
+  /** Reads only index entries committed since the prior poll. */
+  async poll() {
+    await assertPrivateDirectory(this.#directory)
+    const entries = (await readdir(this.#directory)).filter((entry) => entry.endsWith(".jsonl.zst")).sort()
+    const present = new Set(entries)
+    for (const name of this.#segments.keys()) if (!present.has(name)) this.#segments.delete(name)
+    /** @type {Array<Record<string, unknown>>} */
+    const records = []
+    /** @type {Array<{segment: string, code: string}>} */
+    const errors = []
+    for (const entry of entries) {
+      const previous = this.#segments.get(entry)
+      if (previous?.sealed === true) continue
+      try {
+        const result = await readCompressedLogSegmentIncremental(join(this.#directory, entry), this.#options, previous)
+        this.#segments.set(entry, result.cursor)
+        this.#decodedFrames = Math.min(Number.MAX_SAFE_INTEGER, this.#decodedFrames + result.decodedFrames)
+        records.push(...result.records)
+      } catch (error) {
+        if (evidenceErrorCode(error) === "ENOENT") {
+          this.#segments.delete(entry)
+          continue
+        }
+        errors.push({segment: entry, code: evidenceErrorCode(error)})
+      }
+    }
+    records.sort(compareRecords)
+    const filtered = this.#options.filters === undefined ? records : records.filter((record) => matchesFilters(record, this.#options.filters ?? {}))
+    return {records: filtered, incomplete: errors.length > 0, errors}
+  }
+
+  /** Returns bounded cursor count and cumulative decode work for health/tests. */
+  status() { return {trackedSegments: this.#segments.size, decodedFrames: this.#decodedFrames} }
+}
+
+/**
+ * Reads and validates every durable indexed frame, preserving evidence errors.
+ * @param directory
+ * @param options
+ */
+export async function readCompressedLogDirectory(/** @type {string} */ directory, /** @type {{filters?: Record<string, unknown>, maxDecodedFrameBytes?: number}} */ options = {}) {
+  await assertPrivateDirectory(directory)
+  const entries = await readdir(directory)
+  /** @type {Array<Record<string, unknown>>} */
+  const records = []
+  /** @type {Array<{segment: string, code: string}>} */
+  const errors = []
+  for (const entry of entries.sort()) {
+    if (!entry.endsWith(".jsonl.zst")) continue
+    try {
+      const selected = await readCompressedLogSegment(join(directory, entry), options)
+      records.push(...selected)
+    } catch (error) {
+      errors.push({segment: entry, code: evidenceErrorCode(error)})
+    }
+  }
+  records.sort(compareRecords)
+  const filtered = options.filters === undefined ? records : records.filter((record) => matchesFilters(record, options.filters ?? {}))
+  return {records: filtered, incomplete: errors.length > 0, errors}
+}
+
+/**
+ * Reads one segment frame-by-frame from its durable index.
+ * @param dataPath
+ * @param options
+ */
+export async function readCompressedLogSegment(/** @type {string} */ dataPath, /** @type {{maxDecodedFrameBytes?: number}} */ options = {}) {
+  return (await readCompressedLogSegmentIncremental(dataPath, options)).records
+}
+
+/**
+ * Reads newly indexed frames from one segment and advances its validated cursor.
+ * @param {string} dataPath - Segment data path.
+ * @param {{maxDecodedFrameBytes?: number}} options - Decode bounds.
+ * @param {{nextEntry: number, expectedSequence: bigint | undefined, expectedOffset: bigint, sealed: boolean}} [cursor] - Prior cursor.
+ */
+async function readCompressedLogSegmentIncremental(dataPath, options, cursor = undefined) {
+  const name = dataPath.slice(dataPath.lastIndexOf("/") + 1)
+  const match = SEGMENT_NAME.exec(name)
+  if (match === null) throw evidenceError("INVALID_SEGMENT_NAME")
+  const writerEpoch = Buffer.from(/** @type {string} */ (match[2]), "base64url")
+  const segmentUuid = uuidBytes(/** @type {string} */ (match[3]))
+  if (writerEpoch.length !== 16) throw evidenceError("INVALID_WRITER_EPOCH")
+  const indexPath = `${dataPath}.idx`
+  if (await pathExists(`${dataPath}.quarantine`)) throw evidenceError("LOG_EVIDENCE_QUARANTINED")
+  await Promise.all([assertPrivateRegularFile(dataPath), assertPrivateRegularFile(indexPath)])
+  const [dataHandle, indexHandle] = await Promise.all([
+    open(dataPath, constants.O_RDONLY | constants.O_NOFOLLOW),
+    open(indexPath, constants.O_RDONLY | constants.O_NOFOLLOW)
+  ])
+  try {
+    const [dataStats, indexStats] = await Promise.all([dataHandle.stat(), indexHandle.stat()])
+    if (indexStats.size < INDEX_HEADER_BYTES) throw evidenceError("INDEX_HEADER_TORN")
+    const headerBytes = Buffer.alloc(INDEX_HEADER_BYTES)
+    await readExactly(indexHandle, headerBytes, 0)
+    const header = parseIndexHeader(headerBytes)
+    if (!header.segmentUuid.equals(segmentUuid) || !header.writerEpoch.equals(writerEpoch)) throw evidenceError("INDEX_IDENTITY_MISMATCH")
+    const completeEntryBytes = indexStats.size - INDEX_HEADER_BYTES
+    const sealed = await pathExists(`${dataPath}.sealed`)
+    if (completeEntryBytes % INDEX_ENTRY_BYTES !== 0 && sealed) throw evidenceError("INDEX_TAIL_TORN")
+    const entryCount = Math.floor(completeEntryBytes / INDEX_ENTRY_BYTES)
+    const nextEntry = cursor?.nextEntry ?? 0
+    if (!Number.isSafeInteger(nextEntry) || nextEntry < 0 || nextEntry > entryCount) throw evidenceError("INDEX_CURSOR_INVALID")
+    /** @type {Array<Record<string, unknown>>} */
+    const records = []
+    let expectedSequence = cursor?.expectedSequence
+    let expectedOffset = cursor?.expectedOffset ?? 0n
+    let decodedFrames = 0
+    for (let index = nextEntry; index < entryCount; index += 1) {
+      const entryBytes = Buffer.alloc(INDEX_ENTRY_BYTES)
+      await readExactly(indexHandle, entryBytes, INDEX_HEADER_BYTES + index * INDEX_ENTRY_BYTES)
+      const entry = parseIndexEntry(entryBytes, {segmentUuid, writerEpoch})
+      if (expectedSequence !== undefined && entry.firstSequence !== expectedSequence) throw evidenceError("INDEX_SEQUENCE_GAP")
+      if (entry.dataOffset !== expectedOffset) throw evidenceError("INDEX_DATA_GAP")
+      if (entry.dataOffset + entry.compressedLength > BigInt(dataStats.size)) throw evidenceError("INDEX_DATA_BOUNDS")
+      if (entry.compressedLength > BigInt(Number.MAX_SAFE_INTEGER) || entry.uncompressedLength > BigInt(options.maxDecodedFrameBytes ?? DEFAULT_MAX_DECODED_FRAME_BYTES)) throw evidenceError("FRAME_SIZE_LIMIT")
+      const frame = Buffer.alloc(Number(entry.compressedLength))
+      await readExactly(dataHandle, frame, Number(entry.dataOffset))
+      if (!createHash("sha256").update(frame).digest().equals(entry.frameSha256)) throw evidenceError("FRAME_HASH_MISMATCH")
+      const raw = await decompressLogFrame(frame)
+      if (BigInt(raw.length) !== entry.uncompressedLength) throw evidenceError("FRAME_LENGTH_MISMATCH")
+      const inspected = inspectLogBatch(raw, writerEpoch)
+      if (inspected.firstSequence !== entry.firstSequence || inspected.lastSequence !== entry.lastSequence || inspected.recordCount !== entry.recordCount || inspected.minTimestamp !== entry.minTimestamp || inspected.maxTimestamp !== entry.maxTimestamp || !inspected.batchId.equals(entry.batchId)) throw evidenceError("FRAME_METADATA_MISMATCH")
+      records.push(...inspected.records)
+      decodedFrames += 1
+      expectedSequence = entry.lastSequence + 1n
+      expectedOffset = entry.dataOffset + entry.compressedLength
+    }
+    if (sealed && expectedOffset !== BigInt(dataStats.size)) throw evidenceError("SEALED_DATA_TAIL")
+    return {records, decodedFrames, cursor: {nextEntry: entryCount, expectedSequence, expectedOffset, sealed}}
+  } finally {
+    await Promise.allSettled([dataHandle.close(), indexHandle.close()])
+  }
+}
+
+/**
+ * @param {import('node:fs/promises').FileHandle} handle - Open file. @param {Buffer} buffer - Exact destination. @param {number} position - File position.
+ * @param buffer
+ * @param position
+ */
+async function readExactly(handle, buffer, position) {
+  let offset = 0
+  while (offset < buffer.length) {
+    const {bytesRead} = await handle.read(buffer, offset, buffer.length - offset, position + offset)
+    if (bytesRead === 0) throw evidenceError("SHORT_READ")
+    offset += bytesRead
+  }
+}
+
+/**
+ *
+ * @param path
+ */
+async function assertPrivateDirectory(/** @type {string} */ path) {
+  const stats = await lstat(path)
+  if (!stats.isDirectory() || (stats.mode & 0o777) !== 0o700) throw evidenceError("UNSAFE_LOG_DIRECTORY")
+}
+
+/**
+ *
+ * @param path
+ */
+async function assertPrivateRegularFile(/** @type {string} */ path) {
+  const stats = await lstat(path)
+  if (!stats.isFile() || (stats.mode & 0o777) !== 0o600) throw evidenceError("UNSAFE_LOG_FILE")
+}
+
+/**
+ *
+ * @param path
+ */
+async function pathExists(/** @type {string} */ path) {
+  try { await stat(path); return true } catch (error) { if (error !== null && typeof error === "object" && /** @type {{code?: unknown}} */ (error).code === "ENOENT") return false; throw error }
+}
+
+/**
+ *
+ * @param record
+ * @param filters
+ */
+function matchesFilters(/** @type {Record<string, unknown>} */ record, /** @type {Record<string, unknown>} */ filters) {
+  const mappings = {
+    event: "event",
+    incident: "disconnectIncidentId",
+    gateway: "gatewayId",
+    routeClass: "routeClass",
+    clientGeneration: "clientConnectionGeneration",
+    gatewayGeneration: "gatewayConnectionGeneration",
+    reason: "disconnectReason"
+  }
+  for (const [filter, field] of Object.entries(mappings)) if (filters[filter] !== undefined && String(record[field]) !== String(filters[filter])) return false
+  const timestamp = typeof record.timestamp === "string" ? Date.parse(record.timestamp) : Number.NaN
+  if (filters.since !== undefined && timestamp < /** @type {number} */ (filters.since)) return false
+  if (filters.until !== undefined && timestamp > /** @type {number} */ (filters.until)) return false
+  return true
+}
+
+/**
+ *
+ * @param left
+ * @param right
+ */
+function compareRecords(/** @type {Record<string, unknown>} */ left, /** @type {Record<string, unknown>} */ right) {
+  const time = String(left.timestamp).localeCompare(String(right.timestamp))
+  if (time !== 0) return time
+  const leftEpoch = Buffer.from(String(left.writerEpoch), "base64url")
+  const rightEpoch = Buffer.from(String(right.writerEpoch), "base64url")
+  const epoch = Buffer.compare(leftEpoch, rightEpoch)
+  return epoch !== 0 ? epoch : BigInt(String(left.eventSequence)) < BigInt(String(right.eventSequence)) ? -1 : BigInt(String(left.eventSequence)) > BigInt(String(right.eventSequence)) ? 1 : 0
+}
+
+/**
+ *
+ * @param error
+ */
+function evidenceErrorCode(/** @type {unknown} */ error) {
+  return error !== null && typeof error === "object" && typeof /** @type {{code?: unknown}} */ (error).code === "string" ? /** @type {{code: string}} */ (error).code : "LOG_EVIDENCE_INVALID"
+}
+
+/**
+ *
+ * @param code
+ */
+function evidenceError(/** @type {string} */ code) {
+  return Object.assign(new Error("Compressed log evidence is incomplete or corrupt"), {code})
+}