socketduct 0.1.18 → 0.1.20

package/test/integration/logging-compressed-recovery.test.js

@@ -0,0 +1,186 @@
+/* eslint-disable jsdoc/escape-inline-tags, jsdoc/require-param-type, jsdoc/require-returns */
+import assert from "node:assert/strict"
+import {createHash, randomBytes} from "node:crypto"
+import {mkdtemp, readdir, rm, stat, utimes, writeFile} from "node:fs/promises"
+import {tmpdir} from "node:os"
+import {join} from "node:path"
+import test from "node:test"
+
+import {createLogBatchId, inspectLogBatch} from "../../src/logging/log-batch.js"
+import {readCompressedLogDirectory} from "../../src/logging/compressed-log-reader.js"
+import {CompressedLogWorkerSink} from "../../src/logging/compressed-log-worker.js"
+import {enforceLogRetention} from "../../src/logging/log-retention.js"
+import {createIndexEntry, createIndexHeader, INDEX_ENTRY_BYTES, INDEX_HEADER_BYTES, uuidBytes} from "../../src/logging/log-segment-index.js"
+import {compressLogFrame} from "../../src/logging/zstd-frame-codec.js"
+
+test("recovery indexes a complete data-synced orphan without duplicating its batch", async (t) => {
+  const root = await mkdtemp(join(tmpdir(), "socketduct-compressed-orphan-"))
+  t.after(() => rm(root, {recursive: true, force: true}))
+  const fixture = await createRecoveryFixture(root, false)
+  const sink = new CompressedLogWorkerSink(fixture.options)
+  const recovered = await sink.start()
+  assert.equal(recovered.durableHighWater, "2")
+  assert.equal(recovered.orphanRecoveredCount, 1)
+  assert.equal(recovered.lastDurableBatch?.batchId, fixture.secondBatchId)
+  assert.equal((await stat(fixture.indexPath)).size, INDEX_HEADER_BYTES + 2 * INDEX_ENTRY_BYTES)
+
+  const duplicate = await sink.appendBatch(fixture.secondMessage)
+  assert.equal(duplicate.duplicate, true)
+  assert.equal((await stat(fixture.dataPath)).size, fixture.firstFrame.length + fixture.secondFrame.length)
+  await sink.close()
+  const result = await readCompressedLogDirectory(root)
+  assert.equal(result.incomplete, false)
+  assert.deepEqual(result.records.map((record) => record.eventSequence), ["1", "2"])
+})
+
+test("recovery truncates only a torn final data frame and accepts its replay", async (t) => {
+  const root = await mkdtemp(join(tmpdir(), "socketduct-compressed-torn-"))
+  t.after(() => rm(root, {recursive: true, force: true}))
+  const fixture = await createRecoveryFixture(root, true)
+  const sink = new CompressedLogWorkerSink(fixture.options)
+  const recovered = await sink.start()
+  assert.equal(recovered.durableHighWater, "1")
+  assert.equal((await stat(fixture.dataPath)).size, fixture.firstFrame.length)
+
+  const replayed = await sink.appendBatch(fixture.secondMessage)
+  assert.equal(replayed.duplicate, false)
+  await sink.close()
+  const result = await readCompressedLogDirectory(root)
+  assert.equal(result.incomplete, false)
+  assert.deepEqual(result.records.map((record) => record.eventSequence), ["1", "2"])
+})
+
+test("worker acknowledgements retain only the latest durable batch identity", async (t) => {
+  const root = await mkdtemp(join(tmpdir(), "socketduct-compressed-bounded-ack-"))
+  t.after(() => rm(root, {recursive: true, force: true}))
+  const writerEpoch = randomBytes(16)
+  const options = {directory: root, role: "reverse-hub", writerEpoch: writerEpoch.toString("base64url"), maxSegmentBytes: 4096, compressionLevel: 19}
+  const sink = new CompressedLogWorkerSink(options)
+  await sink.start()
+  let latest = await sink.appendBatch(batchMessage(writerEpoch, 1))
+  for (let sequence = 2; sequence <= 40; sequence += 1) latest = await sink.appendBatch(batchMessage(writerEpoch, sequence))
+  assert.equal(Object.hasOwn(latest, "durableBatchIds"), false)
+  assert.deepEqual(latest.lastDurableBatch, {
+    batchId: batchMessage(writerEpoch, 40).batchId,
+    firstSequence: "40",
+    lastSequence: "40"
+  })
+  assert.equal((await sink.appendBatch(batchMessage(writerEpoch, 40))).duplicate, true)
+  await sink.close()
+
+  const restarted = new CompressedLogWorkerSink(options)
+  const recovered = await restarted.start()
+  assert.equal(Object.hasOwn(recovered, "durableBatchIds"), false)
+  assert.deepEqual(recovered.lastDurableBatch, latest.lastDurableBatch)
+  await restarted.close()
+})
+
+test("recovery accepts a retention-deleted prefix but rejects interior gaps", async (t) => {
+  const root = await mkdtemp(join(tmpdir(), "socketduct-compressed-retained-prefix-"))
+  t.after(() => rm(root, {recursive: true, force: true}))
+  const writerEpoch = randomBytes(16)
+  const options = {directory: root, role: "reverse-hub", writerEpoch: writerEpoch.toString("base64url"), maxSegmentBytes: 4096, compressionLevel: 19, retentionBytes: 1_000_000}
+
+  const segments = []
+  for (let sequence = 1; sequence <= 3; sequence += 1) {
+    const sink = new CompressedLogWorkerSink(options)
+    await sink.start()
+    await sink.appendBatch(batchMessage(writerEpoch, sequence))
+    segments.push(/** @type {string} */ (sink.status().segmentPath))
+    await sink.close()
+  }
+  assert.equal(segments.length, 3)
+  for (let index = 0; index < segments.length; index += 1) {
+    await utimes(/** @type {string} */ (segments[index]), new Date(1_000 + index), new Date(1_000 + index))
+  }
+  await enforceLogRetention({directory: root, maxAgeMs: 1, maxBytes: 1_000_000, now: () => new Date(1_003)})
+  assert.equal((await readdir(root)).filter((name) => name.endsWith(".jsonl.zst")).length, 1)
+
+  const recovered = new CompressedLogWorkerSink(options)
+  assert.equal((await recovered.start()).durableHighWater, "3")
+  assert.equal((await recovered.appendBatch(batchMessage(writerEpoch, 4))).durableHighWater, "4")
+  await recovered.close()
+
+  const gapDirectory = join(root, "interior-gap")
+  const gapEpoch = randomBytes(16)
+  const gapOptions = {...options, directory: gapDirectory, writerEpoch: gapEpoch.toString("base64url")}
+  const gapSegments = []
+  for (let sequence = 1; sequence <= 3; sequence += 1) {
+    const sink = new CompressedLogWorkerSink(gapOptions)
+    await sink.start()
+    await sink.appendBatch(batchMessage(gapEpoch, sequence))
+    gapSegments.push(/** @type {string} */ (sink.status().segmentPath))
+    await sink.close()
+  }
+  const missing = /** @type {string} */ (gapSegments[1])
+  await Promise.all([rm(missing), rm(`${missing}.idx`), rm(`${missing}.sealed`)])
+  await assert.rejects(() => new CompressedLogWorkerSink(gapOptions).start(), /interior sequence gap/)
+})
+
+/**
+ * @param {string} directory - Fixture directory. @param {boolean} torn - Whether to tear the orphan frame.
+ * @param torn
+ */
+async function createRecoveryFixture(directory, torn) {
+  const writerEpoch = randomBytes(16)
+  const writerEpochText = writerEpoch.toString("base64url")
+  const segmentUuidText = "123e4567-e89b-42d3-a456-426614174000"
+  const segmentUuid = uuidBytes(segmentUuidText)
+  const stem = `reverse-hub-${writerEpochText}-${segmentUuidText}.jsonl.zst`
+  const dataPath = join(directory, stem)
+  const indexPath = `${dataPath}.idx`
+  const firstRaw = record(writerEpochText, 1, "process.started")
+  const secondRaw = record(writerEpochText, 2, "runtime.status")
+  const firstFrame = await compressLogFrame(firstRaw)
+  const secondFrame = await compressLogFrame(secondRaw)
+  const first = inspectLogBatch(firstRaw, writerEpoch)
+  const firstEntry = createIndexEntry({
+    segmentUuid,
+    writerEpoch,
+    batchId: first.batchId,
+    firstSequence: first.firstSequence,
+    lastSequence: first.lastSequence,
+    dataOffset: 0n,
+    compressedLength: BigInt(firstFrame.length),
+    uncompressedLength: BigInt(firstRaw.length),
+    recordCount: 1,
+    minTimestamp: first.minTimestamp,
+    maxTimestamp: first.maxTimestamp,
+    frameSha256: createHash("sha256").update(firstFrame).digest()
+  })
+  await writeFile(dataPath, Buffer.concat([firstFrame, torn ? secondFrame.subarray(0, secondFrame.length - 3) : secondFrame]), {mode: 0o600})
+  await writeFile(indexPath, Buffer.concat([createIndexHeader({segmentUuid, writerEpoch}), firstEntry]), {mode: 0o600})
+  const secondBatchId = createLogBatchId(writerEpoch, 2n, 2n, secondRaw).toString("base64url")
+  return {
+    options: {directory, role: "reverse-hub", writerEpoch: writerEpochText, maxSegmentBytes: 4096, compressionLevel: 19},
+    dataPath,
+    indexPath,
+    firstFrame,
+    secondFrame,
+    secondBatchId,
+    secondMessage: {batchId: secondBatchId, firstSequence: "2", lastSequence: "2", jsonl: secondRaw}
+  }
+}
+
+/**
+ * @param {string} writerEpoch - Epoch. @param {number} sequence - Event sequence. @param {string} event - Event name.
+ * @param sequence
+ * @param event
+ */
+function record(writerEpoch, sequence, event) {
+  return Buffer.from(`${JSON.stringify({timestamp: new Date(1_795_000_000_000 + sequence).toISOString(), event, processId: 1, role: "reverse-hub", writerEpoch, eventSequence: String(sequence)})}\n`)
+}
+
+/**
+ * @param {Buffer} writerEpoch - Epoch.
+ * @param {number} sequence - Event sequence.
+ */
+function batchMessage(writerEpoch, sequence) {
+  const jsonl = record(writerEpoch.toString("base64url"), sequence, "runtime.status")
+  return {
+    batchId: createLogBatchId(writerEpoch, BigInt(sequence), BigInt(sequence), jsonl).toString("base64url"),
+    firstSequence: String(sequence),
+    lastSequence: String(sequence),
+    jsonl
+  }
+}