socketduct 0.1.19 → 0.1.20

package/src/logging/log-retention.js

@@ -0,0 +1,241 @@
+/* eslint-disable jsdoc/escape-inline-tags, jsdoc/require-param-type, jsdoc/require-returns */
+import {randomUUID} from "node:crypto"
+import {constants, readlinkSync} from "node:fs"
+import {link, lstat, open, readFile, readdir, stat, unlink} from "node:fs/promises"
+import {join} from "node:path"
+
+const SEGMENT_NAME = /^[a-z][a-z0-9-]{0,63}-[A-Za-z0-9_-]{22}-[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}\.jsonl\.zst$/
+const OWNER_TEMPORARY_NAME = /^\.socketduct-retention-owner-([1-9][0-9]*)-([0-9]+)-([1-9][0-9]*)-([0-9]+)-[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}\.tmp$/
+
+/**
+ * Enforces time and aggregate-byte bounds without deleting active/quarantined evidence.
+ * @param options
+ */
+export async function enforceLogRetention(/** @type {{directory: string, maxAgeMs: number, maxBytes: number, now?: () => Date}} */ options) {
+  const now = options.now ?? (() => new Date())
+  if (!Number.isSafeInteger(options.maxAgeMs) || options.maxAgeMs < 1) throw new RangeError("maxAgeMs must be a positive safe integer")
+  if (!Number.isSafeInteger(options.maxBytes) || options.maxBytes < 1) throw new RangeError("maxBytes must be a positive safe integer")
+  const timestamp = now()
+  if (!(timestamp instanceof Date) || !Number.isFinite(timestamp.getTime())) throw new TypeError("retention clock returned an invalid date")
+  const directoryStats = await lstat(options.directory)
+  if (!directoryStats.isDirectory() || (directoryStats.mode & 0o777) !== 0o700) throw new Error("Unsafe compressed log directory")
+  const lockPath = join(options.directory, ".socketduct-retention.lock")
+  const lock = await acquireRetentionLock(options.directory, lockPath)
+  if (lock === undefined) return {skipped: true, deletedSegments: 0, retainedBytes: await directoryBytes(options.directory)}
+  try {
+    const names = await readdir(options.directory)
+    /** @type {Array<{name: string, dataPath: string, paths: string[], bytes: number, modifiedAt: number, sealed: boolean, quarantined: boolean, deleted: boolean}>} */
+    const segments = []
+    let retainedBytes = 0
+    for (const name of names) {
+      if (!SEGMENT_NAME.test(name)) continue
+      const dataPath = join(options.directory, name)
+      const paths = [dataPath, `${dataPath}.idx`]
+      const sealed = await exists(`${dataPath}.sealed`)
+      const quarantined = await exists(`${dataPath}.quarantine`)
+      if (sealed) paths.push(`${dataPath}.sealed`)
+      if (quarantined) paths.push(`${dataPath}.quarantine`)
+      let bytes = 0
+      let modifiedAt = 0
+      for (const path of paths) {
+        const stats = await lstat(path)
+        if (!stats.isFile() || (stats.mode & 0o777) !== 0o600) throw new Error("Unsafe compressed log retention file")
+        bytes += stats.size
+        if (path === dataPath) modifiedAt = stats.mtimeMs
+      }
+      retainedBytes += bytes
+      segments.push({name, dataPath, paths, bytes, modifiedAt, sealed, quarantined, deleted: false})
+    }
+    segments.sort((left, right) => left.modifiedAt !== right.modifiedAt ? left.modifiedAt - right.modifiedAt : left.name.localeCompare(right.name))
+    let deletedSegments = 0
+    const cutoff = timestamp.getTime() - options.maxAgeMs
+    for (const segment of segments) {
+      if (!segment.sealed || segment.quarantined || segment.modifiedAt >= cutoff) continue
+      await deleteSegment(segment)
+      retainedBytes -= segment.bytes
+      deletedSegments += 1
+    }
+    for (const segment of segments) {
+      if (retainedBytes <= options.maxBytes) break
+      if (segment.deleted || !segment.sealed || segment.quarantined) continue
+      await deleteSegment(segment)
+      retainedBytes -= segment.bytes
+      deletedSegments += 1
+    }
+    if (retainedBytes > options.maxBytes) throw Object.assign(new Error("Active or quarantined compressed logs prevent quota compliance"), {code: "LOG_RETENTION_QUOTA_BLOCKED"})
+    if (deletedSegments > 0) await syncDirectory(options.directory)
+    return {skipped: false, deletedSegments, retainedBytes}
+  } finally {
+    await releaseRetentionLock(options.directory, lockPath, lock)
+  }
+}
+
+/**
+ * @param {string} directory - Directory.
+ * @param {string} lockPath - Lock path.
+ * @param {{dev: number, ino: number}} lock - Owned inode.
+ */
+async function releaseRetentionLock(directory, lockPath, lock) {
+  const current = await lstat(lockPath).catch((error) => { if (isCode(error, "ENOENT")) return undefined; throw error })
+  if (current !== undefined && (current.dev !== lock.dev || current.ino !== lock.ino)) throw Object.assign(new Error("Retention lock ownership changed while held"), {code: "LOG_RETENTION_LOCK_RACE"})
+  if (current !== undefined) await unlink(lockPath)
+  await syncDirectory(directory)
+}
+
+/**
+ * Acquires a complete, private owner record and reclaims only a provably stale
+ * process/thread instance. Kernel start times prevent PID/TID-reuse confusion.
+ * @param {string} directory - Private log directory.
+ * @param {string} lockPath - Exclusive lock path.
+ */
+async function acquireRetentionLock(directory, lockPath) {
+  const owner = await localOwnerIdentity()
+  await removeStaleOwnerFiles(directory)
+  for (let attempt = 0; attempt < 3; attempt += 1) {
+    const created = await createOwnerFile(directory, lockPath, owner)
+    if (created !== undefined) return created
+    const existing = await readOwnerFile(lockPath)
+    if (existing === undefined) continue
+    if (await ownerIsLive(existing.owner)) return undefined
+    const current = await lstat(lockPath).catch((error) => { if (isCode(error, "ENOENT")) return undefined; throw error })
+    if (current === undefined) continue
+    if (current.dev !== existing.stats.dev || current.ino !== existing.stats.ino) return undefined
+    await unlink(lockPath).catch((error) => { if (!isCode(error, "ENOENT")) throw error })
+    await syncDirectory(directory)
+  }
+  return undefined
+}
+
+/**
+ * @param {string} directory - Directory.
+ * @param {string} lockPath - Destination.
+ * @param {{version: 1, pid: number, processStart: string, tid: number, threadStart: string}} owner - Owner identity.
+ */
+async function createOwnerFile(directory, lockPath, owner) {
+  const temporary = join(directory, `.socketduct-retention-owner-${owner.pid}-${owner.processStart}-${owner.tid}-${owner.threadStart}-${randomUUID()}.tmp`)
+  const handle = await open(temporary, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600)
+  try {
+    await handle.writeFile(`${JSON.stringify(owner)}\n`)
+    await handle.sync()
+  } finally {
+    await handle.close()
+  }
+  try {
+    await link(temporary, lockPath)
+    const stats = await lstat(lockPath)
+    if (!stats.isFile() || (stats.mode & 0o777) !== 0o600) throw Object.assign(new Error("Unsafe retention lock"), {code: "LOG_RETENTION_LOCK_INVALID"})
+    await syncDirectory(directory)
+    return {dev: stats.dev, ino: stats.ino}
+  } catch (error) {
+    if (isCode(error, "EEXIST")) return undefined
+    throw error
+  } finally {
+    await unlink(temporary).catch((error) => { if (!isCode(error, "ENOENT")) throw error })
+  }
+}
+
+/** @param {string} directory - Private log directory. */
+async function removeStaleOwnerFiles(directory) {
+  let removed = false
+  for (const name of await readdir(directory)) {
+    const match = OWNER_TEMPORARY_NAME.exec(name)
+    if (match === null) continue
+    const owner = {pid: Number(match[1]), processStart: /** @type {string} */ (match[2]), tid: Number(match[3]), threadStart: /** @type {string} */ (match[4])}
+    if (!Number.isSafeInteger(owner.pid) || owner.pid < 1 || !Number.isSafeInteger(owner.tid) || owner.tid < 1) throw Object.assign(new Error("Invalid retention owner temporary"), {code: "LOG_RETENTION_LOCK_INVALID"})
+    if (await ownerIsLive(owner)) continue
+    const path = join(directory, name)
+    const stats = await lstat(path).catch((error) => { if (isCode(error, "ENOENT")) return undefined; throw error })
+    if (stats === undefined) continue
+    if (!stats.isFile() || (stats.mode & 0o777) !== 0o600) throw Object.assign(new Error("Unsafe retention owner temporary"), {code: "LOG_RETENTION_LOCK_INVALID"})
+    await unlink(path).catch((error) => { if (!isCode(error, "ENOENT")) throw error })
+    removed = true
+  }
+  if (removed) await syncDirectory(directory)
+}
+
+/** @param {string} lockPath - Existing lock path. */
+async function readOwnerFile(lockPath) {
+  let handle
+  try { handle = await open(lockPath, constants.O_RDONLY | constants.O_NOFOLLOW) } catch (error) { if (isCode(error, "ENOENT")) return undefined; throw error }
+  try {
+    const stats = await handle.stat()
+    if (!stats.isFile() || (stats.mode & 0o777) !== 0o600 || stats.size < 2 || stats.size > 512) throw Object.assign(new Error("Invalid retention lock owner record"), {code: "LOG_RETENTION_LOCK_INVALID"})
+    let value
+    try { value = JSON.parse(await handle.readFile("utf8")) } catch { throw Object.assign(new Error("Invalid retention lock owner record"), {code: "LOG_RETENTION_LOCK_INVALID"}) }
+    if (typeof value !== "object" || value === null || Array.isArray(value)) throw Object.assign(new Error("Invalid retention lock owner record"), {code: "LOG_RETENTION_LOCK_INVALID"})
+    const candidate = /** @type {Record<string, unknown>} */ (value)
+    if (candidate.version !== 1 || !Number.isSafeInteger(candidate.pid) || /** @type {number} */ (candidate.pid) < 1 || typeof candidate.processStart !== "string" || !/^[0-9]+$/.test(candidate.processStart) || !Number.isSafeInteger(candidate.tid) || /** @type {number} */ (candidate.tid) < 1 || typeof candidate.threadStart !== "string" || !/^[0-9]+$/.test(candidate.threadStart)) throw Object.assign(new Error("Invalid retention lock owner record"), {code: "LOG_RETENTION_LOCK_INVALID"})
+    return {stats, owner: /** @type {{version: 1, pid: number, processStart: string, tid: number, threadStart: string}} */ (candidate)}
+  } finally {
+    await handle.close()
+  }
+}
+
+/** Returns the current Linux process/thread-instance identity. */
+async function localOwnerIdentity() {
+  const processStart = await readProcessStart(process.pid)
+  if (processStart === undefined) throw Object.assign(new Error("Cannot identify retention owner process"), {code: "LOG_RETENTION_OWNER_UNAVAILABLE"})
+  const target = readlinkSync("/proc/thread-self")
+  const tidText = target.slice(target.lastIndexOf("/") + 1)
+  if (!/^[0-9]+$/.test(tidText)) throw Object.assign(new Error("Cannot identify retention owner thread"), {code: "LOG_RETENTION_OWNER_UNAVAILABLE"})
+  const tid = Number(tidText)
+  const threadStart = await readProcessStart(process.pid, tid)
+  if (!Number.isSafeInteger(tid) || tid < 1 || threadStart === undefined) throw Object.assign(new Error("Cannot identify retention owner thread"), {code: "LOG_RETENTION_OWNER_UNAVAILABLE"})
+  return {version: /** @type {const} */ (1), pid: process.pid, processStart, tid, threadStart}
+}
+
+/** @param {{pid: number, processStart: string, tid: number, threadStart: string}} owner - Recorded owner. */
+async function ownerIsLive(owner) {
+  const [currentProcessStart, currentThreadStart] = await Promise.all([readProcessStart(owner.pid), readProcessStart(owner.pid, owner.tid)])
+  return currentProcessStart === owner.processStart && currentThreadStart === owner.threadStart
+}
+
+/**
+ * @param {number} pid - Linux process ID.
+ * @param {number} [tid] - Optional Linux task ID.
+ */
+async function readProcessStart(pid, tid = undefined) {
+  let value
+  const path = tid === undefined ? `/proc/${pid}/stat` : `/proc/${pid}/task/${tid}/stat`
+  try { value = await readFile(path, "utf8") } catch (error) { if (isCode(error, "ENOENT") || isCode(error, "ESRCH")) return undefined; throw error }
+  const close = value.lastIndexOf(")")
+  if (close < 0) throw Object.assign(new Error("Invalid process identity record"), {code: "LOG_RETENTION_OWNER_UNAVAILABLE"})
+  const processStart = value.slice(close + 2).trim().split(/\s+/)[19]
+  if (processStart === undefined || !/^[0-9]+$/.test(processStart)) throw Object.assign(new Error("Invalid process identity record"), {code: "LOG_RETENTION_OWNER_UNAVAILABLE"})
+  return processStart
+}
+
+/** @param {{paths: string[], deleted: boolean}} segment - Closed segment files. */
+async function deleteSegment(segment) {
+  for (const path of [...segment.paths].reverse()) await unlink(path)
+  segment.deleted = true
+}
+
+/** @param {string} directory - Log directory. */
+async function directoryBytes(directory) {
+  let bytes = 0
+  for (const name of await readdir(directory)) {
+    const stats = await lstat(join(directory, name))
+    if (stats.isFile()) bytes += stats.size
+  }
+  return bytes
+}
+
+/** @param {string} path - Candidate path. */
+async function exists(path) {
+  try { await stat(path); return true } catch (error) { if (isCode(error, "ENOENT")) return false; throw error }
+}
+
+/** @param {string} directory - Directory to sync. */
+async function syncDirectory(directory) {
+  const handle = await open(directory, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW)
+  try { await handle.sync() } finally { await handle.close() }
+}
+
+/**
+ * @param {unknown} error - Candidate system error. @param {string} code - Expected code.
+ * @param code
+ */
+function isCode(error, code) {
+  return error !== null && typeof error === "object" && /** @type {{code?: unknown}} */ (error).code === code
+}

package/src/logging/log-segment-index.js

@@ -0,0 +1,176 @@
+/* eslint-disable jsdoc/escape-inline-tags, jsdoc/require-param-type, jsdoc/require-returns */
+export const INDEX_HEADER_BYTES = 48
+export const INDEX_ENTRY_BYTES = 156
+
+const HEADER_MAGIC = Buffer.from([0x53, 0x44, 0x5a, 0x49, 0x44, 0x58, 0x31, 0x00])
+const ENTRY_MAGIC = Buffer.from("FRM1", "ascii")
+const UINT64_MAX = (1n << 64n) - 1n
+const CRC32C_TABLE = createCrc32cTable()
+
+/**
+ * Encodes the exact 48-byte v1 sidecar header.
+ * @param options
+ */
+export function createIndexHeader(/** @type {{segmentUuid: Buffer | Uint8Array, writerEpoch: Buffer | Uint8Array}} */ options) {
+  const segmentUuid = fixedBytes(options.segmentUuid, 16, "segmentUuid")
+  const writerEpoch = fixedBytes(options.writerEpoch, 16, "writerEpoch")
+  const header = Buffer.alloc(INDEX_HEADER_BYTES)
+  HEADER_MAGIC.copy(header, 0)
+  header.writeUInt16BE(1, 8)
+  header.writeUInt16BE(INDEX_HEADER_BYTES, 10)
+  segmentUuid.copy(header, 12)
+  writerEpoch.copy(header, 28)
+  header.writeUInt32BE(crc32c(header.subarray(0, 44)), 44)
+  return header
+}
+
+/**
+ * Parses and authenticates the structural v1 header.
+ * @param bytes
+ */
+export function parseIndexHeader(/** @type {Buffer | Uint8Array} */ bytes) {
+  const header = Buffer.from(bytes)
+  if (header.length !== INDEX_HEADER_BYTES) throw new Error("Invalid log index header length")
+  if (!header.subarray(0, 8).equals(HEADER_MAGIC)) throw new Error("Invalid log index header magic")
+  if (header.readUInt16BE(8) !== 1 || header.readUInt16BE(10) !== INDEX_HEADER_BYTES) throw new Error("Unsupported log index header version")
+  if (header.readUInt32BE(44) !== crc32c(header.subarray(0, 44))) throw new Error("Invalid log index header CRC32C")
+  return {version: 1, segmentUuid: Buffer.from(header.subarray(12, 28)), writerEpoch: Buffer.from(header.subarray(28, 44))}
+}
+
+/**
+ * Encodes one exact 156-byte v1 frame entry.
+ * @param options
+ */
+export function createIndexEntry(/** @type {Record<string, unknown>} */ options) {
+  const segmentUuid = fixedBytes(options.segmentUuid, 16, "segmentUuid")
+  const writerEpoch = fixedBytes(options.writerEpoch, 16, "writerEpoch")
+  const batchId = fixedBytes(options.batchId, 16, "batchId")
+  const frameSha256 = fixedBytes(options.frameSha256, 32, "frameSha256")
+  const firstSequence = uint64(options.firstSequence, "firstSequence")
+  const lastSequence = uint64(options.lastSequence, "lastSequence")
+  const dataOffset = uint64(options.dataOffset, "dataOffset")
+  const compressedLength = uint64(options.compressedLength, "compressedLength")
+  const uncompressedLength = uint64(options.uncompressedLength, "uncompressedLength")
+  const minTimestamp = uint64(options.minTimestamp, "minTimestamp")
+  const maxTimestamp = uint64(options.maxTimestamp, "maxTimestamp")
+  const recordCount = uint32(options.recordCount, "recordCount")
+  if (firstSequence === 0n || lastSequence < firstSequence || BigInt(recordCount) !== lastSequence - firstSequence + 1n) throw new RangeError("Index sequence range must be contiguous with recordCount")
+  if (compressedLength === 0n || uncompressedLength === 0n) throw new RangeError("Index frame lengths must be positive")
+  if (maxTimestamp < minTimestamp) throw new RangeError("Index timestamp range is invalid")
+  const entry = Buffer.alloc(INDEX_ENTRY_BYTES)
+  ENTRY_MAGIC.copy(entry, 0)
+  entry.writeUInt32BE(INDEX_ENTRY_BYTES, 4)
+  entry.writeUInt16BE(1, 8)
+  entry.writeUInt16BE(0, 10)
+  segmentUuid.copy(entry, 12)
+  writerEpoch.copy(entry, 28)
+  batchId.copy(entry, 44)
+  entry.writeBigUInt64BE(firstSequence, 60)
+  entry.writeBigUInt64BE(lastSequence, 68)
+  entry.writeBigUInt64BE(dataOffset, 76)
+  entry.writeBigUInt64BE(compressedLength, 84)
+  entry.writeBigUInt64BE(uncompressedLength, 92)
+  entry.writeUInt32BE(recordCount, 100)
+  entry.writeBigUInt64BE(minTimestamp, 104)
+  entry.writeBigUInt64BE(maxTimestamp, 112)
+  frameSha256.copy(entry, 120)
+  entry.writeUInt32BE(crc32c(entry.subarray(0, 152)), 152)
+  return entry
+}
+
+/**
+ * Parses one entry and requires it to match the selected segment identity.
+ * @param bytes
+ * @param identity
+ */
+export function parseIndexEntry(/** @type {Buffer | Uint8Array} */ bytes, /** @type {{segmentUuid: Buffer | Uint8Array, writerEpoch: Buffer | Uint8Array}} */ identity) {
+  const entry = Buffer.from(bytes)
+  if (entry.length !== INDEX_ENTRY_BYTES) throw new Error("Invalid log index entry length")
+  if (!entry.subarray(0, 4).equals(ENTRY_MAGIC) || entry.readUInt32BE(4) !== INDEX_ENTRY_BYTES) throw new Error("Invalid log index entry framing")
+  if (entry.readUInt16BE(8) !== 1 || entry.readUInt16BE(10) !== 0) throw new Error("Unsupported log index entry version")
+  if (entry.readUInt32BE(152) !== crc32c(entry.subarray(0, 152))) throw new Error("Invalid log index entry CRC32C")
+  const expectedSegment = fixedBytes(identity.segmentUuid, 16, "segmentUuid")
+  const expectedEpoch = fixedBytes(identity.writerEpoch, 16, "writerEpoch")
+  const segmentUuid = Buffer.from(entry.subarray(12, 28))
+  const writerEpoch = Buffer.from(entry.subarray(28, 44))
+  if (!segmentUuid.equals(expectedSegment) || !writerEpoch.equals(expectedEpoch)) throw new Error("Log index entry identity mismatch")
+  const parsed = {
+    version: 1,
+    flags: 0,
+    segmentUuid,
+    writerEpoch,
+    batchId: Buffer.from(entry.subarray(44, 60)),
+    firstSequence: entry.readBigUInt64BE(60),
+    lastSequence: entry.readBigUInt64BE(68),
+    dataOffset: entry.readBigUInt64BE(76),
+    compressedLength: entry.readBigUInt64BE(84),
+    uncompressedLength: entry.readBigUInt64BE(92),
+    recordCount: entry.readUInt32BE(100),
+    minTimestamp: entry.readBigUInt64BE(104),
+    maxTimestamp: entry.readBigUInt64BE(112),
+    frameSha256: Buffer.from(entry.subarray(120, 152))
+  }
+  if (parsed.firstSequence === 0n || parsed.lastSequence < parsed.firstSequence || BigInt(parsed.recordCount) !== parsed.lastSequence - parsed.firstSequence + 1n) throw new Error("Invalid log index sequence range")
+  if (parsed.compressedLength === 0n || parsed.uncompressedLength === 0n || parsed.maxTimestamp < parsed.minTimestamp) throw new Error("Invalid log index frame range")
+  return parsed
+}
+
+/**
+ * Converts the canonical UUID text representation to its 16 raw bytes.
+ * @param value
+ */
+export function uuidBytes(/** @type {string} */ value) {
+  if (typeof value !== "string" || !/^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/i.test(value)) throw new TypeError("segment UUID must use canonical hexadecimal form")
+  return Buffer.from(value.replaceAll("-", ""), "hex")
+}
+
+/**
+ * Computes Castagnoli CRC32C as an unsigned uint32.
+ * @param bytes
+ */
+export function crc32c(/** @type {Buffer | Uint8Array} */ bytes) {
+  let crc = 0xffff_ffff
+  for (const byte of bytes) crc = /** @type {number} */ (CRC32C_TABLE[(crc ^ byte) & 0xff]) ^ (crc >>> 8)
+  return (crc ^ 0xffff_ffff) >>> 0
+}
+
+/**
+ * @param {unknown} value - Fixed byte string. @param {number} length - Exact length. @param {string} name - Field.
+ * @param length
+ * @param name
+ */
+function fixedBytes(value, length, name) {
+  if (!(value instanceof Uint8Array) || value.byteLength !== length) throw new TypeError(`${name} must contain exactly ${length} bytes`)
+  return Buffer.from(value)
+}
+
+/**
+ * @param {unknown} value - Unsigned value. @param {string} name - Field.
+ * @param name
+ */
+function uint64(value, name) {
+  if (typeof value !== "bigint" || value < 0n || value > UINT64_MAX) throw new RangeError(`${name} must be an unsigned 64-bit bigint`)
+  return value
+}
+
+/**
+ * @param {unknown} value - Unsigned value. @param {string} name - Field.
+ * @param name
+ */
+function uint32(value, name) {
+  if (!Number.isSafeInteger(value) || /** @type {number} */ (value) < 0 || /** @type {number} */ (value) > 0xffff_ffff) throw new RangeError(`${name} must be an unsigned 32-bit integer`)
+  return /** @type {number} */ (value)
+}
+
+/**
+ *
+ */
+function createCrc32cTable() {
+  const table = new Uint32Array(256)
+  for (let index = 0; index < 256; index += 1) {
+    let value = index
+    for (let bit = 0; bit < 8; bit += 1) value = (value & 1) === 1 ? (value >>> 1) ^ 0x82f6_3b78 : value >>> 1
+    table[index] = value >>> 0
+  }
+  return table
+}