socketduct 0.1.15

package/src/spool/spool-file.js

@@ -0,0 +1,451 @@
+import {readFileSync, readdirSync, rmSync, statSync} from "node:fs"
+import {appendFile, mkdir, readFile, rename, rm, stat, writeFile} from "node:fs/promises"
+import {join, resolve} from "node:path"
+
+/**
+ * @typedef {object} SpoolMetadata
+ * @property {string} sessionId - sessionId value.
+ * @property {string} ownerId - ownerId value.
+ * @property {string} createdAt - createdAt value.
+ * @typedef {object} SpoolSession
+ * @property {string} directory - directory value.
+ * @property {SpoolMetadata} metadata - metadata value.
+ * @typedef {object} SpoolChunk
+ * @property {number} sequence - sequence value.
+ * @property {Buffer} data - data value.
+ * @typedef {object} AppendChunkInput
+ * @property {string} sessionId - sessionId value.
+ * @property {string} [ownerId] - ownerId value.
+ * @property {number} sequence - sequence value.
+ * @property {Uint8Array} data - data value.
+ * @typedef {object} CreateSpoolInput
+ * @property {string} sessionId - sessionId value.
+ * @property {string} ownerId - ownerId value.
+ * @typedef {object} SessionSpoolOptions
+ * @property {string} rootDir - rootDir value.
+ * @property {number} [maxBytes] - Maximum total spool bytes per session.
+ * @property {number} [maxAgeMs] - Maximum age of a spool session before operations are rejected.
+ * @property {() => number} [now] - Function returning current time in milliseconds (defaults to Date.now).
+ * @typedef {object} SerializedChunk
+ * @property {number} sequence - sequence value.
+ * @property {string} dataBase64 - dataBase64 value.
+ */
+
+const VALID_SESSION_ID = /^[a-zA-Z0-9_-]{8,}$/
+const MAX_RECOVERY_STATE_BYTES = 16 * 1024
+
+export class SpoolQuotaError extends Error {
+  /**
+   * @param {string} message - Error message.
+   * @param {string} code - Error code.
+   */
+  constructor(message, code) {
+    super(message)
+    this.name = "SpoolQuotaError"
+    this.code = code
+  }
+}
+
+export class SessionSpool {
+  /** @type {string} */
+  #rootDir
+
+  /** @type {number | undefined} */
+  #maxBytes
+
+  /** @type {number | undefined} */
+  #maxAgeMs
+
+  /** @type {() => number} */
+  #now
+
+  /** @type {Map<string, number>} */
+  #usedBytes = new Map()
+
+  /** @type {Set<string>} */
+  #usageLoaded = new Set()
+
+  /** @param {SessionSpoolOptions} options - Spool directory options. */
+  constructor(options) {
+    this.#rootDir = resolve(options.rootDir)
+    this.#maxBytes = options.maxBytes
+    this.#maxAgeMs = options.maxAgeMs
+    this.#now = options.now ?? (() => Date.now())
+    this.#cleanupExpiredSync()
+  }
+
+  /**
+   * @param {CreateSpoolInput} input - input value.
+   * @returns {Promise<SpoolSession>} - Result value.
+   */
+  async create(input) {
+    return await this.#create(input, false)
+  }
+
+  /**
+   * Creates a fresh spool without replacing durable state for the same ID.
+   * @param {CreateSpoolInput} input - Session identity.
+   * @returns {Promise<SpoolSession>} Created session.
+   */
+  async createNew(input) {
+    return await this.#create(input, true)
+  }
+
+  /**
+   * Persists bounded protocol-specific recovery metadata atomically.
+   * @param {string} sessionId - Stable session identifier.
+   * @param {Record<string, unknown>} state - JSON recovery state.
+   * @returns {Promise<void>} Completion.
+   */
+  async writeRecoveryState(sessionId, state) {
+    await this.#readSession(sessionId)
+    const encoded = `${JSON.stringify(state)}\n`
+    if (Buffer.byteLength(encoded) > MAX_RECOVERY_STATE_BYTES) throw new Error("Socketduct recovery state exceeds its size limit")
+    await writeFileAtomic(join(this.#sessionDirectory(sessionId), "recovery.json"), encoded, {mode: 0o600})
+  }
+
+  /**
+   * Reads bounded recovery metadata before a synchronous socket is allocated.
+   * @param {string} sessionId - Stable session identifier.
+   * @param {string} ownerId - Expected spool owner.
+   * @returns {Record<string, unknown>} Parsed recovery state.
+   */
+  readRecoveryStateSync(sessionId, ownerId) {
+    try {
+      const directory = this.#sessionDirectory(sessionId)
+      const metadataPath = join(directory, "metadata.json")
+      if (statSync(metadataPath).size > MAX_RECOVERY_STATE_BYTES) throw new Error("recovery ownership metadata is oversized")
+      const metadata = JSON.parse(readFileSync(metadataPath, "utf8"))
+      if (metadata?.sessionId !== sessionId || metadata?.ownerId !== ownerId) throw new Error("recovery ownership mismatch")
+      this.#assertMetadataAge(metadata, sessionId)
+      const path = join(directory, "recovery.json")
+      if (statSync(path).size > MAX_RECOVERY_STATE_BYTES) throw new Error("recovery metadata is oversized")
+      const state = JSON.parse(readFileSync(path, "utf8"))
+      if (typeof state !== "object" || state === null || Array.isArray(state)) throw new Error("recovery metadata is not an object")
+      return state
+    } catch (error) {
+      if (error instanceof SpoolQuotaError) throw error
+      throw new Error(`Invalid Socketduct recovery state for ${sessionId}`, {cause: error})
+    }
+  }
+
+  /**
+   * Removes only well-formed, expired session directories beneath the private spool root.
+   * @returns {void}
+   */
+  #cleanupExpiredSync() {
+    if (this.#maxAgeMs === undefined) return
+    let entries
+    try {
+      entries = readdirSync(this.#rootDir, {withFileTypes: true})
+    } catch (error) {
+      if (/** @type {{code?: unknown}} */ (error).code === "ENOENT") return
+      throw error
+    }
+    for (const entry of entries) {
+      if (!entry.isDirectory() || !VALID_SESSION_ID.test(entry.name)) continue
+      const directory = this.#sessionDirectory(entry.name)
+      const metadataPath = join(directory, "metadata.json")
+      let metadata
+      try {
+        if (statSync(metadataPath).size > MAX_RECOVERY_STATE_BYTES) continue
+        metadata = JSON.parse(readFileSync(metadataPath, "utf8"))
+      } catch {
+        continue
+      }
+      if (metadata?.sessionId !== entry.name || typeof metadata?.ownerId !== "string") continue
+      const createdAt = typeof metadata.createdAt === "string" ? Date.parse(metadata.createdAt) : Number.NaN
+      if (!Number.isFinite(createdAt)) continue
+      if (this.#now() - createdAt > this.#maxAgeMs) rmSync(directory, {recursive: true, force: true})
+    }
+  }
+
+  /**
+   * @param {unknown} metadata - Durable metadata.
+   * @param {string} sessionId - Stable ID.
+   * @returns {void}
+   */
+  #assertMetadataAge(metadata, sessionId) {
+    const createdAtValue = /** @type {{createdAt?: unknown}} */ (metadata).createdAt
+    const createdAt = typeof createdAtValue === "string" ? Date.parse(createdAtValue) : Number.NaN
+    if (!Number.isFinite(createdAt)) throw new Error("recovery ownership metadata has an invalid creation time")
+    const ageMs = this.#now() - createdAt
+    if (this.#maxAgeMs !== undefined && ageMs > this.#maxAgeMs) {
+      throw new SpoolQuotaError(
+        `Spool age ${ageMs}ms exceeds max ${this.#maxAgeMs}ms for session ${sessionId}`,
+        "SPOOL_AGE_EXCEEDED"
+      )
+    }
+  }
+
+  /**
+   * @param {CreateSpoolInput} input - Session identity.
+   * @param {boolean} exclusive - Refuse an existing spool directory.
+   * @returns {Promise<SpoolSession>} Created session.
+   */
+  async #create(input, exclusive) {
+    const directory = this.#sessionDirectory(input.sessionId)
+    const now = this.#now()
+    const createdAt = new Date(now).toISOString()
+    const metadata = {
+      sessionId: input.sessionId,
+      ownerId: input.ownerId,
+      createdAt
+    }
+
+    const ageMs = now - new Date(createdAt).getTime()
+    if (this.#maxAgeMs !== undefined && ageMs > this.#maxAgeMs) {
+      throw new SpoolQuotaError(
+        `Spool age ${ageMs}ms exceeds max ${this.#maxAgeMs}ms for session ${input.sessionId}`,
+        "SPOOL_AGE_EXCEEDED"
+      )
+    }
+
+    if (exclusive) {
+      await mkdir(this.#rootDir, {recursive: true, mode: 0o700})
+      try {
+        await mkdir(directory, {mode: 0o700})
+      } catch (error) {
+        if (/** @type {{code?: unknown}} */ (error).code === "EEXIST") throw new Error(`Spool session ${input.sessionId} already exists; explicit recovery is required`, {cause: error})
+        throw error
+      }
+    } else {
+      await mkdir(directory, {recursive: true, mode: 0o700})
+    }
+    try {
+      await writeFileAtomic(join(directory, "metadata.json"), `${JSON.stringify(metadata)}\n`, {mode: 0o600})
+      await writeFileAtomic(join(directory, "chunks.jsonl"), "", {mode: 0o600})
+      await writeFileAtomic(join(directory, "usage.json"), JSON.stringify({totalBytes: 0}) + "\n", {mode: 0o600})
+      await chmodDirectoryBestEffort(directory)
+    } catch (error) {
+      if (exclusive) await rm(directory, {recursive: true, force: true})
+      throw error
+    }
+
+    this.#usedBytes.set(input.sessionId, 0)
+    this.#usageLoaded.add(input.sessionId)
+
+    return {directory, metadata}
+  }
+
+  /**
+   * @param {AppendChunkInput} input - input value.
+   * @returns {Promise<void>} - Result value.
+   */
+  async appendChunk(input) {
+    const session = await this.#readSession(input.sessionId)
+
+    if (input.ownerId !== undefined && input.ownerId !== session.metadata.ownerId) {
+      throw new Error("Chunk owner does not match spool metadata")
+    }
+
+    const createdAt = new Date(session.metadata.createdAt).getTime()
+    const ageMs = this.#now() - createdAt
+    if (this.#maxAgeMs !== undefined && ageMs > this.#maxAgeMs) {
+      throw new SpoolQuotaError(
+        `Spool age ${ageMs}ms exceeds max ${this.#maxAgeMs}ms for session ${input.sessionId}`,
+        "SPOOL_AGE_EXCEEDED"
+      )
+    }
+
+    const chunkData = Buffer.from(input.data)
+    await this.#ensureUsageLoaded(input.sessionId)
+    const currentUsed = this.#usedBytes.get(input.sessionId) ?? 0
+    const newTotal = currentUsed + chunkData.length
+
+    if (this.#maxBytes !== undefined && newTotal > this.#maxBytes) {
+      throw new SpoolQuotaError(
+        `Spool byte quota exceeded for session ${input.sessionId}: ${newTotal} / max ${this.#maxBytes}`,
+        "SPOOL_QUOTA_EXCEEDED"
+      )
+    }
+
+    const directory = this.#sessionDirectory(input.sessionId)
+    const chunkPath = join(directory, "chunks.jsonl")
+    const line = JSON.stringify({
+      sequence: input.sequence,
+      dataBase64: chunkData.toString("base64")
+    })
+    await appendFile(chunkPath, `${line}\n`, {mode: 0o600})
+
+    const usagePath = join(directory, "usage.json")
+    await writeFileAtomic(usagePath, JSON.stringify({totalBytes: newTotal}) + "\n", {mode: 0o600})
+    this.#usedBytes.set(input.sessionId, newTotal)
+  }
+
+  /**
+   * @param {string} sessionId - sessionId value.
+   * @param {number} acknowledgedSequence - acknowledgedSequence value.
+   * @returns {Promise<SpoolChunk[]>} - Result value.
+   */
+  async readChunksAfter(sessionId, acknowledgedSequence) {
+    const chunks = await this.#readAllChunks(sessionId)
+    return chunks.filter((chunk) => chunk.sequence > acknowledgedSequence)
+  }
+
+  /**
+   * @param {string} sessionId - sessionId value.
+   * @param {number} acknowledgedSequence - acknowledgedSequence value.
+   * @returns {Promise<void>} - Result value.
+   */
+  async truncateAcked(sessionId, acknowledgedSequence) {
+    const chunks = await this.#readAllChunks(sessionId)
+    const remaining = chunks.filter((chunk) => chunk.sequence > acknowledgedSequence)
+    await this.#writeChunks(sessionId, remaining)
+    const newTotal = remaining.reduce((sum, chunk) => sum + chunk.data.length, 0)
+    const directory = this.#sessionDirectory(sessionId)
+    await writeFileAtomic(join(directory, "usage.json"), JSON.stringify({totalBytes: newTotal}) + "\n", {mode: 0o600})
+    this.#usedBytes.set(sessionId, newTotal)
+  }
+
+  /**
+   * @param {string} sessionId - sessionId value.
+   * @returns {Promise<void>} - Result value.
+   */
+  async remove(sessionId) {
+    if (!VALID_SESSION_ID.test(sessionId)) {
+      return
+    }
+    const directory = this.#sessionDirectory(sessionId)
+    // A stream teardown and its enclosing temporary-directory cleanup may race.
+    // Let fs.rm retry transient non-empty directory states instead of surfacing
+    // a cleanup-only ENOTEMPTY as a socket error.
+    await rm(directory, {recursive: true, force: true, maxRetries: 3, retryDelay: 10})
+    this.#usedBytes.delete(sessionId)
+  }
+
+  /**
+   * @param {string} sessionId - sessionId value.
+   * @returns {number | undefined} - Used bytes for the session, or undefined if not tracked.
+   */
+  getUsedBytes(sessionId) {
+    if (!this.#usedBytes.has(sessionId)) {
+      return undefined
+    }
+    return this.#usedBytes.get(sessionId)
+  }
+
+  /**
+   * @param {string} sessionId - sessionId value.
+   * @returns {Promise<void>} - Result value.
+   */
+  async #ensureUsageLoaded(sessionId) {
+    if (this.#usageLoaded.has(sessionId)) {
+      return
+    }
+    const directory = this.#sessionDirectory(sessionId)
+    const usagePath = join(directory, "usage.json")
+    let total = 0
+    try {
+      const raw = await readFile(usagePath, "utf8")
+      const data = JSON.parse(raw)
+      if (typeof data.totalBytes === "number") {
+        total = data.totalBytes
+      }
+    } catch {
+      total = 0
+    }
+    const durableTotal = (await this.#readAllChunks(sessionId)).reduce((sum, chunk) => sum + chunk.data.length, 0)
+    if (total !== durableTotal) {
+      total = durableTotal
+      await writeFileAtomic(usagePath, JSON.stringify({totalBytes: total}) + "\n", {mode: 0o600})
+    }
+    this.#usedBytes.set(sessionId, total)
+    this.#usageLoaded.add(sessionId)
+  }
+
+  /**
+   * @param {string} sessionId - sessionId value.
+   * @returns {Promise<SpoolSession>} - Result value.
+   */
+  async #readSession(sessionId) {
+    const directory = this.#sessionDirectory(sessionId)
+    const metadataPath = join(directory, "metadata.json")
+    const raw = await readFile(metadataPath, "utf8")
+    const metadata = /** @type {SpoolMetadata} */ (JSON.parse(raw))
+
+    if (metadata.sessionId !== sessionId) {
+      throw new Error("Session identifier does not match spool metadata")
+    }
+
+    return {directory, metadata}
+  }
+
+  /**
+   * @param {string} sessionId - sessionId value.
+   * @returns {Promise<SpoolChunk[]>} - Result value.
+   */
+  async #readAllChunks(sessionId) {
+    await this.#readSession(sessionId)
+    const chunkPath = join(this.#sessionDirectory(sessionId), "chunks.jsonl")
+    const raw = await readFile(chunkPath, "utf8")
+    if (raw.trim() === "") {
+      return []
+    }
+
+    return raw.trimEnd().split("\n").map((line) => {
+      const serialized = /** @type {SerializedChunk} */ (JSON.parse(line))
+      return {
+        sequence: serialized.sequence,
+        data: Buffer.from(serialized.dataBase64, "base64")
+      }
+    })
+  }
+
+  /**
+   * @param {string} sessionId - sessionId value.
+   * @param {readonly SpoolChunk[]} chunks - chunks value.
+   * @returns {Promise<void>} - Result value.
+   */
+  async #writeChunks(sessionId, chunks) {
+    const chunkPath = join(this.#sessionDirectory(sessionId), "chunks.jsonl")
+    const lines = chunks.map((chunk) => JSON.stringify(/** @type {SerializedChunk} */ ({
+      sequence: chunk.sequence,
+      dataBase64: chunk.data.toString("base64")
+    })))
+    await writeFileAtomic(chunkPath, lines.length === 0 ? "" : `${lines.join("\n")}\n`, {mode: 0o600})
+  }
+
+  /**
+   * @param {string} sessionId - sessionId value.
+   * @returns {string} - Result value.
+   */
+  #sessionDirectory(sessionId) {
+    if (!VALID_SESSION_ID.test(sessionId)) {
+      throw new Error(`Invalid session identifier: ${sessionId}`)
+    }
+
+    const directory = resolve(this.#rootDir, sessionId)
+    if (!directory.startsWith(`${this.#rootDir}/`) && directory !== this.#rootDir) {
+      throw new Error(`Invalid session identifier: ${sessionId}`)
+    }
+
+    return directory
+  }
+}
+
+/**
+ * @param {string} path - path value.
+ * @param {string} contents - contents value.
+ * @param {{mode: number}} options - File write options.
+ * @returns {Promise<void>} - Result value.
+ */
+async function writeFileAtomic(path, contents, options) {
+  const tempPath = `${path}.${process.pid}.${Date.now()}.tmp`
+  await writeFile(tempPath, contents, {mode: options.mode})
+  await rename(tempPath, path)
+}
+
+/**
+ * @param {string} directory - directory value.
+ * @returns {Promise<void>} - Result value.
+ */
+async function chmodDirectoryBestEffort(directory) {
+  const current = await stat(directory)
+  if ((current.mode & 0o777) === 0o700) {
+    return
+  }
+
+  const {chmod} = await import("node:fs/promises")
+  await chmod(directory, 0o700)
+}

package/tensorbuzz.yml

@@ -0,0 +1,36 @@
+before_install:
+  - curl -fsSL https://deb.nodesource.com/setup_24.x | sudo -E bash -
+  - sudo apt-get install -y docker.io nodejs
+before_script:
+  - node --version
+  - npm ci
+environment:
+  DOCKER_HOST: "tcp://docker_server:2375"
+services:
+  docker_server:
+    image: "docker:29.3.0-dind"
+    environment:
+      DOCKER_TLS_CERTDIR: ""
+    entrypoint:
+      - /bin/sh
+      - -lc
+    command: >-
+      rm -rf /var/run/docker/containerd /run/containerd /var/lib/docker/containerd/daemon &&
+      exec dockerd-entrypoint.sh
+      --host=tcp://0.0.0.0:2375
+      --tls=false
+    expose:
+      - 2375
+    healthcheck:
+      test: ["CMD", "docker", "-H", "tcp://127.0.0.1:2375", "info"]
+      interval: 2s
+      timeout: 2s
+      retries: 30
+      start_period: 5s
+    privileged: true
+    restart_policy: on-failure
+script:
+  - npm run typecheck
+  - npm run lint
+  - npm test
+  - docker build --tag socketduct-ci .