vector-framework 1.2.1 → 1.2.3

package/src/checkpoint/artifacts/materializer.ts

@@ -0,0 +1,211 @@
+import { existsSync, promises as fs } from 'node:fs';
+import { dirname, extname, join, relative } from 'node:path';
+import type { CheckpointAssetRecord, CheckpointManifest } from '../types';
+import { sha256Hex } from './hasher';
+import { computeAssetFingerprint, isAbsolutePathPortable, normalizeLogicalPath } from './manifest';
+import type { CheckpointArtifactMaterializerOptions } from './types';
+import { CheckpointWorkerDecompressor } from './worker-decompressor';
+
+const DEFAULT_MATERIALIZED_DIR = '_materialized';
+const DEFAULT_LOCK_TIMEOUT_MS = 15_000;
+const DEFAULT_LOCK_POLL_MS = 50;
+
+interface MaterializedMarker {
+  fingerprint: string;
+  createdAt: string;
+}
+
+export class CheckpointArtifactMaterializer {
+  private verifyChecksums: boolean;
+  private materializedDirName: string;
+  private lockTimeoutMs: number;
+
+  constructor(options: CheckpointArtifactMaterializerOptions = {}) {
+    this.verifyChecksums = options.verifyChecksums ?? true;
+    this.materializedDirName = options.materializedDirName ?? DEFAULT_MATERIALIZED_DIR;
+    this.lockTimeoutMs = options.lockTimeoutMs ?? DEFAULT_LOCK_TIMEOUT_MS;
+  }
+
+  async materialize(manifest: CheckpointManifest, storageDir: string): Promise<void> {
+    const versionDir = join(storageDir, manifest.version);
+    const markerPath = join(versionDir, '.assets.ready.json');
+    const lockPath = join(versionDir, '.assets.lock');
+    const fingerprint = computeAssetFingerprint(manifest.assets);
+
+    if (await this.isReady(markerPath, fingerprint, manifest.assets, join(versionDir, this.materializedDirName))) {
+      return;
+    }
+
+    await this.acquireLock(lockPath);
+    try {
+      if (await this.isReady(markerPath, fingerprint, manifest.assets, join(versionDir, this.materializedDirName))) {
+        return;
+      }
+
+      const materializedRoot = join(versionDir, this.materializedDirName);
+      await fs.rm(materializedRoot, { recursive: true, force: true });
+      await fs.mkdir(materializedRoot, { recursive: true });
+
+      const decompressor = new CheckpointWorkerDecompressor();
+      try {
+        for (const asset of manifest.assets) {
+          const result = await this.materializeAsset(asset, storageDir, versionDir, materializedRoot, decompressor);
+          asset.materializedPath = result;
+        }
+      } finally {
+        await decompressor.dispose();
+      }
+
+      const marker: MaterializedMarker = {
+        fingerprint,
+        createdAt: new Date().toISOString(),
+      };
+      await fs.writeFile(markerPath, JSON.stringify(marker, null, 2), 'utf-8');
+    } finally {
+      await fs.rm(lockPath, { recursive: true, force: true });
+    }
+  }
+
+  private async materializeAsset(
+    asset: CheckpointAssetRecord,
+    storageDir: string,
+    versionDir: string,
+    root: string,
+    decompressor: CheckpointWorkerDecompressor
+  ): Promise<string> {
+    const sourcePath = this.resolveSourcePath(asset, storageDir);
+    if (!existsSync(sourcePath)) {
+      throw new Error(`Checkpoint asset blob not found: ${sourcePath}`);
+    }
+
+    const blob = await fs.readFile(sourcePath);
+    const blobBytes = new Uint8Array(blob.buffer, blob.byteOffset, blob.byteLength);
+    const expectedBlobHash = asset.blobHash;
+    if (this.verifyChecksums && expectedBlobHash && sha256Hex(blobBytes) !== expectedBlobHash) {
+      throw new Error(`Checkpoint asset blob checksum mismatch for ${asset.logicalPath}`);
+    }
+
+    const codec = asset.codec ?? (asset.blobHash ? 'gzip' : 'none');
+    const contentBytes = await decompressor.decompress(blobBytes, codec);
+    const expectedContentHash = asset.contentHash ?? asset.hash;
+    if (this.verifyChecksums && expectedContentHash && sha256Hex(contentBytes) !== expectedContentHash) {
+      throw new Error(`Checkpoint asset content checksum mismatch for ${asset.logicalPath}`);
+    }
+
+    const cachedFile = await this.writeDecompressedCache(asset, storageDir, contentBytes);
+    const safeLogicalPath = normalizeLogicalPath(asset.logicalPath);
+    const destinationPath = join(root, safeLogicalPath);
+    await fs.mkdir(dirname(destinationPath), { recursive: true });
+    await fs.rm(destinationPath, { force: true });
+    await this.linkWithFallback(cachedFile, destinationPath);
+
+    return normalizePath(relative(versionDir, destinationPath));
+  }
+
+  private async writeDecompressedCache(
+    asset: CheckpointAssetRecord,
+    storageDir: string,
+    bytes: Uint8Array
+  ): Promise<string> {
+    const hash = asset.contentHash ?? asset.hash;
+    const extension = extname(asset.logicalPath) || '.bin';
+    const cacheFile = join(storageDir, '_assets/cache', `${hash}${extension}`);
+    await fs.mkdir(dirname(cacheFile), { recursive: true });
+    if (existsSync(cacheFile)) {
+      if (!this.verifyChecksums) {
+        return cacheFile;
+      }
+      const existing = await fs.readFile(cacheFile);
+      const existingBytes = new Uint8Array(existing.buffer, existing.byteOffset, existing.byteLength);
+      if (sha256Hex(existingBytes) === hash) {
+        return cacheFile;
+      }
+    }
+    await fs.writeFile(cacheFile, bytes);
+    return cacheFile;
+  }
+
+  private resolveSourcePath(asset: CheckpointAssetRecord, storageDir: string): string {
+    const rawPath = asset.blobPath ?? asset.storedPath;
+    if (isAbsolutePathPortable(rawPath)) {
+      return rawPath;
+    }
+    return join(storageDir, rawPath);
+  }
+
+  private async linkWithFallback(sourcePath: string, destinationPath: string): Promise<void> {
+    try {
+      await fs.link(sourcePath, destinationPath);
+      return;
+    } catch {
+      // Fall through to symlink/copy.
+    }
+
+    try {
+      await fs.symlink(sourcePath, destinationPath);
+      return;
+    } catch {
+      // Fall through to copy.
+    }
+
+    await fs.copyFile(sourcePath, destinationPath);
+  }
+
+  private async acquireLock(lockPath: string): Promise<void> {
+    const deadline = Date.now() + this.lockTimeoutMs;
+    while (Date.now() < deadline) {
+      try {
+        await fs.mkdir(lockPath);
+        return;
+      } catch (error) {
+        if (!isAlreadyExists(error)) {
+          throw error;
+        }
+      }
+      await sleep(DEFAULT_LOCK_POLL_MS);
+    }
+
+    throw new Error(`Timed out waiting for checkpoint asset lock: ${lockPath}`);
+  }
+
+  private async isReady(
+    markerPath: string,
+    fingerprint: string,
+    assets: CheckpointAssetRecord[],
+    materializedRoot: string
+  ): Promise<boolean> {
+    if (!existsSync(markerPath)) {
+      return false;
+    }
+
+    try {
+      const marker = JSON.parse(await fs.readFile(markerPath, 'utf-8')) as MaterializedMarker;
+      if (marker.fingerprint !== fingerprint) {
+        return false;
+      }
+      for (const asset of assets) {
+        const expectedPath = join(materializedRoot, normalizeLogicalPath(asset.logicalPath));
+        if (!existsSync(expectedPath)) {
+          return false;
+        }
+      }
+      return true;
+    } catch {
+      return false;
+    }
+  }
+}
+
+function normalizePath(path: string): string {
+  return path.replace(/\\/g, '/');
+}
+
+function isAlreadyExists(error: unknown): boolean {
+  return (
+    typeof error === 'object' && error !== null && 'code' in error && (error as { code?: string }).code === 'EEXIST'
+  );
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}

package/src/checkpoint/artifacts/packager.ts

@@ -0,0 +1,100 @@
+import { promises as fs } from 'node:fs';
+import { join, relative } from 'node:path';
+import type { CheckpointCompressionCodec } from '../types';
+import { sha256Hex } from './hasher';
+import type { CheckpointArtifactPackageRecord } from './types';
+
+const ARCHIVE_DIR = '_archives';
+
+export class CheckpointPackager {
+  private storageDir: string;
+  private codec: CheckpointCompressionCodec;
+
+  constructor(storageDir: string, codec: CheckpointCompressionCodec = 'gzip') {
+    this.storageDir = storageDir;
+    this.codec = codec;
+  }
+
+  async packageVersion(version: string): Promise<CheckpointArtifactPackageRecord> {
+    const versionDir = join(this.storageDir, version);
+    const archiveRelPath = join(ARCHIVE_DIR, `${version}${this.archiveSuffix()}`).replace(/\\/g, '/');
+    const archivePath = join(this.storageDir, archiveRelPath);
+    await fs.mkdir(join(this.storageDir, ARCHIVE_DIR), { recursive: true });
+
+    const files = await collectFiles(versionDir);
+    const archiveBytes = await this.buildArchiveBytes(versionDir, archivePath, files);
+
+    return {
+      archivePath: archiveRelPath,
+      archiveHash: sha256Hex(archiveBytes),
+      archiveSize: archiveBytes.byteLength,
+      codec: this.codec,
+    };
+  }
+
+  private async buildArchiveBytes(versionDir: string, archivePath: string, files: string[]): Promise<Uint8Array> {
+    const ArchiveCtor = (Bun as any).Archive;
+    if (typeof ArchiveCtor === 'function') {
+      const archiveEntries = Object.fromEntries(
+        files.map((filePath) => {
+          const rel = relative(versionDir, filePath).replace(/\\/g, '/');
+          return [rel, Bun.file(filePath)];
+        })
+      );
+
+      const archive = new ArchiveCtor(archiveEntries);
+      const tarBytes = new Uint8Array(await archive.bytes());
+      const archiveBytes = this.codec === 'gzip' ? Bun.gzipSync(tarBytes) : tarBytes;
+      await Bun.write(archivePath, archiveBytes);
+      return archiveBytes;
+    }
+
+    await this.buildArchiveWithTar(versionDir, archivePath, files);
+    const bytes = await fs.readFile(archivePath);
+    return new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+  }
+
+  private async buildArchiveWithTar(versionDir: string, archivePath: string, files: string[]): Promise<void> {
+    const relFiles = files.map((filePath) => relative(versionDir, filePath));
+    if (relFiles.length === 0) {
+      throw new Error(`Cannot package checkpoint: no files found in "${versionDir}"`);
+    }
+
+    const tarArgs = this.codec === 'gzip' ? ['-czf', archivePath] : ['-cf', archivePath];
+    const proc = Bun.spawn(['tar', ...tarArgs, '-C', versionDir, ...relFiles], {
+      stdout: 'pipe',
+      stderr: 'pipe',
+    });
+    const exitCode = await proc.exited;
+    if (exitCode === 0) {
+      return;
+    }
+
+    const stderr = await new Response(proc.stderr).text();
+    throw new Error(`Failed to package checkpoint archive with tar (exit ${exitCode}): ${stderr.trim()}`);
+  }
+
+  private archiveSuffix(): '.tar' | '.tar.gz' {
+    return this.codec === 'gzip' ? '.tar.gz' : '.tar';
+  }
+}
+
+async function collectFiles(root: string): Promise<string[]> {
+  const files: string[] = [];
+  await walk(root, files);
+  return files.filter((filePath) => relative(root, filePath).replace(/\\/g, '/') !== 'manifest.json');
+}
+
+async function walk(dir: string, files: string[]): Promise<void> {
+  const entries = await fs.readdir(dir, { withFileTypes: true });
+  for (const entry of entries) {
+    const fullPath = join(dir, entry.name);
+    if (entry.isDirectory()) {
+      await walk(fullPath, files);
+      continue;
+    }
+    if (entry.isFile()) {
+      files.push(fullPath);
+    }
+  }
+}

package/src/checkpoint/artifacts/repository.ts

@@ -0,0 +1,36 @@
+import { existsSync, promises as fs } from 'node:fs';
+import { dirname, join } from 'node:path';
+import type { CheckpointAssetRecord } from '../types';
+import type { CheckpointArtifactRepository } from './types';
+
+export class LocalCheckpointArtifactRepository implements CheckpointArtifactRepository {
+  private storageDir: string;
+
+  constructor(storageDir: string) {
+    this.storageDir = storageDir;
+  }
+
+  async writeBlob(record: CheckpointAssetRecord, bytes: Uint8Array): Promise<void> {
+    const targetPath = this.resolveBlobPath(record);
+    await fs.mkdir(dirname(targetPath), { recursive: true });
+    if (existsSync(targetPath)) {
+      return;
+    }
+    await fs.writeFile(targetPath, bytes);
+  }
+
+  async hasBlob(record: CheckpointAssetRecord): Promise<boolean> {
+    return existsSync(this.resolveBlobPath(record));
+  }
+
+  async readBlob(record: CheckpointAssetRecord): Promise<Uint8Array> {
+    const targetPath = this.resolveBlobPath(record);
+    const bytes = await fs.readFile(targetPath);
+    return new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+  }
+
+  resolveBlobPath(record: CheckpointAssetRecord): string {
+    const rawPath = record.blobPath ?? record.storedPath;
+    return rawPath.startsWith('/') ? rawPath : join(this.storageDir, rawPath);
+  }
+}

package/src/checkpoint/artifacts/store.ts

@@ -0,0 +1,102 @@
+import { existsSync, promises as fs } from 'node:fs';
+import { join } from 'node:path';
+import type { CheckpointAssetRecord, CheckpointCompressionCodec } from '../types';
+import { compressBytes, DEFAULT_ASSET_CODEC } from './compressor';
+import { sha256Hex } from './hasher';
+import { normalizeLogicalPath, normalizeRelativePath } from './manifest';
+import type { CheckpointArtifactStoreOptions } from './types';
+
+const BLOB_DIR = '_assets/blobs';
+
+export class CheckpointArtifactStore {
+  private storageDir: string;
+  private codec: CheckpointCompressionCodec;
+
+  constructor(storageDir: string, options: CheckpointArtifactStoreOptions = {}) {
+    this.storageDir = storageDir;
+    this.codec = options.assetCodec ?? DEFAULT_ASSET_CODEC;
+  }
+
+  async addEmbedded(logicalPath: string, sourcePath: string): Promise<CheckpointAssetRecord> {
+    return this.addAsset('embedded', logicalPath, sourcePath);
+  }
+
+  async addSidecar(logicalPath: string, sourcePath: string): Promise<CheckpointAssetRecord> {
+    return this.addAsset('sidecar', logicalPath, sourcePath);
+  }
+
+  async collect(embeddedPaths: string[], sidecarPaths: string[]): Promise<CheckpointAssetRecord[]> {
+    const records: CheckpointAssetRecord[] = [];
+
+    for (const sourcePath of embeddedPaths) {
+      records.push(await this.addEmbedded(sourcePath, sourcePath));
+    }
+
+    for (const sourcePath of sidecarPaths) {
+      records.push(await this.addSidecar(sourcePath, sourcePath));
+    }
+
+    return records;
+  }
+
+  private async addAsset(
+    type: CheckpointAssetRecord['type'],
+    logicalPath: string,
+    sourcePath: string
+  ): Promise<CheckpointAssetRecord> {
+    const content = await fs.readFile(sourcePath);
+    const contentBytes = new Uint8Array(content.buffer, content.byteOffset, content.byteLength);
+    const contentHash = sha256Hex(contentBytes);
+    const compressed = compressBytes(contentBytes, this.codec);
+    const blobHash = sha256Hex(compressed);
+    const blobPath = normalizeRelativePath(join(BLOB_DIR, `${blobHash}${this.codec === 'gzip' ? '.gz' : ''}`));
+    const storedPath = join(this.storageDir, blobPath);
+
+    await fs.mkdir(join(this.storageDir, BLOB_DIR), { recursive: true });
+    if (!existsSync(storedPath)) {
+      await this.writeAtomically(storedPath, compressed);
+    } else {
+      const existing = await fs.readFile(storedPath);
+      const existingBytes = new Uint8Array(existing.buffer, existing.byteOffset, existing.byteLength);
+      if (sha256Hex(existingBytes) !== blobHash) {
+        await this.writeAtomically(storedPath, compressed);
+      }
+    }
+
+    return {
+      type,
+      logicalPath: normalizeLogicalPath(logicalPath),
+      storedPath,
+      hash: contentHash,
+      size: content.byteLength,
+      contentHash,
+      contentSize: content.byteLength,
+      blobHash,
+      blobSize: compressed.byteLength,
+      blobPath,
+      codec: this.codec,
+    };
+  }
+
+  private async writeAtomically(path: string, bytes: Uint8Array): Promise<void> {
+    const tempPath = `${path}.tmp.${process.pid}.${Date.now()}`;
+    await fs.writeFile(tempPath, bytes);
+    try {
+      await fs.rename(tempPath, path);
+    } catch (error) {
+      if (!isAlreadyExists(error)) {
+        throw error;
+      }
+      await fs.rm(path, { force: true });
+      await fs.rename(tempPath, path);
+    }
+  }
+}
+
+function isAlreadyExists(error: unknown): boolean {
+  if (typeof error !== 'object' || error === null || !('code' in error)) {
+    return false;
+  }
+  const code = (error as { code?: string }).code;
+  return code === 'EEXIST' || code === 'EPERM';
+}

package/src/checkpoint/artifacts/types.ts

@@ -0,0 +1,24 @@
+import type { CheckpointAssetRecord, CheckpointCompressionCodec } from '../types';
+
+export interface CheckpointArtifactStoreOptions {
+  assetCodec?: CheckpointCompressionCodec;
+}
+
+export interface CheckpointArtifactMaterializerOptions {
+  verifyChecksums?: boolean;
+  materializedDirName?: string;
+  lockTimeoutMs?: number;
+}
+
+export interface CheckpointArtifactPackageRecord {
+  archivePath: string;
+  archiveHash: string;
+  archiveSize: number;
+  codec: CheckpointCompressionCodec;
+}
+
+export interface CheckpointArtifactRepository {
+  writeBlob(record: CheckpointAssetRecord, bytes: Uint8Array): Promise<void>;
+  hasBlob(record: CheckpointAssetRecord): Promise<boolean>;
+  readBlob(record: CheckpointAssetRecord): Promise<Uint8Array>;
+}

package/src/checkpoint/artifacts/worker-decompressor.ts

@@ -0,0 +1,192 @@
+import { availableParallelism, cpus } from 'node:os';
+import type { CheckpointCompressionCodec } from '../types';
+
+interface DecompressRequest {
+  id: number;
+  codec: CheckpointCompressionCodec;
+  input: ArrayBuffer;
+}
+
+interface DecompressResponse {
+  id: number;
+  output?: ArrayBuffer;
+  error?: string;
+}
+
+interface DecompressJob {
+  id: number;
+  request: DecompressRequest;
+  resolve: (output: Uint8Array) => void;
+  reject: (error: Error) => void;
+}
+
+const DEFAULT_MAX_WORKERS = 4;
+
+export class CheckpointWorkerDecompressor {
+  private workers: Worker[] = [];
+  private idleWorkers: Worker[] = [];
+  private queue: DecompressJob[] = [];
+  private activeJobsByWorker: Map<Worker, DecompressJob> = new Map();
+  private nextJobId = 1;
+  private disposed = false;
+
+  constructor(workerCount: number = resolveDefaultWorkerCount()) {
+    const normalizedCount = normalizeWorkerCount(workerCount);
+    const workerUrl = resolveWorkerModuleUrl();
+
+    for (let i = 0; i < normalizedCount; i++) {
+      const worker = new Worker(workerUrl.href);
+      worker.onmessage = (event) => this.handleWorkerMessage(worker, event);
+      worker.onerror = (event) => this.handleWorkerError(worker, event);
+      this.workers.push(worker);
+      this.idleWorkers.push(worker);
+    }
+  }
+
+  async decompress(input: Uint8Array, codec: CheckpointCompressionCodec): Promise<Uint8Array> {
+    if (codec === 'none') {
+      return new Uint8Array(input);
+    }
+
+    if (this.disposed) {
+      throw new Error('Checkpoint worker decompressor is disposed');
+    }
+
+    const copied = new Uint8Array(input);
+
+    return await new Promise<Uint8Array>((resolve, reject) => {
+      const id = this.nextJobId++;
+      this.queue.push({
+        id,
+        request: {
+          id,
+          codec,
+          input: copied.buffer,
+        },
+        resolve,
+        reject,
+      });
+      this.pump();
+    });
+  }
+
+  async dispose(): Promise<void> {
+    if (this.disposed) {
+      return;
+    }
+    this.disposed = true;
+
+    const error = new Error('Checkpoint worker decompressor disposed');
+    this.failAll(error);
+
+    for (const worker of this.workers) {
+      try {
+        worker.terminate();
+      } catch {
+        // Ignore termination failures.
+      }
+    }
+
+    this.workers = [];
+    this.idleWorkers = [];
+    this.activeJobsByWorker.clear();
+  }
+
+  private pump(): void {
+    while (this.idleWorkers.length > 0 && this.queue.length > 0) {
+      const worker = this.idleWorkers.pop()!;
+      const job = this.queue.shift()!;
+      this.activeJobsByWorker.set(worker, job);
+      worker.postMessage(job.request, [job.request.input]);
+    }
+  }
+
+  private handleWorkerMessage(worker: Worker, event: MessageEvent<DecompressResponse>): void {
+    const job = this.activeJobsByWorker.get(worker);
+    this.activeJobsByWorker.delete(worker);
+
+    if (!this.disposed) {
+      this.idleWorkers.push(worker);
+    }
+
+    if (!job) {
+      this.pump();
+      return;
+    }
+
+    const message = event.data;
+    if (message.error) {
+      job.reject(new Error(message.error));
+    } else if (message.output instanceof ArrayBuffer) {
+      job.resolve(new Uint8Array(message.output));
+    } else {
+      job.reject(new Error('Worker returned no output'));
+    }
+
+    this.pump();
+  }
+
+  private handleWorkerError(worker: Worker, event: ErrorEvent): void {
+    const job = this.activeJobsByWorker.get(worker);
+    this.activeJobsByWorker.delete(worker);
+    this.idleWorkers = this.idleWorkers.filter((candidate) => candidate !== worker);
+
+    const message = event.message?.trim() || 'Checkpoint decompression worker crashed';
+    const error = new Error(message);
+
+    if (job) {
+      job.reject(error);
+    }
+
+    this.failAll(error);
+    this.dispose().catch(() => {
+      // Ignore cleanup failures after worker error.
+    });
+  }
+
+  private failAll(error: Error): void {
+    const queued = this.queue.splice(0, this.queue.length);
+    for (const job of queued) {
+      job.reject(error);
+    }
+
+    for (const job of this.activeJobsByWorker.values()) {
+      job.reject(error);
+    }
+    this.activeJobsByWorker.clear();
+  }
+}
+
+function resolveDefaultWorkerCount(): number {
+  const cores = resolveCoreCount();
+  const reserveForMainThread = Math.max(1, cores - 1);
+  return Math.max(1, Math.min(DEFAULT_MAX_WORKERS, reserveForMainThread));
+}
+
+function resolveCoreCount(): number {
+  try {
+    const parallelism = availableParallelism();
+    if (Number.isFinite(parallelism) && parallelism > 0) {
+      return parallelism;
+    }
+  } catch {
+    // Fall through to cpus().
+  }
+
+  const cpuCount = cpus().length;
+  return Number.isFinite(cpuCount) && cpuCount > 0 ? cpuCount : 1;
+}
+
+function normalizeWorkerCount(value: number): number {
+  if (!Number.isFinite(value) || value <= 0) {
+    return 1;
+  }
+  return Math.max(1, Math.floor(value));
+}
+
+function resolveWorkerModuleUrl(): URL {
+  if (import.meta.url.endsWith('.ts')) {
+    return new URL('./decompress-worker.ts', import.meta.url);
+  }
+  return new URL('./decompress-worker.js', import.meta.url);
+}