vector-framework 1.2.2 → 1.2.3

package/src/checkpoint/artifacts/packager.ts

@@ -0,0 +1,100 @@
+import { promises as fs } from 'node:fs';
+import { join, relative } from 'node:path';
+import type { CheckpointCompressionCodec } from '../types';
+import { sha256Hex } from './hasher';
+import type { CheckpointArtifactPackageRecord } from './types';
+
+const ARCHIVE_DIR = '_archives';
+
+export class CheckpointPackager {
+  private storageDir: string;
+  private codec: CheckpointCompressionCodec;
+
+  constructor(storageDir: string, codec: CheckpointCompressionCodec = 'gzip') {
+    this.storageDir = storageDir;
+    this.codec = codec;
+  }
+
+  async packageVersion(version: string): Promise<CheckpointArtifactPackageRecord> {
+    const versionDir = join(this.storageDir, version);
+    const archiveRelPath = join(ARCHIVE_DIR, `${version}${this.archiveSuffix()}`).replace(/\\/g, '/');
+    const archivePath = join(this.storageDir, archiveRelPath);
+    await fs.mkdir(join(this.storageDir, ARCHIVE_DIR), { recursive: true });
+
+    const files = await collectFiles(versionDir);
+    const archiveBytes = await this.buildArchiveBytes(versionDir, archivePath, files);
+
+    return {
+      archivePath: archiveRelPath,
+      archiveHash: sha256Hex(archiveBytes),
+      archiveSize: archiveBytes.byteLength,
+      codec: this.codec,
+    };
+  }
+
+  private async buildArchiveBytes(versionDir: string, archivePath: string, files: string[]): Promise<Uint8Array> {
+    const ArchiveCtor = (Bun as any).Archive;
+    if (typeof ArchiveCtor === 'function') {
+      const archiveEntries = Object.fromEntries(
+        files.map((filePath) => {
+          const rel = relative(versionDir, filePath).replace(/\\/g, '/');
+          return [rel, Bun.file(filePath)];
+        })
+      );
+
+      const archive = new ArchiveCtor(archiveEntries);
+      const tarBytes = new Uint8Array(await archive.bytes());
+      const archiveBytes = this.codec === 'gzip' ? Bun.gzipSync(tarBytes) : tarBytes;
+      await Bun.write(archivePath, archiveBytes);
+      return archiveBytes;
+    }
+
+    await this.buildArchiveWithTar(versionDir, archivePath, files);
+    const bytes = await fs.readFile(archivePath);
+    return new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+  }
+
+  private async buildArchiveWithTar(versionDir: string, archivePath: string, files: string[]): Promise<void> {
+    const relFiles = files.map((filePath) => relative(versionDir, filePath));
+    if (relFiles.length === 0) {
+      throw new Error(`Cannot package checkpoint: no files found in "${versionDir}"`);
+    }
+
+    const tarArgs = this.codec === 'gzip' ? ['-czf', archivePath] : ['-cf', archivePath];
+    const proc = Bun.spawn(['tar', ...tarArgs, '-C', versionDir, ...relFiles], {
+      stdout: 'pipe',
+      stderr: 'pipe',
+    });
+    const exitCode = await proc.exited;
+    if (exitCode === 0) {
+      return;
+    }
+
+    const stderr = await new Response(proc.stderr).text();
+    throw new Error(`Failed to package checkpoint archive with tar (exit ${exitCode}): ${stderr.trim()}`);
+  }
+
+  private archiveSuffix(): '.tar' | '.tar.gz' {
+    return this.codec === 'gzip' ? '.tar.gz' : '.tar';
+  }
+}
+
+async function collectFiles(root: string): Promise<string[]> {
+  const files: string[] = [];
+  await walk(root, files);
+  return files.filter((filePath) => relative(root, filePath).replace(/\\/g, '/') !== 'manifest.json');
+}
+
+async function walk(dir: string, files: string[]): Promise<void> {
+  const entries = await fs.readdir(dir, { withFileTypes: true });
+  for (const entry of entries) {
+    const fullPath = join(dir, entry.name);
+    if (entry.isDirectory()) {
+      await walk(fullPath, files);
+      continue;
+    }
+    if (entry.isFile()) {
+      files.push(fullPath);
+    }
+  }
+}

package/src/checkpoint/artifacts/repository.ts

@@ -0,0 +1,36 @@
+import { existsSync, promises as fs } from 'node:fs';
+import { dirname, join } from 'node:path';
+import type { CheckpointAssetRecord } from '../types';
+import type { CheckpointArtifactRepository } from './types';
+
+export class LocalCheckpointArtifactRepository implements CheckpointArtifactRepository {
+  private storageDir: string;
+
+  constructor(storageDir: string) {
+    this.storageDir = storageDir;
+  }
+
+  async writeBlob(record: CheckpointAssetRecord, bytes: Uint8Array): Promise<void> {
+    const targetPath = this.resolveBlobPath(record);
+    await fs.mkdir(dirname(targetPath), { recursive: true });
+    if (existsSync(targetPath)) {
+      return;
+    }
+    await fs.writeFile(targetPath, bytes);
+  }
+
+  async hasBlob(record: CheckpointAssetRecord): Promise<boolean> {
+    return existsSync(this.resolveBlobPath(record));
+  }
+
+  async readBlob(record: CheckpointAssetRecord): Promise<Uint8Array> {
+    const targetPath = this.resolveBlobPath(record);
+    const bytes = await fs.readFile(targetPath);
+    return new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+  }
+
+  resolveBlobPath(record: CheckpointAssetRecord): string {
+    const rawPath = record.blobPath ?? record.storedPath;
+    return rawPath.startsWith('/') ? rawPath : join(this.storageDir, rawPath);
+  }
+}

package/src/checkpoint/artifacts/store.ts

@@ -0,0 +1,102 @@
+import { existsSync, promises as fs } from 'node:fs';
+import { join } from 'node:path';
+import type { CheckpointAssetRecord, CheckpointCompressionCodec } from '../types';
+import { compressBytes, DEFAULT_ASSET_CODEC } from './compressor';
+import { sha256Hex } from './hasher';
+import { normalizeLogicalPath, normalizeRelativePath } from './manifest';
+import type { CheckpointArtifactStoreOptions } from './types';
+
+const BLOB_DIR = '_assets/blobs';
+
+export class CheckpointArtifactStore {
+  private storageDir: string;
+  private codec: CheckpointCompressionCodec;
+
+  constructor(storageDir: string, options: CheckpointArtifactStoreOptions = {}) {
+    this.storageDir = storageDir;
+    this.codec = options.assetCodec ?? DEFAULT_ASSET_CODEC;
+  }
+
+  async addEmbedded(logicalPath: string, sourcePath: string): Promise<CheckpointAssetRecord> {
+    return this.addAsset('embedded', logicalPath, sourcePath);
+  }
+
+  async addSidecar(logicalPath: string, sourcePath: string): Promise<CheckpointAssetRecord> {
+    return this.addAsset('sidecar', logicalPath, sourcePath);
+  }
+
+  async collect(embeddedPaths: string[], sidecarPaths: string[]): Promise<CheckpointAssetRecord[]> {
+    const records: CheckpointAssetRecord[] = [];
+
+    for (const sourcePath of embeddedPaths) {
+      records.push(await this.addEmbedded(sourcePath, sourcePath));
+    }
+
+    for (const sourcePath of sidecarPaths) {
+      records.push(await this.addSidecar(sourcePath, sourcePath));
+    }
+
+    return records;
+  }
+
+  private async addAsset(
+    type: CheckpointAssetRecord['type'],
+    logicalPath: string,
+    sourcePath: string
+  ): Promise<CheckpointAssetRecord> {
+    const content = await fs.readFile(sourcePath);
+    const contentBytes = new Uint8Array(content.buffer, content.byteOffset, content.byteLength);
+    const contentHash = sha256Hex(contentBytes);
+    const compressed = compressBytes(contentBytes, this.codec);
+    const blobHash = sha256Hex(compressed);
+    const blobPath = normalizeRelativePath(join(BLOB_DIR, `${blobHash}${this.codec === 'gzip' ? '.gz' : ''}`));
+    const storedPath = join(this.storageDir, blobPath);
+
+    await fs.mkdir(join(this.storageDir, BLOB_DIR), { recursive: true });
+    if (!existsSync(storedPath)) {
+      await this.writeAtomically(storedPath, compressed);
+    } else {
+      const existing = await fs.readFile(storedPath);
+      const existingBytes = new Uint8Array(existing.buffer, existing.byteOffset, existing.byteLength);
+      if (sha256Hex(existingBytes) !== blobHash) {
+        await this.writeAtomically(storedPath, compressed);
+      }
+    }
+
+    return {
+      type,
+      logicalPath: normalizeLogicalPath(logicalPath),
+      storedPath,
+      hash: contentHash,
+      size: content.byteLength,
+      contentHash,
+      contentSize: content.byteLength,
+      blobHash,
+      blobSize: compressed.byteLength,
+      blobPath,
+      codec: this.codec,
+    };
+  }
+
+  private async writeAtomically(path: string, bytes: Uint8Array): Promise<void> {
+    const tempPath = `${path}.tmp.${process.pid}.${Date.now()}`;
+    await fs.writeFile(tempPath, bytes);
+    try {
+      await fs.rename(tempPath, path);
+    } catch (error) {
+      if (!isAlreadyExists(error)) {
+        throw error;
+      }
+      await fs.rm(path, { force: true });
+      await fs.rename(tempPath, path);
+    }
+  }
+}
+
+function isAlreadyExists(error: unknown): boolean {
+  if (typeof error !== 'object' || error === null || !('code' in error)) {
+    return false;
+  }
+  const code = (error as { code?: string }).code;
+  return code === 'EEXIST' || code === 'EPERM';
+}

package/src/checkpoint/artifacts/types.ts

@@ -0,0 +1,24 @@
+import type { CheckpointAssetRecord, CheckpointCompressionCodec } from '../types';
+
+export interface CheckpointArtifactStoreOptions {
+  assetCodec?: CheckpointCompressionCodec;
+}
+
+export interface CheckpointArtifactMaterializerOptions {
+  verifyChecksums?: boolean;
+  materializedDirName?: string;
+  lockTimeoutMs?: number;
+}
+
+export interface CheckpointArtifactPackageRecord {
+  archivePath: string;
+  archiveHash: string;
+  archiveSize: number;
+  codec: CheckpointCompressionCodec;
+}
+
+export interface CheckpointArtifactRepository {
+  writeBlob(record: CheckpointAssetRecord, bytes: Uint8Array): Promise<void>;
+  hasBlob(record: CheckpointAssetRecord): Promise<boolean>;
+  readBlob(record: CheckpointAssetRecord): Promise<Uint8Array>;
+}

package/src/checkpoint/artifacts/worker-decompressor.ts

@@ -0,0 +1,192 @@
+import { availableParallelism, cpus } from 'node:os';
+import type { CheckpointCompressionCodec } from '../types';
+
+interface DecompressRequest {
+  id: number;
+  codec: CheckpointCompressionCodec;
+  input: ArrayBuffer;
+}
+
+interface DecompressResponse {
+  id: number;
+  output?: ArrayBuffer;
+  error?: string;
+}
+
+interface DecompressJob {
+  id: number;
+  request: DecompressRequest;
+  resolve: (output: Uint8Array) => void;
+  reject: (error: Error) => void;
+}
+
+const DEFAULT_MAX_WORKERS = 4;
+
+export class CheckpointWorkerDecompressor {
+  private workers: Worker[] = [];
+  private idleWorkers: Worker[] = [];
+  private queue: DecompressJob[] = [];
+  private activeJobsByWorker: Map<Worker, DecompressJob> = new Map();
+  private nextJobId = 1;
+  private disposed = false;
+
+  constructor(workerCount: number = resolveDefaultWorkerCount()) {
+    const normalizedCount = normalizeWorkerCount(workerCount);
+    const workerUrl = resolveWorkerModuleUrl();
+
+    for (let i = 0; i < normalizedCount; i++) {
+      const worker = new Worker(workerUrl.href);
+      worker.onmessage = (event) => this.handleWorkerMessage(worker, event);
+      worker.onerror = (event) => this.handleWorkerError(worker, event);
+      this.workers.push(worker);
+      this.idleWorkers.push(worker);
+    }
+  }
+
+  async decompress(input: Uint8Array, codec: CheckpointCompressionCodec): Promise<Uint8Array> {
+    if (codec === 'none') {
+      return new Uint8Array(input);
+    }
+
+    if (this.disposed) {
+      throw new Error('Checkpoint worker decompressor is disposed');
+    }
+
+    const copied = new Uint8Array(input);
+
+    return await new Promise<Uint8Array>((resolve, reject) => {
+      const id = this.nextJobId++;
+      this.queue.push({
+        id,
+        request: {
+          id,
+          codec,
+          input: copied.buffer,
+        },
+        resolve,
+        reject,
+      });
+      this.pump();
+    });
+  }
+
+  async dispose(): Promise<void> {
+    if (this.disposed) {
+      return;
+    }
+    this.disposed = true;
+
+    const error = new Error('Checkpoint worker decompressor disposed');
+    this.failAll(error);
+
+    for (const worker of this.workers) {
+      try {
+        worker.terminate();
+      } catch {
+        // Ignore termination failures.
+      }
+    }
+
+    this.workers = [];
+    this.idleWorkers = [];
+    this.activeJobsByWorker.clear();
+  }
+
+  private pump(): void {
+    while (this.idleWorkers.length > 0 && this.queue.length > 0) {
+      const worker = this.idleWorkers.pop()!;
+      const job = this.queue.shift()!;
+      this.activeJobsByWorker.set(worker, job);
+      worker.postMessage(job.request, [job.request.input]);
+    }
+  }
+
+  private handleWorkerMessage(worker: Worker, event: MessageEvent<DecompressResponse>): void {
+    const job = this.activeJobsByWorker.get(worker);
+    this.activeJobsByWorker.delete(worker);
+
+    if (!this.disposed) {
+      this.idleWorkers.push(worker);
+    }
+
+    if (!job) {
+      this.pump();
+      return;
+    }
+
+    const message = event.data;
+    if (message.error) {
+      job.reject(new Error(message.error));
+    } else if (message.output instanceof ArrayBuffer) {
+      job.resolve(new Uint8Array(message.output));
+    } else {
+      job.reject(new Error('Worker returned no output'));
+    }
+
+    this.pump();
+  }
+
+  private handleWorkerError(worker: Worker, event: ErrorEvent): void {
+    const job = this.activeJobsByWorker.get(worker);
+    this.activeJobsByWorker.delete(worker);
+    this.idleWorkers = this.idleWorkers.filter((candidate) => candidate !== worker);
+
+    const message = event.message?.trim() || 'Checkpoint decompression worker crashed';
+    const error = new Error(message);
+
+    if (job) {
+      job.reject(error);
+    }
+
+    this.failAll(error);
+    this.dispose().catch(() => {
+      // Ignore cleanup failures after worker error.
+    });
+  }
+
+  private failAll(error: Error): void {
+    const queued = this.queue.splice(0, this.queue.length);
+    for (const job of queued) {
+      job.reject(error);
+    }
+
+    for (const job of this.activeJobsByWorker.values()) {
+      job.reject(error);
+    }
+    this.activeJobsByWorker.clear();
+  }
+}
+
+function resolveDefaultWorkerCount(): number {
+  const cores = resolveCoreCount();
+  const reserveForMainThread = Math.max(1, cores - 1);
+  return Math.max(1, Math.min(DEFAULT_MAX_WORKERS, reserveForMainThread));
+}
+
+function resolveCoreCount(): number {
+  try {
+    const parallelism = availableParallelism();
+    if (Number.isFinite(parallelism) && parallelism > 0) {
+      return parallelism;
+    }
+  } catch {
+    // Fall through to cpus().
+  }
+
+  const cpuCount = cpus().length;
+  return Number.isFinite(cpuCount) && cpuCount > 0 ? cpuCount : 1;
+}
+
+function normalizeWorkerCount(value: number): number {
+  if (!Number.isFinite(value) || value <= 0) {
+    return 1;
+  }
+  return Math.max(1, Math.floor(value));
+}
+
+function resolveWorkerModuleUrl(): URL {
+  if (import.meta.url.endsWith('.ts')) {
+    return new URL('./decompress-worker.ts', import.meta.url);
+  }
+  return new URL('./decompress-worker.js', import.meta.url);
+}

package/src/checkpoint/asset-store.ts

@@ -0,0 +1,61 @@
+import { promises as fs } from 'node:fs';
+import type { CheckpointAssetRecord } from './types';
+import { CheckpointArtifactStore } from './artifacts/store';
+
+const EMBEDDED_PER_FILE_BUDGET = 64 * 1024; // 64 KB per file
+const EMBEDDED_TOTAL_BUDGET = 512 * 1024; // 512 KB total
+
+export class AssetStore {
+  private artifactStore: CheckpointArtifactStore;
+
+  constructor(storageDir: string) {
+    this.artifactStore = new CheckpointArtifactStore(storageDir);
+  }
+
+  async addEmbedded(logicalPath: string, sourcePath: string): Promise<CheckpointAssetRecord> {
+    const content = await fs.readFile(sourcePath);
+    if (content.byteLength > EMBEDDED_PER_FILE_BUDGET) {
+      throw new Error(
+        `Embedded asset "${logicalPath}" is ${formatBytes(content.byteLength)} — exceeds ${formatBytes(EMBEDDED_PER_FILE_BUDGET)} per-file budget. Use sidecar instead.`
+      );
+    }
+
+    return await this.artifactStore.addEmbedded(logicalPath, sourcePath);
+  }
+
+  async addSidecar(logicalPath: string, sourcePath: string): Promise<CheckpointAssetRecord> {
+    return await this.artifactStore.addSidecar(logicalPath, sourcePath);
+  }
+
+  async collect(embeddedPaths: string[], sidecarPaths: string[]): Promise<CheckpointAssetRecord[]> {
+    const records: CheckpointAssetRecord[] = [];
+
+    for (const p of embeddedPaths) {
+      records.push(await this.addEmbedded(p, p));
+    }
+
+    for (const p of sidecarPaths) {
+      records.push(await this.addSidecar(p, p));
+    }
+
+    return records;
+  }
+
+  validateBudgets(records: CheckpointAssetRecord[]): void {
+    const embeddedTotal = records
+      .filter((r) => r.type === 'embedded')
+      .reduce((acc, r) => acc + (r.contentSize ?? r.size), 0);
+
+    if (embeddedTotal > EMBEDDED_TOTAL_BUDGET) {
+      throw new Error(
+        `Total embedded asset size ${formatBytes(embeddedTotal)} exceeds ${formatBytes(EMBEDDED_TOTAL_BUDGET)} budget.`
+      );
+    }
+  }
+}
+
+function formatBytes(bytes: number): string {
+  if (bytes < 1024) return `${bytes} B`;
+  if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`;
+  return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
+}

package/src/checkpoint/bundler.ts

@@ -0,0 +1,64 @@
+import { existsSync } from 'node:fs';
+import { join } from 'node:path';
+
+export interface BundleOptions {
+  entrypointPath: string;
+  outputDir: string;
+  outputFile?: string;
+}
+
+export interface BundleResult {
+  outputPath: string;
+  hash: string;
+  size: number;
+}
+
+export class CheckpointBundler {
+  async bundle(options: BundleOptions): Promise<BundleResult> {
+    const outfile = options.outputFile ?? 'checkpoint.js';
+
+    const result = await Bun.build({
+      entrypoints: [options.entrypointPath],
+      outdir: options.outputDir,
+      target: 'bun',
+      format: 'esm',
+      minify: true,
+      naming: { entry: outfile },
+    });
+
+    if (!result.success) {
+      const messages = result.logs.map((l) => l.message ?? String(l)).join('\n');
+      throw new Error(`Checkpoint bundle failed:\n${messages}`);
+    }
+
+    const outputPath = join(options.outputDir, outfile);
+
+    if (!existsSync(outputPath)) {
+      // Bun.build may output with a different name; find the actual output
+      const actualOutput = result.outputs.find((o) => o.kind === 'entry-point');
+      if (actualOutput) {
+        const actualPath = actualOutput.path;
+        if (existsSync(actualPath)) {
+          return this.hashFile(actualPath);
+        }
+      }
+      throw new Error(`Bundle output not found at expected path: ${outputPath}`);
+    }
+
+    return this.hashFile(outputPath);
+  }
+
+  private async hashFile(path: string): Promise<BundleResult> {
+    const file = Bun.file(path);
+    const content = await file.arrayBuffer();
+    const hashBuffer = Bun.SHA256.hash(new Uint8Array(content));
+    const hashBytes = new Uint8Array(hashBuffer.buffer, hashBuffer.byteOffset, hashBuffer.byteLength);
+    const hash = Buffer.from(hashBytes).toString('hex');
+
+    return {
+      outputPath: path,
+      hash,
+      size: content.byteLength,
+    };
+  }
+}