waypoi 0.0.0

package/src/storage/files.ts

@@ -0,0 +1,178 @@
+import { promises as fs } from "fs";
+import path from "path";
+import os from "os";
+import crypto from "crypto";
+import YAML from "yaml";
+import { EndpointDoc, EndpointHealth, ProviderModelHealth, RequestLog } from "../types";
+
+export interface ConfigFile {
+  endpoints: Array<Omit<EndpointDoc, "health">>;
+  // Auth configuration
+  authEnabled?: boolean;
+}
+
+export interface HealthFile {
+  endpoints: Record<string, EndpointHealth>;
+}
+
+export interface ProviderHealthFile {
+  models: Record<string, ProviderModelHealth>;
+}
+
+export interface StoragePaths {
+  baseDir: string;
+  configPath: string;
+  healthPath: string;
+  providerHealthPath: string;
+  requestLogPath: string;
+  providersPath: string;
+  poolsPath: string;
+  poolStatePath: string;
+}
+
+export function resolveStoragePaths(): StoragePaths {
+  const baseDir = process.env.WAYPOI_DIR ?? path.join(os.homedir(), ".config", "waypoi");
+  const configPath = process.env.WAYPOI_CONFIG ?? path.join(baseDir, "config.yaml");
+  return {
+    baseDir,
+    configPath,
+    healthPath: path.join(baseDir, "health.json"),
+    providerHealthPath: path.join(baseDir, "providers_health.json"),
+    requestLogPath: path.join(baseDir, "request_logs.jsonl"),
+    providersPath: path.join(baseDir, "providers.json"),
+    poolsPath: path.join(baseDir, "pools.json"),
+    poolStatePath: path.join(baseDir, "pool_state.json"),
+  };
+}
+
+export async function ensureStorageDir(paths: StoragePaths): Promise<void> {
+  await fs.mkdir(paths.baseDir, { recursive: true });
+}
+
+export async function loadConfig(paths: StoragePaths): Promise<ConfigFile> {
+  await ensureStorageDir(paths);
+  try {
+    const raw = await fs.readFile(paths.configPath, "utf8");
+    const doc = YAML.parse(raw) as ConfigFile | null;
+    if (doc?.endpoints) {
+      return doc;
+    }
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code !== "ENOENT") {
+      throw error;
+    }
+  }
+  return { endpoints: [] };
+}
+
+export async function saveConfig(paths: StoragePaths, config: ConfigFile): Promise<void> {
+  await ensureStorageDir(paths);
+  const yaml = YAML.stringify(config);
+  await writeAtomic(paths.configPath, yaml);
+}
+
+export async function loadHealth(paths: StoragePaths): Promise<HealthFile> {
+  await ensureStorageDir(paths);
+  try {
+    const raw = await fs.readFile(paths.healthPath, "utf8");
+    const data = JSON.parse(raw) as HealthFile | null;
+    if (data?.endpoints) {
+      return data;
+    }
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code !== "ENOENT") {
+      throw error;
+    }
+  }
+  return { endpoints: {} };
+}
+
+export async function saveHealth(paths: StoragePaths, health: HealthFile): Promise<void> {
+  await ensureStorageDir(paths);
+  await writeAtomic(paths.healthPath, JSON.stringify(health, null, 2));
+}
+
+export async function loadProviderHealth(paths: StoragePaths): Promise<ProviderHealthFile> {
+  await ensureStorageDir(paths);
+  try {
+    const raw = await fs.readFile(paths.providerHealthPath, "utf8");
+    const data = JSON.parse(raw) as ProviderHealthFile | null;
+    if (data?.models) {
+      return data;
+    }
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code !== "ENOENT") {
+      throw error;
+    }
+  }
+  return { models: {} };
+}
+
+export async function saveProviderHealth(paths: StoragePaths, health: ProviderHealthFile): Promise<void> {
+  await ensureStorageDir(paths);
+  await writeAtomic(paths.providerHealthPath, JSON.stringify(health, null, 2));
+}
+
+export async function readJsonFile<T>(
+  filePath: string,
+  fallback: T
+): Promise<T> {
+  try {
+    const raw = await fs.readFile(filePath, "utf8");
+    return JSON.parse(raw) as T;
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === "ENOENT") {
+      return fallback;
+    }
+    throw error;
+  }
+}
+
+export async function writeJsonFile(filePath: string, value: unknown): Promise<void> {
+  await ensureDir(path.dirname(filePath));
+  await writeAtomic(filePath, JSON.stringify(value, null, 2));
+}
+
+export function newEndpointId(): string {
+  return crypto.randomUUID();
+}
+
+export function defaultHealth(): EndpointHealth {
+  return {
+    status: "up",
+    consecutiveFailures: 0
+  };
+}
+
+export async function appendRequestLog(paths: StoragePaths, log: RequestLog): Promise<void> {
+  await ensureStorageDir(paths);
+  const line = `${JSON.stringify(log)}\n`;
+  await fs.appendFile(paths.requestLogPath, line, "utf8");
+}
+
+export async function readRequestLogs(paths: StoragePaths): Promise<RequestLog[]> {
+  await ensureStorageDir(paths);
+  try {
+    const raw = await fs.readFile(paths.requestLogPath, "utf8");
+    return raw
+      .split("\n")
+      .filter((line) => line.trim().length > 0)
+      .map((line) => JSON.parse(line) as RequestLog);
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === "ENOENT") {
+      return [];
+    }
+    throw error;
+  }
+}
+
+async function writeAtomic(filePath: string, content: string): Promise<void> {
+  const dir = path.dirname(filePath);
+  const tmp = path.join(dir, `.${path.basename(filePath)}.${crypto.randomUUID()}`);
+  await fs.writeFile(tmp, content, "utf8");
+  await fs.rename(tmp, filePath);
+}
+
+async function ensureDir(dir: string): Promise<void> {
+  await fs.mkdir(dir, { recursive: true });
+}

package/src/storage/imageCache.ts

@@ -0,0 +1,405 @@
+import { promises as fs } from "fs";
+import path from "path";
+import crypto from "crypto";
+import { StoragePaths, ensureStorageDir } from "./files";
+
+/**
+ * Media Cache
+ *
+ * Stores generated and uploaded media locally with LRU eviction.
+ * Backward-compatible with previous image-only APIs.
+ */
+
+export interface MediaCacheEntry {
+  hash: string;
+  filename: string;
+  size: number;
+  mimeType: string;
+  createdAt: Date;
+  model?: string;
+}
+
+interface CacheIndex {
+  entries: MediaCacheEntry[];
+  totalSize: number;
+  evictionBlockedCount?: number;
+}
+
+interface MediaRefIndex {
+  refs: Record<string, string[]>;
+}
+
+const DEFAULT_MAX_SIZE_BYTES = 1024 * 1024 * 1024; // 1GB
+
+export function resolveMediaDir(paths: StoragePaths): string {
+  return path.join(paths.baseDir, "media");
+}
+
+function cacheIndexPath(paths: StoragePaths): string {
+  return path.join(resolveMediaDir(paths), "index.json");
+}
+
+function refsIndexPath(paths: StoragePaths): string {
+  return path.join(resolveMediaDir(paths), "media_refs.json");
+}
+
+async function ensureMediaDir(paths: StoragePaths): Promise<void> {
+  await ensureStorageDir(paths);
+  await fs.mkdir(resolveMediaDir(paths), { recursive: true });
+}
+
+export async function ensureMediaCacheReady(paths: StoragePaths): Promise<void> {
+  await ensureMediaDir(paths);
+  const index = await loadCacheIndex(paths);
+  await saveCacheIndex(paths, index);
+  const refs = await loadRefsIndex(paths);
+  await saveRefsIndex(paths, refs);
+}
+
+async function loadCacheIndex(paths: StoragePaths): Promise<CacheIndex> {
+  const indexPath = cacheIndexPath(paths);
+  try {
+    const raw = await fs.readFile(indexPath, "utf8");
+      const data = JSON.parse(raw) as CacheIndex;
+      data.entries = data.entries.map((entry) => ({
+        ...entry,
+        createdAt: new Date(entry.createdAt),
+      }));
+      data.evictionBlockedCount = typeof data.evictionBlockedCount === "number" ? data.evictionBlockedCount : 0;
+      return data;
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === "ENOENT") {
+      return { entries: [], totalSize: 0, evictionBlockedCount: 0 };
+    }
+    throw error;
+  }
+}
+
+async function saveCacheIndex(paths: StoragePaths, index: CacheIndex): Promise<void> {
+  await fs.writeFile(cacheIndexPath(paths), JSON.stringify(index, null, 2), "utf8");
+}
+
+async function loadRefsIndex(paths: StoragePaths): Promise<MediaRefIndex> {
+  const refsPath = refsIndexPath(paths);
+  try {
+    const raw = await fs.readFile(refsPath, "utf8");
+    const parsed = JSON.parse(raw) as MediaRefIndex;
+    const refs = parsed?.refs && typeof parsed.refs === "object" ? parsed.refs : {};
+    const cleaned: Record<string, string[]> = {};
+    for (const [hash, sessionIds] of Object.entries(refs)) {
+      if (!Array.isArray(sessionIds)) continue;
+      const deduped = Array.from(new Set(sessionIds.filter((id) => typeof id === "string" && id.length > 0)));
+      if (deduped.length > 0) {
+        cleaned[hash] = deduped;
+      }
+    }
+    return { refs: cleaned };
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === "ENOENT") {
+      return { refs: {} };
+    }
+    throw error;
+  }
+}
+
+async function saveRefsIndex(paths: StoragePaths, index: MediaRefIndex): Promise<void> {
+  await fs.writeFile(refsIndexPath(paths), JSON.stringify(index, null, 2), "utf8");
+}
+
+export async function storeMedia(
+  paths: StoragePaths,
+  data: Buffer | string,
+  options?: { model?: string; maxSizeBytes?: number; mimeType?: string }
+): Promise<{ filePath: string; hash: string; mimeType: string; evicted: string[] }> {
+  await ensureMediaDir(paths);
+
+  const normalized = normalizeMediaInput(data, options?.mimeType);
+  const buffer = normalized.buffer;
+  const mimeType = normalized.mimeType;
+  const extension = extensionFromMime(mimeType, buffer);
+
+  const hash = crypto.createHash("sha256").update(buffer).digest("hex").slice(0, 16);
+  const filename = `${hash}.${extension}`;
+  const filePath = path.join(resolveMediaDir(paths), filename);
+
+  const index = await loadCacheIndex(paths);
+  const refs = await loadRefsIndex(paths);
+  const existing = index.entries.find((entry) => entry.hash === hash);
+  if (existing) {
+    index.entries = index.entries.filter((entry) => entry.hash !== hash);
+    existing.createdAt = new Date();
+    index.entries.push(existing);
+    await saveCacheIndex(paths, index);
+    return { filePath, hash, mimeType: existing.mimeType, evicted: [] };
+  }
+
+  await fs.writeFile(filePath, buffer);
+
+  const entry: MediaCacheEntry = {
+    hash,
+    filename,
+    size: buffer.length,
+    mimeType,
+    createdAt: new Date(),
+    model: options?.model,
+  };
+  index.entries.push(entry);
+  index.totalSize += buffer.length;
+
+  const maxSize = options?.maxSizeBytes ?? DEFAULT_MAX_SIZE_BYTES;
+  const evicted: string[] = [];
+  while (index.totalSize > maxSize && index.entries.length > 1) {
+    const evictionTargetIndex = index.entries.findIndex((entry) => !isHashReferenced(refs, entry.hash));
+    if (evictionTargetIndex < 0) {
+      index.evictionBlockedCount = (index.evictionBlockedCount ?? 0) + 1;
+      break;
+    }
+
+    const [oldest] = index.entries.splice(evictionTargetIndex, 1);
+    if (!oldest) {
+      break;
+    }
+    try {
+      await fs.unlink(path.join(resolveMediaDir(paths), oldest.filename));
+      evicted.push(oldest.hash);
+    } catch {
+      // ignore missing file
+    }
+    index.totalSize -= oldest.size;
+  }
+
+  await saveCacheIndex(paths, index);
+
+  return { filePath, hash, mimeType, evicted };
+}
+
+export async function getMediaPath(paths: StoragePaths, hash: string): Promise<string | null> {
+  const index = await loadCacheIndex(paths);
+  const entry = index.entries.find((item) => item.hash === hash);
+  if (!entry) {
+    return null;
+  }
+  return path.join(resolveMediaDir(paths), entry.filename);
+}
+
+export async function getMediaEntry(paths: StoragePaths, hash: string): Promise<MediaCacheEntry | null> {
+  const index = await loadCacheIndex(paths);
+  return index.entries.find((item) => item.hash === hash) ?? null;
+}
+
+export async function getCacheStats(paths: StoragePaths): Promise<{
+  count: number;
+  totalSizeBytes: number;
+  referencedCount: number;
+  unreferencedCount: number;
+  evictionBlockedCount: number;
+  oldestEntry?: Date;
+  newestEntry?: Date;
+}> {
+  const index = await loadCacheIndex(paths);
+  const refs = await loadRefsIndex(paths);
+  if (index.entries.length === 0) {
+    return {
+      count: 0,
+      totalSizeBytes: 0,
+      referencedCount: 0,
+      unreferencedCount: 0,
+      evictionBlockedCount: index.evictionBlockedCount ?? 0,
+    };
+  }
+
+  const sorted = [...index.entries].sort((a, b) => a.createdAt.getTime() - b.createdAt.getTime());
+  const referencedCount = index.entries.filter((entry) => isHashReferenced(refs, entry.hash)).length;
+  return {
+    count: index.entries.length,
+    totalSizeBytes: index.totalSize,
+    referencedCount,
+    unreferencedCount: index.entries.length - referencedCount,
+    evictionBlockedCount: index.evictionBlockedCount ?? 0,
+    oldestEntry: sorted[0].createdAt,
+    newestEntry: sorted[sorted.length - 1].createdAt,
+  };
+}
+
+export async function clearCache(paths: StoragePaths): Promise<number> {
+  const index = await loadCacheIndex(paths);
+  let deleted = 0;
+
+  for (const entry of index.entries) {
+    try {
+      await fs.unlink(path.join(resolveMediaDir(paths), entry.filename));
+      deleted += 1;
+    } catch {
+      // ignore missing file
+    }
+  }
+
+  await saveCacheIndex(paths, { entries: [], totalSize: 0, evictionBlockedCount: 0 });
+  await saveRefsIndex(paths, { refs: {} });
+  return deleted;
+}
+
+export async function syncSessionMediaReferences(
+  paths: StoragePaths,
+  sessionId: string,
+  hashes: string[]
+): Promise<void> {
+  await ensureMediaDir(paths);
+  const refs = await loadRefsIndex(paths);
+  const nextHashes = Array.from(new Set(hashes.filter((hash) => /^[a-f0-9]{16}$/i.test(hash))));
+
+  // Remove previous references for this session from all hashes.
+  for (const [hash, sessionIds] of Object.entries(refs.refs)) {
+    const filtered = sessionIds.filter((id) => id !== sessionId);
+    if (filtered.length > 0) {
+      refs.refs[hash] = filtered;
+    } else {
+      delete refs.refs[hash];
+    }
+  }
+
+  for (const hash of nextHashes) {
+    const existing = refs.refs[hash] ?? [];
+    refs.refs[hash] = Array.from(new Set([...existing, sessionId]));
+  }
+
+  await saveRefsIndex(paths, refs);
+}
+
+export async function unmarkSessionMediaReferences(
+  paths: StoragePaths,
+  sessionId: string
+): Promise<void> {
+  await ensureMediaDir(paths);
+  const refs = await loadRefsIndex(paths);
+  for (const [hash, sessionIds] of Object.entries(refs.refs)) {
+    const filtered = sessionIds.filter((id) => id !== sessionId);
+    if (filtered.length > 0) {
+      refs.refs[hash] = filtered;
+    } else {
+      delete refs.refs[hash];
+    }
+  }
+  await saveRefsIndex(paths, refs);
+}
+
+export async function getMediaRefCount(paths: StoragePaths, hash: string): Promise<number> {
+  const refs = await loadRefsIndex(paths);
+  return refs.refs[hash]?.length ?? 0;
+}
+
+/**
+ * Remove all media entries that are no longer referenced by any session.
+ * Call this after deleting a session to free up disk space.
+ * Returns the number of files deleted.
+ */
+export async function cleanOrphanedMedia(paths: StoragePaths): Promise<number> {
+  const index = await loadCacheIndex(paths);
+  const refs = await loadRefsIndex(paths);
+  let deleted = 0;
+  const surviving: typeof index.entries = [];
+  let survivingSize = 0;
+
+  for (const entry of index.entries) {
+    if (isHashReferenced(refs, entry.hash)) {
+      surviving.push(entry);
+      survivingSize += entry.size;
+    } else {
+      try {
+        await fs.unlink(path.join(resolveMediaDir(paths), entry.filename));
+        deleted += 1;
+      } catch {
+        // File already gone — still remove from index
+        deleted += 1;
+      }
+    }
+  }
+
+  if (deleted > 0) {
+    await saveCacheIndex(paths, { ...index, entries: surviving, totalSize: survivingSize });
+  }
+  return deleted;
+}
+
+function isHashReferenced(refs: MediaRefIndex, hash: string): boolean {
+  const sessionIds = refs.refs[hash];
+  return Array.isArray(sessionIds) && sessionIds.length > 0;
+}
+
+function normalizeMediaInput(
+  data: Buffer | string,
+  hintedMime?: string
+): { buffer: Buffer; mimeType: string } {
+  if (Buffer.isBuffer(data)) {
+    const mimeType = hintedMime ?? detectMimeType(data);
+    return { buffer: data, mimeType };
+  }
+
+  const trimmed = data.trim();
+  const dataUrlMatch = trimmed.match(/^data:([^;]+);base64,(.+)$/i);
+  if (dataUrlMatch) {
+    const mimeType = dataUrlMatch[1].toLowerCase();
+    const base64Payload = dataUrlMatch[2].replace(/\s+/g, "");
+    return { buffer: Buffer.from(base64Payload, "base64"), mimeType };
+  }
+
+  const buffer = Buffer.from(trimmed.replace(/\s+/g, ""), "base64");
+  const mimeType = hintedMime ?? detectMimeType(buffer);
+  return { buffer, mimeType };
+}
+
+function extensionFromMime(mimeType: string, buffer: Buffer): string {
+  const map: Record<string, string> = {
+    "image/png": "png",
+    "image/jpeg": "jpg",
+    "image/jpg": "jpg",
+    "image/gif": "gif",
+    "image/webp": "webp",
+    "audio/wav": "wav",
+    "audio/x-wav": "wav",
+    "audio/mpeg": "mp3",
+    "audio/mp3": "mp3",
+    "audio/ogg": "ogg",
+    "audio/webm": "webm",
+    "audio/mp4": "m4a",
+  };
+  if (map[mimeType]) {
+    return map[mimeType];
+  }
+
+  // fallback by magic bytes
+  if (buffer[0] === 0x89 && buffer[1] === 0x50 && buffer[2] === 0x4e && buffer[3] === 0x47) return "png";
+  if (buffer[0] === 0xff && buffer[1] === 0xd8 && buffer[2] === 0xff) return "jpg";
+  if (buffer[0] === 0x47 && buffer[1] === 0x49 && buffer[2] === 0x46) return "gif";
+  if (buffer[0] === 0x52 && buffer[1] === 0x49 && buffer[2] === 0x46 && buffer[3] === 0x46) {
+    if (buffer.length > 11) {
+      const chunk = buffer.subarray(8, 12).toString("ascii");
+      if (chunk === "WAVE") return "wav";
+      if (chunk === "WEBP") return "webp";
+    }
+  }
+  return "bin";
+}
+
+function detectMimeType(buffer: Buffer): string {
+  if (buffer[0] === 0x89 && buffer[1] === 0x50 && buffer[2] === 0x4e && buffer[3] === 0x47) {
+    return "image/png";
+  }
+  if (buffer[0] === 0xff && buffer[1] === 0xd8 && buffer[2] === 0xff) {
+    return "image/jpeg";
+  }
+  if (buffer[0] === 0x47 && buffer[1] === 0x49 && buffer[2] === 0x46) {
+    return "image/gif";
+  }
+  if (buffer[0] === 0x52 && buffer[1] === 0x49 && buffer[2] === 0x46 && buffer[3] === 0x46) {
+    if (buffer.length > 11) {
+      const chunk = buffer.subarray(8, 12).toString("ascii");
+      if (chunk === "WAVE") return "audio/wav";
+      if (chunk === "WEBP") return "image/webp";
+    }
+  }
+  if (buffer.length > 3 && buffer[0] === 0x49 && buffer[1] === 0x44 && buffer[2] === 0x33) {
+    return "audio/mpeg";
+  }
+  return "application/octet-stream";
+}