waypoi 0.0.0

package/dist/src/storage/files.js

@@ -0,0 +1,157 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.resolveStoragePaths = resolveStoragePaths;
+exports.ensureStorageDir = ensureStorageDir;
+exports.loadConfig = loadConfig;
+exports.saveConfig = saveConfig;
+exports.loadHealth = loadHealth;
+exports.saveHealth = saveHealth;
+exports.loadProviderHealth = loadProviderHealth;
+exports.saveProviderHealth = saveProviderHealth;
+exports.readJsonFile = readJsonFile;
+exports.writeJsonFile = writeJsonFile;
+exports.newEndpointId = newEndpointId;
+exports.defaultHealth = defaultHealth;
+exports.appendRequestLog = appendRequestLog;
+exports.readRequestLogs = readRequestLogs;
+const fs_1 = require("fs");
+const path_1 = __importDefault(require("path"));
+const os_1 = __importDefault(require("os"));
+const crypto_1 = __importDefault(require("crypto"));
+const yaml_1 = __importDefault(require("yaml"));
+function resolveStoragePaths() {
+    const baseDir = process.env.WAYPOI_DIR ?? path_1.default.join(os_1.default.homedir(), ".config", "waypoi");
+    const configPath = process.env.WAYPOI_CONFIG ?? path_1.default.join(baseDir, "config.yaml");
+    return {
+        baseDir,
+        configPath,
+        healthPath: path_1.default.join(baseDir, "health.json"),
+        providerHealthPath: path_1.default.join(baseDir, "providers_health.json"),
+        requestLogPath: path_1.default.join(baseDir, "request_logs.jsonl"),
+        providersPath: path_1.default.join(baseDir, "providers.json"),
+        poolsPath: path_1.default.join(baseDir, "pools.json"),
+        poolStatePath: path_1.default.join(baseDir, "pool_state.json"),
+    };
+}
+async function ensureStorageDir(paths) {
+    await fs_1.promises.mkdir(paths.baseDir, { recursive: true });
+}
+async function loadConfig(paths) {
+    await ensureStorageDir(paths);
+    try {
+        const raw = await fs_1.promises.readFile(paths.configPath, "utf8");
+        const doc = yaml_1.default.parse(raw);
+        if (doc?.endpoints) {
+            return doc;
+        }
+    }
+    catch (error) {
+        if (error.code !== "ENOENT") {
+            throw error;
+        }
+    }
+    return { endpoints: [] };
+}
+async function saveConfig(paths, config) {
+    await ensureStorageDir(paths);
+    const yaml = yaml_1.default.stringify(config);
+    await writeAtomic(paths.configPath, yaml);
+}
+async function loadHealth(paths) {
+    await ensureStorageDir(paths);
+    try {
+        const raw = await fs_1.promises.readFile(paths.healthPath, "utf8");
+        const data = JSON.parse(raw);
+        if (data?.endpoints) {
+            return data;
+        }
+    }
+    catch (error) {
+        if (error.code !== "ENOENT") {
+            throw error;
+        }
+    }
+    return { endpoints: {} };
+}
+async function saveHealth(paths, health) {
+    await ensureStorageDir(paths);
+    await writeAtomic(paths.healthPath, JSON.stringify(health, null, 2));
+}
+async function loadProviderHealth(paths) {
+    await ensureStorageDir(paths);
+    try {
+        const raw = await fs_1.promises.readFile(paths.providerHealthPath, "utf8");
+        const data = JSON.parse(raw);
+        if (data?.models) {
+            return data;
+        }
+    }
+    catch (error) {
+        if (error.code !== "ENOENT") {
+            throw error;
+        }
+    }
+    return { models: {} };
+}
+async function saveProviderHealth(paths, health) {
+    await ensureStorageDir(paths);
+    await writeAtomic(paths.providerHealthPath, JSON.stringify(health, null, 2));
+}
+async function readJsonFile(filePath, fallback) {
+    try {
+        const raw = await fs_1.promises.readFile(filePath, "utf8");
+        return JSON.parse(raw);
+    }
+    catch (error) {
+        if (error.code === "ENOENT") {
+            return fallback;
+        }
+        throw error;
+    }
+}
+async function writeJsonFile(filePath, value) {
+    await ensureDir(path_1.default.dirname(filePath));
+    await writeAtomic(filePath, JSON.stringify(value, null, 2));
+}
+function newEndpointId() {
+    return crypto_1.default.randomUUID();
+}
+function defaultHealth() {
+    return {
+        status: "up",
+        consecutiveFailures: 0
+    };
+}
+async function appendRequestLog(paths, log) {
+    await ensureStorageDir(paths);
+    const line = `${JSON.stringify(log)}\n`;
+    await fs_1.promises.appendFile(paths.requestLogPath, line, "utf8");
+}
+async function readRequestLogs(paths) {
+    await ensureStorageDir(paths);
+    try {
+        const raw = await fs_1.promises.readFile(paths.requestLogPath, "utf8");
+        return raw
+            .split("\n")
+            .filter((line) => line.trim().length > 0)
+            .map((line) => JSON.parse(line));
+    }
+    catch (error) {
+        if (error.code === "ENOENT") {
+            return [];
+        }
+        throw error;
+    }
+}
+async function writeAtomic(filePath, content) {
+    const dir = path_1.default.dirname(filePath);
+    const tmp = path_1.default.join(dir, `.${path_1.default.basename(filePath)}.${crypto_1.default.randomUUID()}`);
+    await fs_1.promises.writeFile(tmp, content, "utf8");
+    await fs_1.promises.rename(tmp, filePath);
+}
+async function ensureDir(dir) {
+    await fs_1.promises.mkdir(dir, { recursive: true });
+}

package/dist/src/storage/imageCache.js

@@ -0,0 +1,346 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.resolveMediaDir = resolveMediaDir;
+exports.ensureMediaCacheReady = ensureMediaCacheReady;
+exports.storeMedia = storeMedia;
+exports.getMediaPath = getMediaPath;
+exports.getMediaEntry = getMediaEntry;
+exports.getCacheStats = getCacheStats;
+exports.clearCache = clearCache;
+exports.syncSessionMediaReferences = syncSessionMediaReferences;
+exports.unmarkSessionMediaReferences = unmarkSessionMediaReferences;
+exports.getMediaRefCount = getMediaRefCount;
+exports.cleanOrphanedMedia = cleanOrphanedMedia;
+const fs_1 = require("fs");
+const path_1 = __importDefault(require("path"));
+const crypto_1 = __importDefault(require("crypto"));
+const files_1 = require("./files");
+const DEFAULT_MAX_SIZE_BYTES = 1024 * 1024 * 1024; // 1GB
+function resolveMediaDir(paths) {
+    return path_1.default.join(paths.baseDir, "media");
+}
+function cacheIndexPath(paths) {
+    return path_1.default.join(resolveMediaDir(paths), "index.json");
+}
+function refsIndexPath(paths) {
+    return path_1.default.join(resolveMediaDir(paths), "media_refs.json");
+}
+async function ensureMediaDir(paths) {
+    await (0, files_1.ensureStorageDir)(paths);
+    await fs_1.promises.mkdir(resolveMediaDir(paths), { recursive: true });
+}
+async function ensureMediaCacheReady(paths) {
+    await ensureMediaDir(paths);
+    const index = await loadCacheIndex(paths);
+    await saveCacheIndex(paths, index);
+    const refs = await loadRefsIndex(paths);
+    await saveRefsIndex(paths, refs);
+}
+async function loadCacheIndex(paths) {
+    const indexPath = cacheIndexPath(paths);
+    try {
+        const raw = await fs_1.promises.readFile(indexPath, "utf8");
+        const data = JSON.parse(raw);
+        data.entries = data.entries.map((entry) => ({
+            ...entry,
+            createdAt: new Date(entry.createdAt),
+        }));
+        data.evictionBlockedCount = typeof data.evictionBlockedCount === "number" ? data.evictionBlockedCount : 0;
+        return data;
+    }
+    catch (error) {
+        if (error.code === "ENOENT") {
+            return { entries: [], totalSize: 0, evictionBlockedCount: 0 };
+        }
+        throw error;
+    }
+}
+async function saveCacheIndex(paths, index) {
+    await fs_1.promises.writeFile(cacheIndexPath(paths), JSON.stringify(index, null, 2), "utf8");
+}
+async function loadRefsIndex(paths) {
+    const refsPath = refsIndexPath(paths);
+    try {
+        const raw = await fs_1.promises.readFile(refsPath, "utf8");
+        const parsed = JSON.parse(raw);
+        const refs = parsed?.refs && typeof parsed.refs === "object" ? parsed.refs : {};
+        const cleaned = {};
+        for (const [hash, sessionIds] of Object.entries(refs)) {
+            if (!Array.isArray(sessionIds))
+                continue;
+            const deduped = Array.from(new Set(sessionIds.filter((id) => typeof id === "string" && id.length > 0)));
+            if (deduped.length > 0) {
+                cleaned[hash] = deduped;
+            }
+        }
+        return { refs: cleaned };
+    }
+    catch (error) {
+        if (error.code === "ENOENT") {
+            return { refs: {} };
+        }
+        throw error;
+    }
+}
+async function saveRefsIndex(paths, index) {
+    await fs_1.promises.writeFile(refsIndexPath(paths), JSON.stringify(index, null, 2), "utf8");
+}
+async function storeMedia(paths, data, options) {
+    await ensureMediaDir(paths);
+    const normalized = normalizeMediaInput(data, options?.mimeType);
+    const buffer = normalized.buffer;
+    const mimeType = normalized.mimeType;
+    const extension = extensionFromMime(mimeType, buffer);
+    const hash = crypto_1.default.createHash("sha256").update(buffer).digest("hex").slice(0, 16);
+    const filename = `${hash}.${extension}`;
+    const filePath = path_1.default.join(resolveMediaDir(paths), filename);
+    const index = await loadCacheIndex(paths);
+    const refs = await loadRefsIndex(paths);
+    const existing = index.entries.find((entry) => entry.hash === hash);
+    if (existing) {
+        index.entries = index.entries.filter((entry) => entry.hash !== hash);
+        existing.createdAt = new Date();
+        index.entries.push(existing);
+        await saveCacheIndex(paths, index);
+        return { filePath, hash, mimeType: existing.mimeType, evicted: [] };
+    }
+    await fs_1.promises.writeFile(filePath, buffer);
+    const entry = {
+        hash,
+        filename,
+        size: buffer.length,
+        mimeType,
+        createdAt: new Date(),
+        model: options?.model,
+    };
+    index.entries.push(entry);
+    index.totalSize += buffer.length;
+    const maxSize = options?.maxSizeBytes ?? DEFAULT_MAX_SIZE_BYTES;
+    const evicted = [];
+    while (index.totalSize > maxSize && index.entries.length > 1) {
+        const evictionTargetIndex = index.entries.findIndex((entry) => !isHashReferenced(refs, entry.hash));
+        if (evictionTargetIndex < 0) {
+            index.evictionBlockedCount = (index.evictionBlockedCount ?? 0) + 1;
+            break;
+        }
+        const [oldest] = index.entries.splice(evictionTargetIndex, 1);
+        if (!oldest) {
+            break;
+        }
+        try {
+            await fs_1.promises.unlink(path_1.default.join(resolveMediaDir(paths), oldest.filename));
+            evicted.push(oldest.hash);
+        }
+        catch {
+            // ignore missing file
+        }
+        index.totalSize -= oldest.size;
+    }
+    await saveCacheIndex(paths, index);
+    return { filePath, hash, mimeType, evicted };
+}
+async function getMediaPath(paths, hash) {
+    const index = await loadCacheIndex(paths);
+    const entry = index.entries.find((item) => item.hash === hash);
+    if (!entry) {
+        return null;
+    }
+    return path_1.default.join(resolveMediaDir(paths), entry.filename);
+}
+async function getMediaEntry(paths, hash) {
+    const index = await loadCacheIndex(paths);
+    return index.entries.find((item) => item.hash === hash) ?? null;
+}
+async function getCacheStats(paths) {
+    const index = await loadCacheIndex(paths);
+    const refs = await loadRefsIndex(paths);
+    if (index.entries.length === 0) {
+        return {
+            count: 0,
+            totalSizeBytes: 0,
+            referencedCount: 0,
+            unreferencedCount: 0,
+            evictionBlockedCount: index.evictionBlockedCount ?? 0,
+        };
+    }
+    const sorted = [...index.entries].sort((a, b) => a.createdAt.getTime() - b.createdAt.getTime());
+    const referencedCount = index.entries.filter((entry) => isHashReferenced(refs, entry.hash)).length;
+    return {
+        count: index.entries.length,
+        totalSizeBytes: index.totalSize,
+        referencedCount,
+        unreferencedCount: index.entries.length - referencedCount,
+        evictionBlockedCount: index.evictionBlockedCount ?? 0,
+        oldestEntry: sorted[0].createdAt,
+        newestEntry: sorted[sorted.length - 1].createdAt,
+    };
+}
+async function clearCache(paths) {
+    const index = await loadCacheIndex(paths);
+    let deleted = 0;
+    for (const entry of index.entries) {
+        try {
+            await fs_1.promises.unlink(path_1.default.join(resolveMediaDir(paths), entry.filename));
+            deleted += 1;
+        }
+        catch {
+            // ignore missing file
+        }
+    }
+    await saveCacheIndex(paths, { entries: [], totalSize: 0, evictionBlockedCount: 0 });
+    await saveRefsIndex(paths, { refs: {} });
+    return deleted;
+}
+async function syncSessionMediaReferences(paths, sessionId, hashes) {
+    await ensureMediaDir(paths);
+    const refs = await loadRefsIndex(paths);
+    const nextHashes = Array.from(new Set(hashes.filter((hash) => /^[a-f0-9]{16}$/i.test(hash))));
+    // Remove previous references for this session from all hashes.
+    for (const [hash, sessionIds] of Object.entries(refs.refs)) {
+        const filtered = sessionIds.filter((id) => id !== sessionId);
+        if (filtered.length > 0) {
+            refs.refs[hash] = filtered;
+        }
+        else {
+            delete refs.refs[hash];
+        }
+    }
+    for (const hash of nextHashes) {
+        const existing = refs.refs[hash] ?? [];
+        refs.refs[hash] = Array.from(new Set([...existing, sessionId]));
+    }
+    await saveRefsIndex(paths, refs);
+}
+async function unmarkSessionMediaReferences(paths, sessionId) {
+    await ensureMediaDir(paths);
+    const refs = await loadRefsIndex(paths);
+    for (const [hash, sessionIds] of Object.entries(refs.refs)) {
+        const filtered = sessionIds.filter((id) => id !== sessionId);
+        if (filtered.length > 0) {
+            refs.refs[hash] = filtered;
+        }
+        else {
+            delete refs.refs[hash];
+        }
+    }
+    await saveRefsIndex(paths, refs);
+}
+async function getMediaRefCount(paths, hash) {
+    const refs = await loadRefsIndex(paths);
+    return refs.refs[hash]?.length ?? 0;
+}
+/**
+ * Remove all media entries that are no longer referenced by any session.
+ * Call this after deleting a session to free up disk space.
+ * Returns the number of files deleted.
+ */
+async function cleanOrphanedMedia(paths) {
+    const index = await loadCacheIndex(paths);
+    const refs = await loadRefsIndex(paths);
+    let deleted = 0;
+    const surviving = [];
+    let survivingSize = 0;
+    for (const entry of index.entries) {
+        if (isHashReferenced(refs, entry.hash)) {
+            surviving.push(entry);
+            survivingSize += entry.size;
+        }
+        else {
+            try {
+                await fs_1.promises.unlink(path_1.default.join(resolveMediaDir(paths), entry.filename));
+                deleted += 1;
+            }
+            catch {
+                // File already gone — still remove from index
+                deleted += 1;
+            }
+        }
+    }
+    if (deleted > 0) {
+        await saveCacheIndex(paths, { ...index, entries: surviving, totalSize: survivingSize });
+    }
+    return deleted;
+}
+function isHashReferenced(refs, hash) {
+    const sessionIds = refs.refs[hash];
+    return Array.isArray(sessionIds) && sessionIds.length > 0;
+}
+function normalizeMediaInput(data, hintedMime) {
+    if (Buffer.isBuffer(data)) {
+        const mimeType = hintedMime ?? detectMimeType(data);
+        return { buffer: data, mimeType };
+    }
+    const trimmed = data.trim();
+    const dataUrlMatch = trimmed.match(/^data:([^;]+);base64,(.+)$/i);
+    if (dataUrlMatch) {
+        const mimeType = dataUrlMatch[1].toLowerCase();
+        const base64Payload = dataUrlMatch[2].replace(/\s+/g, "");
+        return { buffer: Buffer.from(base64Payload, "base64"), mimeType };
+    }
+    const buffer = Buffer.from(trimmed.replace(/\s+/g, ""), "base64");
+    const mimeType = hintedMime ?? detectMimeType(buffer);
+    return { buffer, mimeType };
+}
+function extensionFromMime(mimeType, buffer) {
+    const map = {
+        "image/png": "png",
+        "image/jpeg": "jpg",
+        "image/jpg": "jpg",
+        "image/gif": "gif",
+        "image/webp": "webp",
+        "audio/wav": "wav",
+        "audio/x-wav": "wav",
+        "audio/mpeg": "mp3",
+        "audio/mp3": "mp3",
+        "audio/ogg": "ogg",
+        "audio/webm": "webm",
+        "audio/mp4": "m4a",
+    };
+    if (map[mimeType]) {
+        return map[mimeType];
+    }
+    // fallback by magic bytes
+    if (buffer[0] === 0x89 && buffer[1] === 0x50 && buffer[2] === 0x4e && buffer[3] === 0x47)
+        return "png";
+    if (buffer[0] === 0xff && buffer[1] === 0xd8 && buffer[2] === 0xff)
+        return "jpg";
+    if (buffer[0] === 0x47 && buffer[1] === 0x49 && buffer[2] === 0x46)
+        return "gif";
+    if (buffer[0] === 0x52 && buffer[1] === 0x49 && buffer[2] === 0x46 && buffer[3] === 0x46) {
+        if (buffer.length > 11) {
+            const chunk = buffer.subarray(8, 12).toString("ascii");
+            if (chunk === "WAVE")
+                return "wav";
+            if (chunk === "WEBP")
+                return "webp";
+        }
+    }
+    return "bin";
+}
+function detectMimeType(buffer) {
+    if (buffer[0] === 0x89 && buffer[1] === 0x50 && buffer[2] === 0x4e && buffer[3] === 0x47) {
+        return "image/png";
+    }
+    if (buffer[0] === 0xff && buffer[1] === 0xd8 && buffer[2] === 0xff) {
+        return "image/jpeg";
+    }
+    if (buffer[0] === 0x47 && buffer[1] === 0x49 && buffer[2] === 0x46) {
+        return "image/gif";
+    }
+    if (buffer[0] === 0x52 && buffer[1] === 0x49 && buffer[2] === 0x46 && buffer[3] === 0x46) {
+        if (buffer.length > 11) {
+            const chunk = buffer.subarray(8, 12).toString("ascii");
+            if (chunk === "WAVE")
+                return "audio/wav";
+            if (chunk === "WEBP")
+                return "image/webp";
+        }
+    }
+    if (buffer.length > 3 && buffer[0] === 0x49 && buffer[1] === 0x44 && buffer[2] === 0x33) {
+        return "audio/mpeg";
+    }
+    return "application/octet-stream";
+}