videobook 0.0.0-stage → 0.1.0

package/dist/app.js

@@ -0,0 +1,257 @@
+import cors from "cors";
+import express from "express";
+import { createHash } from "node:crypto";
+import { createWriteStream, existsSync } from "node:fs";
+import { mkdtemp, rm } from "node:fs/promises";
+import { createRequire } from "node:module";
+import { tmpdir } from "node:os";
+import { extname, join, resolve } from "node:path";
+import { Transform } from "node:stream";
+import { pipeline } from "node:stream/promises";
+import { fileURLToPath } from "node:url";
+import { toNodeHandler } from "@modelcontextprotocol/node";
+import { SCHEMA_VERSION } from "videobook-engine";
+import { v7 as uuidv7 } from "uuid";
+import { localArchiveDir } from "./archive-store.js";
+import { bookManager, scopedEngine } from "./engine.js";
+import { parseHttpByteRange } from "./http-byte-range.js";
+import { restrictHttpAccess, restrictMediaContent } from "./http-access.js";
+import { uploadLocalStream } from "./local-upload.js";
+import { queue } from "./queue.js";
+import { searchReferenceStore } from "./search-reference-store.js";
+import { SERVER_VERSION, createHttpHandler } from "./server.js";
+import { VoiceSessions } from "./voice-session.js";
+import { voiceRouter } from "./voice-http.js";
+const enginePackage = createRequire(import.meta.url)("videobook-engine/package.json");
+const SEARCH_REFERENCE_LIMIT_BYTES = 500 * 1024 * 1024;
+const RESERVED_ROOTS = new Set(["mcp", "api", "files", "archive-files", "healthz", "assets"]);
+function defaultUiDir(env = process.env) {
+    const override = env.VIDEOBOOK_UI_DIR?.trim();
+    return override ? resolve(override) : fileURLToPath(new URL("../ui/", import.meta.url));
+}
+function limitUploadBytes(maxBytes) {
+    let received = 0;
+    return new Transform({
+        transform(chunk, _encoding, callback) {
+            received += chunk.byteLength;
+            if (received > maxBytes) {
+                callback(new Error("Search reference exceeds the 500 MB limit"));
+                return;
+            }
+            callback(null, chunk);
+        },
+    });
+}
+async function engineForBook(bookId) {
+    return bookManager.ensure(bookId);
+}
+function sendHealth(_req, res) {
+    res.json({
+        ok: true,
+        service: "videobook",
+        version: SERVER_VERSION,
+        engineVersion: enginePackage.version,
+        schemaVersion: SCHEMA_VERSION,
+    });
+}
+async function sendArtifactFile(req, res) {
+    const bookId = String(req.params[0]);
+    const artifactId = String(req.params[1]);
+    const filename = String(req.params[2]);
+    const book = await engineForBook(bookId);
+    if (!book.ok) {
+        res.status(404).json({ error: book.error.message });
+        return;
+    }
+    const result = await book.value.engine.files.read(artifactId, filename);
+    if (!result.ok) {
+        res.status(result.error.code === "NOT_FOUND" ? 404 : 400).json({ error: result.error.message });
+        return;
+    }
+    res.type(filename);
+    res.setHeader("Accept-Ranges", "bytes");
+    const etag = `"${createHash("sha256").update(result.value).digest("hex")}"`;
+    res.setHeader("ETag", etag);
+    if (req.fresh) {
+        res.status(304).end();
+        return;
+    }
+    const rangeHeader = req.headers.range;
+    const ifRange = req.get("If-Range");
+    if (req.method === "HEAD" || !rangeHeader || (ifRange !== undefined && ifRange !== etag)) {
+        res.send(result.value);
+        return;
+    }
+    const range = parseHttpByteRange(rangeHeader, result.value.byteLength);
+    if (!range.ok) {
+        res.setHeader("Content-Range", `bytes */${result.value.byteLength}`);
+        res.sendStatus(416);
+        return;
+    }
+    res.status(206);
+    res.setHeader("Content-Range", `bytes ${range.start}-${range.end}/${result.value.byteLength}`);
+    res.send(result.value.subarray(range.start, range.end + 1));
+}
+async function receiveUpload(req, res) {
+    const filename = typeof req.query.filename === "string" ? req.query.filename : "";
+    if (!filename) {
+        res.status(400).json({ error: "Missing filename" });
+        return;
+    }
+    const book = await engineForBook(String(req.params.bookId));
+    if (!book.ok) {
+        res.status(404).json({ error: book.error.message });
+        return;
+    }
+    const targetAssetId = typeof req.query.targetAssetId === "string" ? req.query.targetAssetId : undefined;
+    const result = await uploadLocalStream(book.value.engine, {
+        bookId: book.value.bookId,
+        filename,
+        stream: req,
+        targetAssetId,
+    });
+    if (!result.ok) {
+        res.status(result.statusCode).json({ error: result.message });
+        return;
+    }
+    res.json(result.value);
+}
+async function receiveSearchReference(req, res) {
+    const filename = typeof req.query.filename === "string" ? req.query.filename : "";
+    const bookId = typeof req.query.bookId === "string" ? req.query.bookId : "";
+    const kind = req.query.kind === "image" || req.query.kind === "video"
+        ? req.query.kind
+        : null;
+    if (!filename || !bookId || !kind) {
+        res.status(400).json({ error: "filename, bookId, and kind are required" });
+        return;
+    }
+    const book = await engineForBook(bookId);
+    if (!book.ok) {
+        res.status(404).json({ error: book.error.message });
+        return;
+    }
+    const declaredLength = Number(req.headers["content-length"] ?? 0);
+    if (declaredLength > SEARCH_REFERENCE_LIMIT_BYTES) {
+        res.status(413).json({ error: "Search reference exceeds the 500 MB limit" });
+        return;
+    }
+    const uploadRoot = await mkdtemp(join(tmpdir(), "videobook-search-upload-"));
+    const suffix = extname(filename).toLowerCase().replace(/[^a-z0-9.]/g, "");
+    const uploadPath = join(uploadRoot, `reference${suffix}`);
+    try {
+        await pipeline(req, limitUploadBytes(SEARCH_REFERENCE_LIMIT_BYTES), createWriteStream(uploadPath, { flags: "wx" }));
+        const result = await queue.enqueueAndWait(`prepare-search-reference-${uuidv7()}`, {
+            type: "prepare_search_reference",
+            bookId,
+            mediaKind: kind,
+            name: filename,
+            uploadPath,
+            cleanupRoot: uploadRoot,
+        }, 15 * 60_000);
+        res.json(result);
+    }
+    catch (cause) {
+        await rm(uploadRoot, { recursive: true, force: true });
+        const message = cause instanceof Error ? cause.message : String(cause);
+        res.status(message.includes("500 MB limit") ? 413 : 400).json({
+            error: message,
+        });
+    }
+}
+function sendSearchReferencePreview(req, res) {
+    try {
+        const preview = searchReferenceStore.get(String(req.params.referenceId)).preview;
+        if (!preview) {
+            res.status(404).json({ error: "Search reference preview is unavailable" });
+            return;
+        }
+        res.set("Cache-Control", "no-store").type("png").send(preview);
+    }
+    catch (cause) {
+        res.status(404).json({ error: cause instanceof Error ? cause.message : String(cause) });
+    }
+}
+function releaseSearchReference(req, res) {
+    res.json({
+        released: searchReferenceStore.release(String(req.params.referenceId)),
+    });
+}
+async function sendArtifactManifest(req, res) {
+    const book = await engineForBook(String(req.params.bookId));
+    if (!book.ok) {
+        res.status(404).json({ error: book.error.message });
+        return;
+    }
+    const result = await book.value.engine.files.manifest(String(req.params.artifactId), {
+        includeDotfiles: true,
+    });
+    if (!result.ok) {
+        res.status(result.error.code === "NOT_FOUND" ? 404 : 400).json({ error: result.error.message });
+        return;
+    }
+    res.json(result.value);
+}
+function registerAccessPolicy(app) {
+    app.use(restrictHttpAccess);
+    app.use(cors({ origin: true }));
+    app.use(["/files", "/archive-files"], restrictMediaContent);
+}
+function registerMediaRoutes(app) {
+    app.use("/archive-files", express.static(localArchiveDir(), { dotfiles: "allow" }));
+    app.get("/healthz", sendHealth);
+    app.get(/^\/files\/([^/]+)\/([^/]+)\/(.+)$/, sendArtifactFile);
+}
+function registerApiRoutes(app, voiceSessions) {
+    app.post("/api/books/:bookId/upload", receiveUpload);
+    app.post("/api/search-references", receiveSearchReference);
+    app.get("/api/search-references/:referenceId/preview", sendSearchReferencePreview);
+    app.delete("/api/search-references/:referenceId", releaseSearchReference);
+    app.use("/api/voice", voiceRouter(voiceSessions));
+    app.use(express.json({ limit: "500mb" }));
+    app.get("/api/books/:bookId/artifacts/:artifactId/manifest", sendArtifactManifest);
+}
+function registerMcpRoute(app, mcpHandler) {
+    const handleMcpRequest = toNodeHandler(mcpHandler);
+    app.all("/mcp", (req, res) => {
+        void handleMcpRequest(req, res, req.body);
+    });
+}
+function isAppRoute(path) {
+    return !RESERVED_ROOTS.has(path.split("/")[1] ?? "");
+}
+function spaFallback(indexPath) {
+    return (req, res, next) => {
+        if ((req.method !== "GET" && req.method !== "HEAD") || !isAppRoute(req.path) || !req.accepts("html")) {
+            next();
+            return;
+        }
+        res.set("Cache-Control", "no-cache");
+        res.sendFile(indexPath);
+    };
+}
+function registerUi(app, uiDir) {
+    app.use("/assets", express.static(join(uiDir, "assets"), { immutable: true, maxAge: "1y", index: false }));
+    app.use(express.static(uiDir, { index: false }));
+    app.use(spaFallback(join(uiDir, "index.html")));
+}
+function mountUi(app, options, uiDir) {
+    if (!options.ui)
+        return "disabled";
+    if (!existsSync(join(uiDir, "index.html")))
+        return "missing";
+    registerUi(app, uiDir);
+    return "served";
+}
+export function createApp(options) {
+    const app = express();
+    const voiceSessions = new VoiceSessions(bookManager);
+    const mcpHandler = createHttpHandler(scopedEngine, bookManager);
+    registerAccessPolicy(app);
+    registerMediaRoutes(app);
+    registerApiRoutes(app, voiceSessions);
+    registerMcpRoute(app, mcpHandler);
+    const uiDir = options.uiDir ?? defaultUiDir();
+    const uiStatus = mountUi(app, options, uiDir);
+    return { app, mcpHandler, voiceSessions, uiDir, uiStatus };
+}

package/dist/archive-b2.js

@@ -0,0 +1,332 @@
+import { createB2 } from "@apicity/b2";
+import { createHash } from "node:crypto";
+import { createReadStream } from "node:fs";
+import { open, readFile, stat } from "node:fs/promises";
+import { extname } from "node:path";
+import { getApiKey, requireApiKey } from "./api-key.js";
+const JSON_CONTENT_TYPE = "application/json";
+const TEXT_CONTENT_TYPE = "text/plain; charset=utf-8";
+const MULTIPART_THRESHOLD_BYTES = 64 * 1024 * 1024;
+const MULTIPART_PART_BYTES = 64 * 1024 * 1024;
+function isNotFoundError(error) {
+    if (!error || typeof error !== "object")
+        return false;
+    const status = error.status;
+    return status === 404;
+}
+function cleanPrefix(value) {
+    return (value ?? "").replace(/^\/+|\/+$/g, "");
+}
+export function getArchiveConfig() {
+    const accessKeyId = requireApiKey("B2_ACCESS_KEY_ID");
+    const secretAccessKey = requireApiKey("B2_SECRET_ACCESS_KEY");
+    const region = process.env.B2_REGION;
+    const bucket = process.env.B2_BUCKET;
+    if (!region)
+        throw new Error("B2_REGION environment variable is not set");
+    if (!bucket)
+        throw new Error("B2_BUCKET environment variable is not set");
+    return {
+        accessKeyId,
+        secretAccessKey,
+        region,
+        bucket,
+        ...(process.env.B2_ENDPOINT ? { endpoint: process.env.B2_ENDPOINT } : {}),
+        prefix: cleanPrefix(process.env.B2_ARCHIVE_PREFIX),
+    };
+}
+export function missingArchiveConfigKeys() {
+    const names = ["B2_ACCESS_KEY_ID", "B2_SECRET_ACCESS_KEY", "B2_REGION", "B2_BUCKET"];
+    return names.filter((name) => !process.env[name] || getApiKey(name) === null);
+}
+export function createArchiveB2(config = getArchiveConfig()) {
+    return createB2({
+        accessKeyId: config.accessKeyId,
+        secretAccessKey: config.secretAccessKey,
+        region: config.region,
+        ...(config.endpoint ? { endpoint: config.endpoint } : {}),
+        timeout: 300_000,
+    });
+}
+export function archiveKey(config, key) {
+    return config.prefix ? `${config.prefix}/${key}` : key;
+}
+export function clipPrefix(sha256) {
+    return `clips/${sha256}/`;
+}
+export function clipKey(config, sha256, filename) {
+    return archiveKey(config, `${clipPrefix(sha256)}${filename}`);
+}
+export function assetSnapshotPrefix(config, sha256) {
+    return archiveKey(config, clipPrefix(sha256));
+}
+function flattenSnapshotPath(relativePath) {
+    return relativePath
+        .replace(/^\/+/, "")
+        .split("/")
+        .filter(Boolean)
+        .join("__");
+}
+export function assetSnapshotKey(config, sha256, relativePath) {
+    return `${assetSnapshotPrefix(config, sha256)}${flattenSnapshotPath(relativePath)}`;
+}
+export function indexKey(config, sha256) {
+    return clipKey(config, sha256, "archive.json");
+}
+export function transcriptJsonKey(config, sha256) {
+    return clipKey(config, sha256, "transcript.json");
+}
+export function transcriptTextKey(config, sha256) {
+    return clipKey(config, sha256, "transcript.txt");
+}
+export function chunkKey(config, sha256, index) {
+    return clipKey(config, sha256, `chunk-${String(index).padStart(3, "0")}.json`);
+}
+export function ingestKey(config, sha256, ingestId) {
+    return clipKey(config, sha256, `ingest-${ingestId}.json`);
+}
+export function topicMarkerKey(config, topicKey, sha256, ingestId) {
+    return archiveKey(config, `topics/${topicKey}/${sha256}-${ingestId}.json`);
+}
+export function personMarkerKey(config, personKey, sha256, ingestId) {
+    return archiveKey(config, `people/${personKey}/${sha256}-${ingestId}.json`);
+}
+export function normalizeArchiveKey(value) {
+    return value
+        .normalize("NFKD")
+        .replace(/[\u0300-\u036f]/g, "")
+        .toLowerCase()
+        .replace(/[^a-z0-9]+/g, "-")
+        .replace(/^-+|-+$/g, "")
+        .slice(0, 96);
+}
+export function archiveTags(values, source) {
+    const seen = new Set();
+    const tags = [];
+    for (const value of values ?? []) {
+        const label = value.trim().replace(/\s+/g, " ");
+        const key = normalizeArchiveKey(label);
+        if (!label || !key || seen.has(key))
+            continue;
+        seen.add(key);
+        tags.push({ key, label, source });
+    }
+    return tags;
+}
+function mergeTags(groups) {
+    const merged = new Map();
+    for (const group of groups) {
+        for (const tag of group) {
+            const existing = merged.get(tag.key);
+            if (!existing || (existing.source === "auto" && tag.source === "manual")) {
+                merged.set(tag.key, tag);
+            }
+        }
+    }
+    return [...merged.values()].sort((a, b) => a.label.localeCompare(b.label));
+}
+export function mergeArchiveIngests(ingests, args) {
+    const now = new Date().toISOString();
+    const sorted = [...ingests].sort((a, b) => a.created_at.localeCompare(b.created_at));
+    const existing = args.existing ?? null;
+    const createdAt = existing?.created_at ?? sorted[0]?.created_at ?? now;
+    const title = sorted.find((ingest) => ingest.title)?.title ?? existing?.title ?? null;
+    return {
+        schema_version: 1,
+        sha256: args.sha256,
+        archive_id: args.sha256,
+        title,
+        topics: mergeTags([existing?.topics ?? [], ...sorted.map((ingest) => ingest.topics)]),
+        people: mergeTags([existing?.people ?? [], ...sorted.map((ingest) => ingest.people)]),
+        sources: sorted.map((ingest) => ingest.source),
+        ingests: sorted.map((ingest) => ingest.ingest_id),
+        media: {
+            key: args.mediaKey,
+            filename: args.filename,
+            content_type: args.contentType,
+            size_bytes: args.sizeBytes,
+        },
+        transcript: args.transcript ?? existing?.transcript ?? null,
+        duration: args.duration ?? existing?.duration ?? null,
+        created_at: createdAt,
+        updated_at: now,
+    };
+}
+export function contentTypeForFilename(filename) {
+    switch (extname(filename).toLowerCase()) {
+        case ".aac":
+            return "audio/aac";
+        case ".flac":
+            return "audio/flac";
+        case ".gif":
+            return "image/gif";
+        case ".jpg":
+        case ".jpeg":
+            return "image/jpeg";
+        case ".json":
+            return "application/json";
+        case ".md":
+            return "text/markdown; charset=utf-8";
+        case ".mp4":
+            return "video/mp4";
+        case ".mov":
+            return "video/quicktime";
+        case ".webm":
+            return "video/webm";
+        case ".weba":
+            return "audio/webm";
+        case ".mkv":
+            return "video/x-matroska";
+        case ".mp3":
+            return "audio/mpeg";
+        case ".wav":
+            return "audio/wav";
+        case ".m4a":
+            return "audio/mp4";
+        case ".ogg":
+            return "audio/ogg";
+        case ".png":
+            return "image/png";
+        case ".txt":
+            return "text/plain; charset=utf-8";
+        case ".webp":
+            return "image/webp";
+        default:
+            return "application/octet-stream";
+    }
+}
+export async function hashFile(filePath) {
+    const hash = createHash("sha256");
+    await new Promise((resolve, reject) => {
+        const stream = createReadStream(filePath);
+        stream.on("data", (chunk) => hash.update(chunk));
+        stream.on("error", reject);
+        stream.on("end", resolve);
+    });
+    return hash.digest("hex");
+}
+export async function objectExists(b2, bucket, key) {
+    try {
+        await b2.objects.head({ bucket, key });
+        return true;
+    }
+    catch (error) {
+        if (isNotFoundError(error))
+            return false;
+        throw error;
+    }
+}
+export async function putJson(b2, bucket, key, value) {
+    await b2.objects.put({
+        bucket,
+        key,
+        body: JSON.stringify(value, null, 2),
+        contentType: JSON_CONTENT_TYPE,
+    });
+}
+export async function putText(b2, bucket, key, value) {
+    await b2.objects.put({
+        bucket,
+        key,
+        body: value,
+        contentType: TEXT_CONTENT_TYPE,
+    });
+}
+export async function getJson(b2, bucket, key) {
+    try {
+        const response = await b2.objects.get({ bucket, key });
+        const text = new TextDecoder().decode(response.body);
+        return JSON.parse(text);
+    }
+    catch (error) {
+        if (isNotFoundError(error))
+            return null;
+        throw error;
+    }
+}
+export async function getText(b2, bucket, key) {
+    try {
+        const response = await b2.objects.get({ bucket, key });
+        return new TextDecoder().decode(response.body);
+    }
+    catch (error) {
+        if (isNotFoundError(error))
+            return null;
+        throw error;
+    }
+}
+export async function listObjects(b2, bucket, prefix) {
+    const out = [];
+    let continuationToken;
+    do {
+        const response = await b2.objects.list({
+            bucket,
+            prefix,
+            ...(continuationToken ? { continuationToken } : {}),
+        });
+        const contents = response.contents;
+        out.push(...contents.map((item) => ({
+            key: item.key,
+            ...(item.size !== undefined ? { size: item.size } : {}),
+            ...(item.lastModified ? { lastModified: item.lastModified } : {}),
+        })));
+        continuationToken = response.nextContinuationToken;
+    } while (continuationToken);
+    return out;
+}
+export async function uploadFileToB2(b2, bucket, key, filePath, contentType) {
+    const fileStat = await stat(filePath);
+    if (fileStat.size <= MULTIPART_THRESHOLD_BYTES) {
+        const body = await readFile(filePath);
+        await b2.objects.put({
+            bucket,
+            key,
+            body: new Uint8Array(body.buffer.slice(body.byteOffset, body.byteOffset + body.byteLength)),
+            contentType,
+        });
+        return;
+    }
+    const create = await b2.objects.createMultipartUpload({ bucket, key, contentType });
+    const parts = [];
+    const handle = await open(filePath, "r");
+    try {
+        let offset = 0;
+        let partNumber = 1;
+        while (offset < fileStat.size) {
+            const length = Math.min(MULTIPART_PART_BYTES, fileStat.size - offset);
+            const buffer = Buffer.allocUnsafe(length);
+            const read = await handle.read(buffer, 0, length, offset);
+            if (read.bytesRead <= 0)
+                break;
+            const chunk = buffer.subarray(0, read.bytesRead);
+            const uploaded = await b2.objects.uploadPart({
+                bucket,
+                key,
+                uploadId: create.uploadId,
+                partNumber,
+                body: new Uint8Array(chunk.buffer.slice(chunk.byteOffset, chunk.byteOffset + chunk.byteLength)),
+            });
+            if (!uploaded.eTag)
+                throw new Error(`B2 uploadPart did not return an ETag for ${key} part ${partNumber}`);
+            parts.push({ partNumber, eTag: uploaded.eTag });
+            offset += read.bytesRead;
+            partNumber += 1;
+        }
+        if (offset < fileStat.size) {
+            throw new Error(`Short read uploading ${key}: got ${offset} of ${fileStat.size} bytes`);
+        }
+        await b2.objects.completeMultipartUpload({
+            bucket,
+            key,
+            uploadId: create.uploadId,
+            parts,
+        });
+    }
+    catch (error) {
+        await b2.objects.abortMultipartUpload({ bucket, key, uploadId: create.uploadId }).catch(() => undefined);
+        throw error;
+    }
+    finally {
+        await handle.close();
+    }
+}