videobook 0.1.0 → 0.2.0

package/dist/one-password.js

@@ -0,0 +1,200 @@
+import { execFileSync } from "node:child_process";
+import { randomUUID } from "node:crypto";
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+export const OP_VAULT_SETTING = "VIDEOBOOK_OP_VAULT";
+export const OP_TOKEN_SETTING = "VIDEOBOOK_OP_SERVICE_TOKEN";
+const OP_TIMEOUT_MS = 10_000;
+const TEMPLATE_UNSAFE = /[\r\n]|\{\{|\}\}/u;
+const ENV_NAME = /^[A-Za-z_][A-Za-z0-9_]*$/u;
+export function isOpReference(value) {
+    return typeof value === "string" && value.trim().startsWith("op://");
+}
+export function vaultReference(vault, key) {
+    return `op://${vault}/${key}/password`;
+}
+export function resolveServiceToken(raw, env) {
+    const value = raw?.trim();
+    if (!value)
+        return { value: null, source: null };
+    const name = value.startsWith("env:") ? value.slice(4) : value.startsWith("$") ? value.slice(1) : value;
+    if (value !== name || /^[A-Z_][A-Z0-9_]*$/u.test(value)) {
+        if (!ENV_NAME.test(name))
+            return { value: null, source: `env:${name}` };
+        return { value: env[name]?.trim() || null, source: `env:${name}` };
+    }
+    return { value, source: "literal" };
+}
+export function readOnePasswordSettings(env) {
+    return {
+        vault: env[OP_VAULT_SETTING]?.trim() || null,
+        token: resolveServiceToken(env[OP_TOKEN_SETTING], env),
+    };
+}
+export function redactToken(text, token) {
+    return token ? text.split(token).join("***") : text;
+}
+export function planReferences(env, managedKeys, vault, vaultTitles) {
+    const explicit = Object.entries(env)
+        .filter((entry) => isOpReference(entry[1]))
+        .map(([key, reference]) => [key, reference.trim()]);
+    const covered = new Set(explicit.map(([key]) => key));
+    const convention = vault
+        ? managedKeys
+            .filter((key) => !covered.has(key) && !env[key]?.trim() && vaultTitles.has(key))
+            .map((key) => [key, vaultReference(vault, key)])
+        : [];
+    return { explicit, convention };
+}
+function startMarker(nonce, key) {
+    return `<<<${nonce}:${key}>>>`;
+}
+function endMarker(nonce, key) {
+    return `<<<${nonce}/${key}>>>`;
+}
+export function injectionTemplate(entries, nonce) {
+    return entries.map(([key, reference]) => `${startMarker(nonce, key)}{{ ${reference} }}${endMarker(nonce, key)}`).join("\n");
+}
+export function parseInjection(output, keys, nonce) {
+    const values = new Map();
+    for (const key of keys) {
+        const start = output.indexOf(startMarker(nonce, key));
+        if (start < 0)
+            continue;
+        const from = start + startMarker(nonce, key).length;
+        const end = output.indexOf(endMarker(nonce, key), from);
+        if (end < 0)
+            continue;
+        const value = output.slice(from, end);
+        if (value !== "")
+            values.set(key, value);
+    }
+    return values;
+}
+function opEnvironment(env, token) {
+    return token ? { ...env, OP_SERVICE_ACCOUNT_TOKEN: token } : env;
+}
+export function opErrorMessage(error, token) {
+    const failure = error;
+    if (failure.code === "ENOENT")
+        return "the 1Password CLI (op) is not on PATH";
+    if (failure.code === "ETIMEDOUT" || failure.signal === "SIGTERM")
+        return "op timed out";
+    const line = String(failure.stderr ?? "").trim().split("\n")[0]?.trim().replace(/^\[ERROR\]\s+\S+\s+\S+\s+/u, "");
+    return redactToken(line || failure.message || "op failed", token);
+}
+function runOp(args, env, token) {
+    return execFileSync("op", args, {
+        env: opEnvironment(env, token),
+        encoding: "utf8",
+        timeout: OP_TIMEOUT_MS,
+        maxBuffer: 8 * 1024 * 1024,
+        stdio: ["ignore", "pipe", "pipe"],
+    });
+}
+export function listVaultItemTitles(vault, env, token) {
+    const parsed = JSON.parse(runOp(["item", "list", "--vault", vault, "--format", "json"], env, token));
+    if (!Array.isArray(parsed))
+        throw new Error("op item list did not return a list");
+    return new Set(parsed.flatMap((item) => {
+        const title = item && typeof item === "object" ? item.title : undefined;
+        return typeof title === "string" ? [title] : [];
+    }));
+}
+function injectBatch(entries, env, token) {
+    const nonce = randomUUID().replaceAll("-", "");
+    const directory = mkdtempSync(join(tmpdir(), "videobook-op-"));
+    try {
+        const template = join(directory, "template");
+        writeFileSync(template, injectionTemplate(entries, nonce), { mode: 0o600 });
+        const output = runOp(["inject", "--in-file", template], env, token);
+        return parseInjection(output, entries.map(([key]) => key), nonce);
+    }
+    finally {
+        rmSync(directory, { recursive: true, force: true });
+    }
+}
+function readOne(key, reference, env, token) {
+    try {
+        const value = runOp(["read", "--no-newline", reference], env, token);
+        return value === ""
+            ? { values: new Map(), failures: [{ key, reference, reason: "the item has no value" }] }
+            : { values: new Map([[key, value]]), failures: [] };
+    }
+    catch (error) {
+        return { values: new Map(), failures: [{ key, reference, reason: opErrorMessage(error, token) }] };
+    }
+}
+function mergeResults(results) {
+    return {
+        values: new Map(results.flatMap((result) => [...result.values])),
+        failures: results.flatMap((result) => result.failures),
+    };
+}
+function resolveEntries(entries, env, token) {
+    const unsafe = entries.filter(([, reference]) => TEMPLATE_UNSAFE.test(reference));
+    const safe = entries.filter(([, reference]) => !TEMPLATE_UNSAFE.test(reference));
+    const refused = unsafe.map(([key, reference]) => ({ key, reference, reason: "malformed op:// reference" }));
+    if (safe.length === 0)
+        return { values: new Map(), failures: refused };
+    try {
+        const values = injectBatch(safe, env, token);
+        const missing = safe.filter(([key]) => !values.has(key)).map(([key, reference]) => ({ key, reference, reason: "op inject returned no value" }));
+        return { values, failures: [...refused, ...missing] };
+    }
+    catch {
+        const individual = mergeResults(safe.map(([key, reference]) => readOne(key, reference, env, token)));
+        return { values: individual.values, failures: [...refused, ...individual.failures] };
+    }
+}
+function vaultTitles(settings, env) {
+    if (!settings.vault)
+        return { titles: new Set(), error: null };
+    try {
+        return { titles: listVaultItemTitles(settings.vault, env, settings.token.value), error: null };
+    }
+    catch (error) {
+        return { titles: new Set(), error: `could not list vault ${settings.vault}: ${opErrorMessage(error, settings.token.value)}` };
+    }
+}
+function emptyResolution(settings) {
+    return {
+        attempted: false,
+        vault: settings.vault,
+        tokenSource: settings.token.source,
+        fromReferences: [],
+        fromVault: [],
+        unresolved: [],
+        error: null,
+    };
+}
+export function resolveOnePasswordEnv(env, managedKeys) {
+    const settings = readOnePasswordSettings(env);
+    const hasReferences = Object.values(env).some((value) => isOpReference(value));
+    if (!hasReferences && !settings.vault)
+        return emptyResolution(settings);
+    const listing = vaultTitles(settings, env);
+    const plan = planReferences(env, managedKeys, settings.vault, listing.titles);
+    const result = resolveEntries([...plan.explicit, ...plan.convention], env, settings.token.value);
+    for (const [key] of plan.explicit) {
+        const value = result.values.get(key);
+        if (value === undefined)
+            delete env[key];
+        else
+            env[key] = value;
+    }
+    for (const [key] of plan.convention) {
+        const value = result.values.get(key);
+        if (value !== undefined)
+            env[key] = value;
+    }
+    return {
+        ...emptyResolution(settings),
+        attempted: true,
+        fromReferences: plan.explicit.map(([key]) => key).filter((key) => result.values.has(key)),
+        fromVault: plan.convention.map(([key]) => key).filter((key) => result.values.has(key)),
+        unresolved: result.failures,
+        error: listing.error,
+    };
+}

package/dist/queue.js

@@ -517,6 +517,10 @@ export function enqueueOptions(jobId, payload) {
         artifactWorkOrientation: classified?.orientation ?? null,
     };
 }
+export function jobIdForKey(jobKey, bookId) {
+    const entry = jobIdIndex.get(jobKey);
+    return entry && entry.bookId === bookId ? entry.internalId : null;
+}
 function rememberJob(jobId, entry) {
     jobIdIndex.set(jobId, entry);
     while (jobIdIndex.size > JOB_INDEX_MAX) {

package/dist/serve.js

@@ -3,45 +3,19 @@ import { setTimeout as delay } from "node:timers/promises";
 import { stopAceServer } from "./acestep/process.js";
 import { createApp } from "./app.js";
 import { bookManager, scopedEngine } from "./engine.js";
+import { getDefaultSecretStore } from "./env-secrets.js";
+import { hydrateLibrary } from "./hydration.js";
 import { registerAllJobHandlers } from "./jobs/index.js";
 import { logger } from "./logger.js";
-import { startPostMetricsScheduler, stopPostMetricsScheduler } from "./posts/post-metrics.js";
+import { stopPostMetricsScheduler } from "./posts/post-metrics.js";
 import { queue } from "./queue.js";
-import { loadAndArmAll, stopScheduler } from "./scheduled-posts.js";
+import { stopScheduler } from "./scheduled-posts.js";
 import { SERVER_VERSION } from "./server.js";
 import { ServerAlreadyRunningError, acquireServerLock, inspectServerOwner, } from "./server-lock.js";
 import { attachVoiceAudio } from "./voice-audio.js";
 const LOOPBACK_HOST = "127.0.0.1";
 const SHUTDOWN_WATCHDOG_MS = 20_000;
 const HTTP_CLOSE_GRACE_MS = 2_000;
-function openBooks(books) {
-    return books.list().flatMap((book) => {
-        const selected = books.get(book.bookId);
-        return selected.ok ? [selected.value] : [];
-    });
-}
-async function hydrateLibrary(books, options) {
-    await books.ready();
-    if (options.devFixtures) {
-        const { ensureDancingWorkflow } = await import("./dancing-workflow.js");
-        await ensureDancingWorkflow(books);
-    }
-    const imageSubjects = await import("./jobs/image-subjects.js");
-    const runNotebook = await import("./jobs/run-notebook.js");
-    for (const book of openBooks(books)) {
-        await book.engine.jobs.queue.reap();
-        await imageSubjects.reconcileImageSubjectJobFailures(book.engine, book.bookId);
-        await runNotebook.abortLeasedNotebookRunsOnStartup(book.engine);
-        await runNotebook.reconcileGenerationMediaOutputs(book.engine);
-        await runNotebook.reconcileQueuedNotebookCells(book.engine);
-    }
-    await queue.start();
-    for (const book of openBooks(books)) {
-        await runNotebook.reconcileAwaitingNotebookOutputs(book.engine, book.bookId);
-    }
-    await loadAndArmAll();
-    startPostMetricsScheduler();
-}
 async function shutdownServices(services) {
     stopScheduler();
     stopPostMetricsScheduler();
@@ -110,6 +84,7 @@ async function claimLibrary(server, libraryDir, port, url) {
 export async function startServer(options) {
     const libraryDir = bookManager.rootDir;
     await refuseRunningOwner(libraryDir);
+    getDefaultSecretStore().resolveOnePassword();
     registerAllJobHandlers(scopedEngine);
     const services = createApp({ ui: options.ui, uiDir: options.uiDir });
     const server = services.app.listen(options.port, LOOPBACK_HOST);

package/dist/tools/videobook-domain.js

@@ -6,7 +6,7 @@ import { z } from "zod";
 import { v7 as uuidv7 } from "uuid";
 import { ensureGenerateAudioOutput, normalizeLegacyExtractCells, } from "../notebook-cell-outputs.js";
 import { mutateNotebook, withNotebookMutation } from "../notebook-mutations.js";
-import { queue, stopBookQueue } from "../queue.js";
+import { jobIdForKey, queue, stopBookQueue } from "../queue.js";
 import { claimNotebookGenerationPlan, failQueuedNotebookCells, generationPlanClaimOutputCellId, notebookCellDefinitionFingerprint, notebookCellIsPaid, notebookCellIsRunnable, notebookCellMediaType, markNotebookCellQueued, prepareNotebookGeneration, } from "../jobs/run-notebook.js";
 import { buildGenerationInsertPlan, continueQueuedInsert, } from "./command-plan.js";
 import { assessToolCallEstimate, logUnavailableCostEstimate, } from "../jobs/cost-estimate-assessment.js";
@@ -271,6 +271,24 @@ function pinnedResultFromInput(input) {
         createdAt: input.created_at,
     };
 }
+function jobsByIdOrKey(engine, bookId, ids, keys) {
+    const byId = ids.map((id) => ({ id, jobKey: null }));
+    const byKey = keys.flatMap((jobKey) => {
+        const id = jobIdForKey(jobKey, bookId);
+        return id === null ? [] : [{ id, jobKey }];
+    });
+    return [...byId, ...byKey].flatMap(({ id, jobKey }) => {
+        const job = engine.jobs.queue.get(id);
+        return job ? [{ job, jobKey }] : [];
+    });
+}
+function selectJobs(engine, args) {
+    const ids = Array.isArray(args.job_ids) ? args.job_ids.map(Number) : null;
+    const keys = Array.isArray(args.job_keys) ? args.job_keys.map(String) : null;
+    if (!ids && !keys)
+        return engine.jobs.queue.list({ limit: Number(args.limit) || 200 }).map((job) => ({ job, jobKey: null }));
+    return jobsByIdOrKey(engine, String(args.book_id), ids ?? [], keys ?? []);
+}
 function json(value) {
     return { content: [{ type: "text", text: JSON.stringify(value) }] };
 }
@@ -1898,21 +1916,18 @@ export function videobookDomainDefs(engine, books) {
         },
         {
             name: "list_jobs",
-            description: "List persistent jobs for a book",
+            description: "List persistent jobs for a book. Filter by numeric job_ids, or by job_keys, the job_id strings other tools return while the server that queued them is running.",
             scope: "book",
             schema: {
                 ...bookSchema,
                 limit: z.number().int().positive().max(500).optional(),
                 job_ids: z.array(z.number().int().positive()).min(1).max(100).optional(),
+                job_keys: z.array(z.string().min(1).max(500)).min(1).max(100).optional(),
             },
             handler: async (args) => {
-                const jobs = Array.isArray(args.job_ids) ? args.job_ids.flatMap((id) => {
-                    const job = engine.jobs.queue.get(Number(id));
-                    return job ? [job] : [];
-                })
-                    : engine.jobs.queue.list({ limit: Number(args.limit) || 200 });
-                return json(await Promise.all(jobs.map(async (job) => ({
-                    ...job, providerTask: await jobProviderTaskStatus(engine, job),
+                const jobs = selectJobs(engine, args);
+                return json(await Promise.all(jobs.map(async ({ job, jobKey }) => ({
+                    ...job, ...(jobKey ? { jobKey } : {}), providerTask: await jobProviderTaskStatus(engine, job),
                 }))));
             },
         },

package/dist/ytdlp-binary.js

@@ -0,0 +1,65 @@
+import { accessSync, constants, statSync } from "node:fs";
+import { createRequire } from "node:module";
+import { delimiter, isAbsolute, join } from "node:path";
+import { FFMPEG } from "./ffmpeg-binary.js";
+const YTDLP_OVERRIDE = "VIDEOBOOK_YTDLP_PATH";
+const require = createRequire(import.meta.url);
+export const YTDLP_MISSING_MESSAGE = "yt-dlp is not installed. Put yt-dlp on PATH (for example brew install yt-dlp), set VIDEOBOOK_YTDLP_PATH, "
+    + "or reinstall with npm install -g videobook --allow-scripts=ffmpeg-static,ytdlp-nodejs,@dolthub/doltlite,usearch,onnxruntime-node,esbuild";
+function isExecutableFile(path) {
+    try {
+        if (!statSync(path).isFile())
+            return false;
+        accessSync(path, constants.X_OK);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+function executableNames(name, env) {
+    if (process.platform !== "win32")
+        return [name];
+    const extensions = (env.PATHEXT ?? ".EXE;.CMD;.BAT;.COM").split(";").filter(Boolean);
+    return [name, ...extensions.map((extension) => `${name}${extension.toLowerCase()}`)];
+}
+export function findOnPath(name, env = process.env) {
+    const directories = (env.PATH ?? "").split(delimiter).filter((directory) => isAbsolute(directory));
+    for (const directory of directories) {
+        const match = executableNames(name, env).map((candidate) => join(directory, candidate)).find(isExecutableFile);
+        if (match)
+            return match;
+    }
+    return null;
+}
+export function bundledYtdlp() {
+    try {
+        const { helpers } = require("ytdlp-nodejs");
+        const path = helpers.findYtdlpBinary();
+        return path && isExecutableFile(path) ? path : null;
+    }
+    catch {
+        return null;
+    }
+}
+export function resolveYtdlp(env = process.env, bundled = bundledYtdlp()) {
+    const override = env[YTDLP_OVERRIDE]?.trim();
+    if (override)
+        return { path: override, source: "VIDEOBOOK_YTDLP_PATH" };
+    if (bundled)
+        return { path: bundled, source: "bundled" };
+    const onPath = findOnPath("yt-dlp", env);
+    return onPath ? { path: onPath, source: "PATH" } : null;
+}
+function ffmpegForYtdlp(env) {
+    if (isAbsolute(FFMPEG))
+        return isExecutableFile(FFMPEG) ? FFMPEG : null;
+    return findOnPath(FFMPEG, env);
+}
+export function ytdlpRunOptions(env = process.env, bundled = bundledYtdlp()) {
+    const binary = resolveYtdlp(env, bundled);
+    if (!binary)
+        throw new Error(YTDLP_MISSING_MESSAGE);
+    const ffmpeg = ffmpegForYtdlp(env);
+    return { binaryPath: binary.path, ...(ffmpeg ? { ffmpegPath: ffmpeg } : {}) };
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "videobook",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "description": "Local-first MCP server and web UI for AI video creation: notebooks, generation, editing, rendering and publishing.",
   "keywords": [
     "mcp",
@@ -13,15 +13,7 @@
     "cli"
   ],
   "license": "MIT",
-  "homepage": "https://github.com/justintanner/videobook#readme",
-  "repository": {
-    "type": "git",
-    "url": "git+https://github.com/justintanner/videobook.git",
-    "directory": "server"
-  },
-  "bugs": {
-    "url": "https://github.com/justintanner/videobook/issues"
-  },
+  "homepage": "https://www.npmjs.com/package/videobook",
   "type": "module",
   "bin": {
     "videobook": "bin/videobook.js"
@@ -57,6 +49,7 @@
     "@apicity/x": "^0.16.1",
     "@apicity/xai": "^0.16.1",
     "@devmehq/open-graph-extractor": "^1.1.1",
+    "@ffprobe-installer/ffprobe": "2.1.2",
     "@modelcontextprotocol/client": "^2.2.0",
     "@modelcontextprotocol/node": "2.0.0",
     "@modelcontextprotocol/server": "^2.0.0",
@@ -68,7 +61,6 @@
     "express": "^5.2.1",
     "ffmpeg-static": "^5.3.0",
     "ffmpeggy": "^3.1.3",
-    "ffprobe-static": "^3.1.0",
     "heic-decode": "^2.1.0",
     "hono": "^4.12.31",
     "qrcode": "^1.5.4",