poddrill 0.0.0-stage → 0.1.0

package/dist/package-identity.js

@@ -0,0 +1,73 @@
+import { readFile } from "node:fs/promises";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { asRecord } from "./disk.js";
+const serverName = "poddrill";
+/** `package.json` one directory above the running module. */
+export async function readPackageIdentity(moduleUrl) {
+    const root = dirname(dirname(fileURLToPath(moduleUrl)));
+    const text = await readFile(join(root, "package.json"), "utf8");
+    const identity = parseManifest(root, text);
+    if (identity === undefined) {
+        throw new Error(`package.json at ${root} must name ${serverName}, pin a numeric version, and pin cactusNeedle`);
+    }
+    return identity;
+}
+function parsePackageVersion(text) {
+    if (!isPackageVersion(text)) {
+        return undefined;
+    }
+    return text;
+}
+function isPackageVersion(text) {
+    return /^\d+\.\d+\.\d+$/.test(text);
+}
+function parseManifest(root, text) {
+    const fields = readManifestFields(text);
+    if (fields === undefined) {
+        return undefined;
+    }
+    return identityFromFields(root, fields.name, fields.version, fields.cactusNeedle, fields.files);
+}
+function readManifestFields(text) {
+    let parsed;
+    try {
+        parsed = JSON.parse(text);
+    }
+    catch {
+        return undefined;
+    }
+    const record = asRecord(parsed);
+    if (record === undefined) {
+        return undefined;
+    }
+    const name = record["name"];
+    const version = record["version"];
+    const cactusNeedle = record["cactusNeedle"];
+    if (typeof name !== "string" || typeof version !== "string" || typeof cactusNeedle !== "string") {
+        return undefined;
+    }
+    const filesValue = record["files"];
+    if (filesValue === undefined) {
+        return { name, version, cactusNeedle, files: [] };
+    }
+    if (!Array.isArray(filesValue)) {
+        return undefined;
+    }
+    const files = [];
+    for (const entry of filesValue) {
+        if (typeof entry !== "string") {
+            return undefined;
+        }
+        files.push(entry);
+    }
+    return { name, version, cactusNeedle, files };
+}
+function identityFromFields(root, name, versionText, cactusNeedle, files) {
+    const version = parsePackageVersion(versionText);
+    if (name !== serverName || version === undefined || cactusNeedle.length === 0) {
+        return undefined;
+    }
+    const spec = `${serverName}@${version}`;
+    return { name: serverName, version, spec, cactusNeedle, root, files };
+}

package/dist/provider.js

@@ -0,0 +1,123 @@
+/**
+ * Hosts that run a transcript model. The episode module never imports this file.
+ * A new host is a new id in `providerIds` and a `WhistleHost` factory.
+ */
+import { openFleet } from "./fleet.js";
+import { fleetSize, modelForLanguage } from "./models.js";
+import { moonshineFailureMessage, spawnMoonshineWorker } from "./moonshine.js";
+import { spawnNeedleWorker } from "./needle.js";
+export const providerIds = ["localhost", "grok-bot", "cursor-cloud"];
+export function isProviderId(value) {
+    return providerIds.includes(value);
+}
+/**
+ * `undefined` when this model can run.
+ * A missing Moonshine weight path fails here and does not download. A present path runs the worker.
+ */
+export function transcriptRefusal(model, weights) {
+    switch (model.engine) {
+        case "whistle":
+            return undefined;
+        case "moonshine":
+            return weights[model.id] === undefined ? `${model.id} has no weights path` : undefined;
+        default: {
+            const leftover = model;
+            throw new Error(`unexpected model ${JSON.stringify(leftover)}`);
+        }
+    }
+}
+/**
+ * Mac runs one warm pool per model id. Nothing spawns until that id's first slice.
+ * Grok Bot's computer and Cursor's cloud computer are not wired yet.
+ */
+const factories = {
+    localhost: (config, whistleSpawn, moonshineSpawn) => openMacHost(config, whistleSpawn, moonshineSpawn),
+    "grok-bot": () => unwiredHost("grok-bot"),
+    "cursor-cloud": () => unwiredHost("cursor-cloud"),
+};
+/** Open the host selected at setup. Spawns nothing until the first slice of a model. */
+export function openWhistleCall(providerId, config, whistleSpawn = spawnNeedleWorker, moonshineSpawn = spawnMoonshineWorker) {
+    if (!Number.isInteger(config.fleetLimit) || config.fleetLimit < 1) {
+        throw new Error(`fleet limit must be a positive integer, got ${config.fleetLimit}`);
+    }
+    return factories[providerId](config, whistleSpawn, moonshineSpawn);
+}
+function openMacHost(config, whistleSpawn, moonshineSpawn) {
+    const pools = new Map();
+    const poolFor = (model) => {
+        const existing = pools.get(model.id);
+        if (existing !== undefined) {
+            return existing;
+        }
+        const pool = openFleet({
+            limit: fleetSize(model.id, config),
+            spawn: spawnFor(model, config.modelWeights, whistleSpawn, moonshineSpawn),
+            closedMessage: "whistle host is closed",
+        });
+        pools.set(model.id, pool);
+        return pool;
+    };
+    return {
+        transcribeSlice(job) {
+            const model = modelForLanguage(job.language);
+            if (model === undefined) {
+                return Promise.reject(new Error(`unknown language ${job.language}`));
+            }
+            const refusal = transcriptRefusal(model, config.modelWeights);
+            if (refusal !== undefined) {
+                return Promise.reject(new Error(refusal));
+            }
+            return poolFor(model).run({ wavPath: job.wavPath, language: job.language });
+        },
+        async close() {
+            await Promise.all([...pools.values()].map((pool) => pool.close()));
+        },
+        release() {
+            let clear = true;
+            for (const pool of pools.values()) {
+                if (!pool.release()) {
+                    clear = false;
+                }
+            }
+            return clear;
+        },
+    };
+}
+function spawnFor(model, weights, whistleSpawn, moonshineSpawn) {
+    switch (model.engine) {
+        case "whistle":
+            return whistleSpawn;
+        case "moonshine": {
+            const weightsDir = weights[model.id];
+            if (weightsDir === undefined) {
+                throw new Error(`${model.id} has no weights path`);
+            }
+            const spawn = moonshineSpawn(model, weightsDir);
+            return (events) => {
+                const link = spawn(events);
+                return {
+                    request: (query) => link.request({ wavPath: query.wavPath }).catch((error) => {
+                        const message = error instanceof Error ? error.message : String(error);
+                        return Promise.reject(new Error(moonshineFailureMessage(message)));
+                    }),
+                    close: () => link.close(),
+                };
+            };
+        }
+        default: {
+            const leftover = model;
+            throw new Error(`unexpected model ${JSON.stringify(leftover)}`);
+        }
+    }
+}
+function unwiredHost(providerId) {
+    return {
+        async transcribeSlice() {
+            throw new Error(`${providerId} provider transport is not wired`);
+        },
+        async close() { },
+        release() {
+            return true;
+        },
+    };
+}

package/dist/stretch.js

@@ -0,0 +1,46 @@
+/**
+ * The embedding unit, not a speaker rule. Cues join while the silence between them is at most
+ * `STRETCH_GAP_MS` and the joined span is at most `STRETCH_MAX_MS`. A joined span shorter than
+ * `STRETCH_MIN_MS` is too short for a stable vector and is dropped.
+ */
+export const STRETCH_GAP_MS = 500;
+export const STRETCH_MAX_MS = 14_000;
+export const STRETCH_MIN_MS = 4_000;
+/** Cue spans in file order, from the WebVTT that `projectSlices` renders. */
+export function readCueSpans(vtt) {
+    const spans = [];
+    for (const line of vtt.split("\n")) {
+        const match = /^(\d{2}):(\d{2}):(\d{2})\.(\d{3}) --> (\d{2}):(\d{2}):(\d{2})\.(\d{3})$/.exec(line);
+        if (match === null) {
+            continue;
+        }
+        const [, sh, sm, ss, sms, eh, em, es, ems] = match;
+        spans.push({ startMs: stampMs(sh, sm, ss, sms), endMs: stampMs(eh, em, es, ems) });
+    }
+    return spans;
+}
+export function planStretches(cues) {
+    const joined = [];
+    let open;
+    for (const cue of cues) {
+        if (open !== undefined &&
+            cue.startMs - open.endMs <= STRETCH_GAP_MS &&
+            cue.endMs - open.startMs <= STRETCH_MAX_MS) {
+            open = { startMs: open.startMs, endMs: Math.max(open.endMs, cue.endMs) };
+            continue;
+        }
+        if (open !== undefined) {
+            joined.push(open);
+        }
+        open = cue;
+    }
+    if (open !== undefined) {
+        joined.push(open);
+    }
+    return joined
+        .filter((span) => span.endMs - span.startMs >= STRETCH_MIN_MS)
+        .map((span, index) => ({ index, startMs: span.startMs, endMs: span.endMs }));
+}
+function stampMs(hours, minutes, seconds, millis) {
+    return Number(hours) * 3_600_000 + Number(minutes) * 60_000 + Number(seconds) * 1_000 + Number(millis);
+}

package/dist/summary.js

@@ -0,0 +1,258 @@
+import { randomBytes } from "node:crypto";
+import { parseDocument, stringify } from "yaml";
+import { asRecord } from "./disk.js";
+const SUMMARY_ID = /^sm_[0-9a-f]{16}$/;
+const POST_FIELDS = ["text", "cues", "startMs", "endMs"];
+const PUT_FIELDS = ["index", "text", "cues", "startMs", "endMs"];
+const FILE_FIELDS = ["version", "episodeId", "summaryId", "summary", "paragraphs"];
+const PARAGRAPH_FIELDS = ["index", "text", "cues", "startMs", "endMs"];
+const parseOptions = {
+    schema: "core",
+    uniqueKeys: true,
+    maxAliasCount: 0,
+};
+const dumpOptions = {
+    aliasDuplicateObjects: false,
+    lineWidth: 0,
+    schema: "core",
+};
+export function freshSummaryId() {
+    return asSummaryId(`sm_${randomBytes(8).toString("hex")}`);
+}
+export function parseSummaryId(value) {
+    return SUMMARY_ID.test(value) ? asSummaryId(value) : undefined;
+}
+export function requireSummaryText(summary) {
+    if (typeof summary !== "string" || summary === "") {
+        return { ok: false, message: "summary must be a non-empty string" };
+    }
+    return { ok: true, value: summary };
+}
+export function emptySummary(episodeId, summaryId) {
+    return {
+        version: 1,
+        episodeId,
+        summaryId,
+        summary: "",
+        paragraphs: [],
+    };
+}
+export function dumpSummary(document) {
+    const dumped = stringify({
+        version: document.version,
+        episodeId: document.episodeId,
+        summaryId: document.summaryId,
+        summary: document.summary,
+        paragraphs: document.paragraphs.map((paragraph) => ({
+            index: paragraph.index,
+            text: paragraph.text,
+            cues: paragraph.cues,
+            startMs: paragraph.startMs,
+            endMs: paragraph.endMs,
+        })),
+    }, dumpOptions);
+    return dumped;
+}
+export function parseSummaryFile(text, episodeId, summaryId) {
+    const doc = parseDocument(text, parseOptions);
+    // yaml 2 reports an unresolved tag such as !!js/function as a warning, not an error.
+    if (doc.errors.length > 0 || doc.warnings.length > 0) {
+        return undefined;
+    }
+    let value;
+    try {
+        value = doc.toJS({ maxAliasCount: 0 });
+    }
+    catch {
+        return undefined;
+    }
+    return narrowSummary(value, episodeId, summaryId);
+}
+export function inspectPostBody(body) {
+    const record = asRecord(body);
+    if (record === undefined) {
+        return { ok: false, message: "postParagraph expects { text, cues }" };
+    }
+    if (Object.hasOwn(record, "index")) {
+        return { ok: false, message: "postParagraph rejects index" };
+    }
+    return readCueSet(record, POST_FIELDS, "postParagraph expects { text, cues }");
+}
+export function inspectPutBody(body) {
+    const record = asRecord(body);
+    if (record === undefined || !Object.hasOwn(record, "index")) {
+        return { ok: false, message: "putParagraph requires index" };
+    }
+    const index = positiveIndex(record["index"]);
+    if (index === undefined) {
+        return { ok: false, message: "putParagraph requires index" };
+    }
+    const span = readCueSet(record, PUT_FIELDS, "putParagraph expects { index, text, cues }");
+    if (!span.ok) {
+        return span;
+    }
+    return { ok: true, value: { index, ...span.value } };
+}
+export function orderCueSet(ids, startMs, endMs, cues) {
+    if (ids.length === 0) {
+        return { ok: false, message: "cues must list at least one cue" };
+    }
+    const seen = new Set();
+    const indexes = [];
+    for (const id of ids) {
+        if (seen.has(id)) {
+            return { ok: false, message: "a cue is listed twice" };
+        }
+        seen.add(id);
+        const index = cues.findIndex((cue) => cue.id === id);
+        if (index < 0) {
+            return { ok: false, message: "a cue is not in the transcript" };
+        }
+        indexes.push(index);
+    }
+    indexes.sort((left, right) => left - right);
+    const ordered = [];
+    let first;
+    let last;
+    for (const index of indexes) {
+        const cue = cues[index];
+        if (cue === undefined) {
+            return { ok: false, message: "a cue is not in the transcript" };
+        }
+        ordered.push(cue.id);
+        if (first === undefined) {
+            first = cue;
+        }
+        last = cue;
+    }
+    if (first === undefined || last === undefined) {
+        return { ok: false, message: "cues must list at least one cue" };
+    }
+    if (startMs !== undefined && startMs !== first.startMs) {
+        return { ok: false, message: "startMs does not match the cue" };
+    }
+    if (endMs !== undefined && endMs !== last.endMs) {
+        return { ok: false, message: "endMs does not match the cue" };
+    }
+    return { ok: true, value: { cues: ordered, startMs: first.startMs, endMs: last.endMs } };
+}
+function narrowSummary(value, episodeId, summaryId) {
+    const record = asRecord(value);
+    if (record === undefined || !exactKeys(record, FILE_FIELDS)) {
+        return undefined;
+    }
+    const summary = record["summary"];
+    const paragraphs = record["paragraphs"];
+    if (record["version"] !== 1 ||
+        record["episodeId"] !== episodeId ||
+        record["summaryId"] !== summaryId ||
+        typeof summary !== "string" ||
+        !Array.isArray(paragraphs)) {
+        return undefined;
+    }
+    const parsed = [];
+    for (let position = 0; position < paragraphs.length; position += 1) {
+        const paragraph = narrowParagraph(paragraphs[position], position + 1);
+        if (paragraph === undefined) {
+            return undefined;
+        }
+        parsed.push(paragraph);
+    }
+    return {
+        version: 1,
+        episodeId,
+        summaryId,
+        summary,
+        paragraphs: parsed,
+    };
+}
+function narrowParagraph(value, index) {
+    const record = asRecord(value);
+    if (record === undefined || !exactKeys(record, PARAGRAPH_FIELDS)) {
+        return undefined;
+    }
+    const text = record["text"];
+    const cues = cueIdList(record["cues"]);
+    const startMs = record["startMs"];
+    const endMs = record["endMs"];
+    if (record["index"] !== index ||
+        typeof text !== "string" ||
+        text === "" ||
+        cues === undefined ||
+        typeof startMs !== "number" ||
+        !Number.isFinite(startMs) ||
+        typeof endMs !== "number" ||
+        !Number.isFinite(endMs)) {
+        return undefined;
+    }
+    return { index, text, cues, startMs, endMs };
+}
+function readCueSet(record, fields, message) {
+    for (const key of Object.keys(record)) {
+        if (key === "index") {
+            continue;
+        }
+        if (!fields.includes(key)) {
+            return { ok: false, message };
+        }
+    }
+    const text = nonEmpty(record["text"]);
+    const cues = cueIdList(record["cues"]);
+    if (text === undefined || cues === undefined) {
+        return { ok: false, message };
+    }
+    const startMs = optionalMs(record, "startMs");
+    if (!startMs.ok) {
+        return startMs;
+    }
+    const endMs = optionalMs(record, "endMs");
+    if (!endMs.ok) {
+        return endMs;
+    }
+    return {
+        ok: true,
+        value: { text, cues, startMs: startMs.value, endMs: endMs.value },
+    };
+}
+function cueIdList(value) {
+    if (!Array.isArray(value) || value.length === 0) {
+        return undefined;
+    }
+    const ids = [];
+    for (const item of value) {
+        if (typeof item !== "string" || item === "") {
+            return undefined;
+        }
+        ids.push(item);
+    }
+    return ids;
+}
+function optionalMs(record, key) {
+    if (!Object.hasOwn(record, key)) {
+        return { ok: true, value: undefined };
+    }
+    const value = record[key];
+    if (typeof value !== "number" || !Number.isFinite(value)) {
+        return { ok: false, message: `${key} must be a number` };
+    }
+    return { ok: true, value };
+}
+function nonEmpty(value) {
+    if (typeof value !== "string" || value === "") {
+        return undefined;
+    }
+    return value;
+}
+function positiveIndex(value) {
+    if (typeof value !== "number" || !Number.isInteger(value) || value < 1) {
+        return undefined;
+    }
+    return value;
+}
+function exactKeys(record, keys) {
+    const found = Object.keys(record);
+    return found.length === keys.length && keys.every((key) => Object.hasOwn(record, key));
+}
+function asSummaryId(value) {
+    return value;
+}

package/dist/transcribe.js

@@ -0,0 +1,53 @@
+/**
+ * Public transcribe entry.
+ * Planning and WebVTT live in episode.ts. Hosts live in provider.ts.
+ */
+import { planSlices, projectSlices, } from "./episode.js";
+import { isProviderId, openWhistleCall } from "./provider.js";
+/** The installing agent calls this once. Later transcribe calls read the same id. */
+export function selectProvider(providerId) {
+    if (!isProviderId(providerId)) {
+        throw new Error(`unknown provider ${providerId}`);
+    }
+    return { providerId };
+}
+/**
+ * Run Whistle on slices that were already cut, then project WebVTT.
+ * The host is the profile from setup, not an argument that changes the cutter.
+ */
+export async function transcribePrepared(input) {
+    const language = input.language ?? "en";
+    const host = openWhistleCall(input.profile.providerId, {
+        fleetLimit: input.fleetLimit,
+        modelFleet: input.modelFleet ?? {},
+        modelWeights: input.modelWeights ?? {},
+    });
+    try {
+        const records = [];
+        for (const slice of input.slices) {
+            const job = {
+                sliceIndex: slice.index,
+                startMs: slice.startMs,
+                wavPath: slice.wavPath,
+                language,
+            };
+            const whistle = await host.transcribeSlice(job);
+            records.push({
+                plan: {
+                    index: slice.index,
+                    startMs: slice.startMs,
+                    endMs: slice.endMs,
+                    keepStartMs: slice.keepStartMs,
+                    keepEndMs: slice.keepEndMs,
+                },
+                slicePath: slice.wavPath,
+                whistle,
+            });
+        }
+        return projectSlices(records);
+    }
+    finally {
+        await host.close();
+    }
+}
+export { planSlices, projectSlices };