poddrill 0.0.0-stage → 0.1.0

package/dist/worker.js

@@ -0,0 +1,215 @@
+/**
+ * One long-lived python3 process that answers one framed request at a time on stdin/stdout.
+ * Frame: 8-byte big-endian body length, then UTF-8 JSON.
+ * Shared request: { "wavPath": string }. A Whistle request also sends "language".
+ * Replies: { "kind": "ready", ... } once, then per request
+ *          { "kind": "result", "result": <value> } or { "kind": "error", "message": string }.
+ * Wire types stay in this file. The fleet sees parsed results and reason strings.
+ */
+import { spawn } from "node:child_process";
+import { asRecord } from "./disk.js";
+const MAX_FRAME_BYTES = 64 * 1024 * 1024;
+const STDERR_TAIL_BYTES = 8 * 1024;
+/**
+ * Python that every worker script starts with. Defines `send`, `recv`, and `serve(handle)`.
+ * Frames get a private copy of fd 1. Fd 1 itself points at stderr before any later import,
+ * so Python prints and native library writes cannot interleave with frames.
+ */
+export const PYTHON_FRAMES = [
+    "import json",
+    "import os",
+    "import struct",
+    "import sys",
+    "out = os.fdopen(os.dup(1), \"wb\")",
+    "os.dup2(2, 1)",
+    "sys.stdout = sys.stderr",
+    "inp = sys.stdin.buffer",
+    "def send(obj):",
+    "    body = json.dumps(obj).encode(\"utf-8\")",
+    "    out.write(struct.pack(\">Q\", len(body)) + body)",
+    "    out.flush()",
+    "def recv():",
+    "    head = inp.read(8)",
+    "    if len(head) < 8:",
+    "        return None",
+    "    (size,) = struct.unpack(\">Q\", head)",
+    "    return json.loads(inp.read(size).decode(\"utf-8\"))",
+    "def serve(handle):",
+    "    while True:",
+    "        message = recv()",
+    "        if message is None:",
+    "            break",
+    "        try:",
+    "            result = handle(message)",
+    "        except Exception as error:",
+    "            send({\"kind\": \"error\", \"message\": str(error)})",
+    "            continue",
+    "        send({\"kind\": \"result\", \"result\": result})",
+].join("\n");
+export function spawnFramedWorker(worker, events) {
+    const label = worker.label;
+    const child = spawn("python3", [...worker.args], { stdio: ["pipe", "pipe", "pipe"] });
+    let state = "booting";
+    let pending;
+    let stdout = Buffer.alloc(0);
+    let stderrTail = Buffer.alloc(0);
+    const exitOnce = (reason) => {
+        if (state === "exited") {
+            return;
+        }
+        state = "exited";
+        events.exit(reason);
+        const settled = pending;
+        pending = undefined;
+        settled?.reject(new Error(reason));
+    };
+    const fail = (reason) => {
+        child.kill("SIGKILL");
+        exitOnce(reason);
+    };
+    const onReply = (reply) => {
+        if (reply === undefined) {
+            fail(`${label} worker sent a malformed reply`);
+            return;
+        }
+        switch (reply.kind) {
+            case "ready":
+                if (state !== "booting") {
+                    fail(`${label} worker sent ready twice`);
+                    return;
+                }
+                state = "checking";
+                void worker.checkReady(reply.frame).then((refusal) => {
+                    if (state !== "checking") {
+                        return;
+                    }
+                    if (refusal !== undefined) {
+                        fail(refusal);
+                        return;
+                    }
+                    state = "ready";
+                    events.ready();
+                }, (error) => {
+                    fail(error instanceof Error ? error.message : String(error));
+                });
+                return;
+            case "result":
+            case "error": {
+                const settled = pending;
+                if (state !== "ready" || settled === undefined) {
+                    fail(`${label} worker replied without a request`);
+                    return;
+                }
+                pending = undefined;
+                if (reply.kind === "result") {
+                    settled.resolve(reply.result);
+                }
+                else {
+                    settled.reject(new Error(reply.message));
+                }
+                return;
+            }
+            default: {
+                const leftover = reply;
+                fail(`unexpected ${label} reply ${JSON.stringify(leftover)}`);
+            }
+        }
+    };
+    child.stdout.on("data", (chunk) => {
+        const frames = takeFrames(Buffer.concat([stdout, chunk]));
+        stdout = Buffer.from(frames.rest);
+        for (const body of frames.bodies) {
+            if (state === "exited") {
+                return;
+            }
+            onReply(parseReply(body, worker));
+        }
+        if (frames.oversized) {
+            fail(`${label} worker sent a frame over ${MAX_FRAME_BYTES} bytes`);
+        }
+    });
+    child.stderr.on("data", (chunk) => {
+        const joined = Buffer.concat([stderrTail, chunk]);
+        stderrTail = Buffer.from(joined.subarray(Math.max(0, joined.length - STDERR_TAIL_BYTES)));
+    });
+    // Writing to a worker that already died raises EPIPE here. The close event reports the death.
+    child.stdin.on("error", () => { });
+    child.on("error", (error) => {
+        exitOnce(error.message);
+    });
+    child.on("close", (code) => {
+        const tail = stderrTail.toString("utf8").trim();
+        exitOnce(tail === "" ? `${label} exited with code ${code === null ? "null" : String(code)}` : tail);
+    });
+    return {
+        request(frame) {
+            if (state !== "ready" || pending !== undefined) {
+                return Promise.reject(new Error(`${label} worker is not ready for a request`));
+            }
+            return new Promise((resolve, reject) => {
+                // A death mid-request reports this request's stderr, not chatter from earlier requests.
+                stderrTail = Buffer.alloc(0);
+                pending = { resolve, reject };
+                child.stdin.write(encodeFrame(frame));
+            });
+        },
+        close() {
+            // An idle worker reads EOF and exits. A busy one finishes, hits EPIPE on write, and exits.
+            child.stdin.end();
+            child.stdout.destroy();
+        },
+    };
+}
+function encodeFrame(body) {
+    const json = Buffer.from(JSON.stringify(body), "utf8");
+    const header = Buffer.alloc(8);
+    header.writeBigUInt64BE(BigInt(json.length));
+    return Buffer.concat([header, json]);
+}
+/** Same shape as `takeMessages` in owner.ts. `oversized` means the stream is corrupt. */
+function takeFrames(buffer) {
+    const bodies = [];
+    let rest = buffer;
+    while (rest.length >= 8) {
+        const size = rest.readBigUInt64BE(0);
+        if (size > BigInt(MAX_FRAME_BYTES)) {
+            return { bodies, rest: Buffer.alloc(0), oversized: true };
+        }
+        const end = 8 + Number(size);
+        if (rest.length < end) {
+            break;
+        }
+        const text = rest.subarray(8, end).toString("utf8");
+        rest = rest.subarray(end);
+        try {
+            bodies.push(JSON.parse(text));
+        }
+        catch {
+            bodies.push(undefined);
+        }
+    }
+    return { bodies, rest, oversized: false };
+}
+/** `result` goes through `parseResult`, so a malformed result is a per-request error, not a crash. */
+function parseReply(value, worker) {
+    const record = asRecord(value);
+    if (record === undefined) {
+        return undefined;
+    }
+    const kind = record["kind"];
+    if (kind === "ready") {
+        return { kind: "ready", frame: record };
+    }
+    if (kind === "error" && typeof record["message"] === "string") {
+        return { kind: "error", message: record["message"] };
+    }
+    if (kind === "result") {
+        try {
+            return { kind: "result", result: worker.parseResult(record["result"]) };
+        }
+        catch (error) {
+            return { kind: "error", message: error instanceof Error ? error.message : String(error) };
+        }
+    }
+    return undefined;
+}

package/dist/workspace.js

@@ -0,0 +1,306 @@
+/**
+ * Workspace layout and setup config.
+ * The only module that builds a path under the workspace. Writers and readers both call `episodeLayout`.
+ */
+import { randomBytes } from "node:crypto";
+import { mkdir, realpath, rename, writeFile } from "node:fs/promises";
+import { homedir } from "node:os";
+import { isAbsolute, join, resolve } from "node:path";
+import { asRecord, errorCode, pathExists, readJsonFile } from "./disk.js";
+import { isModelId, modelIds } from "./models.js";
+import { isProviderId, providerIds } from "./provider.js";
+/** Every artifact an episode lists, in the order it lists them. */
+export const artifactKinds = ["transcript", "embedding"];
+const CONFIG_FILE = "poddrill.json";
+const AUDIO_DIR = "audio";
+const EPISODES_DIR = "episodes";
+const ARTIFACT_OUTPUTS = {
+    transcript: "transcript.vtt",
+    embedding: "embedding.json",
+};
+/** `PODDRILL_WORKSPACE`, else `~/poddrill`, resolved with realpath. */
+export async function resolveWorkspace(env) {
+    const configured = env["PODDRILL_WORKSPACE"];
+    const candidate = resolve(configured === undefined || configured === "" ? join(homedir(), "poddrill") : configured);
+    let root;
+    try {
+        root = asWorkspaceRoot(await realpath(candidate));
+    }
+    catch (error) {
+        if (errorCode(error) === "ENOENT") {
+            throw new Error(`no poddrill workspace at ${candidate}. Run poddrill setup first.`);
+        }
+        throw error;
+    }
+    if (!(await pathExists(workspaceDirs(root).config))) {
+        throw new Error(`no ${CONFIG_FILE} in ${root}. Run poddrill setup first.`);
+    }
+    return root;
+}
+export async function readConfig(root) {
+    const path = workspaceDirs(root).config;
+    const config = parseConfig(await readJsonFile(path));
+    if (config === undefined) {
+        throw new Error([
+            `${path} must name a provider (${providerIds.join(", ")}), a positive integer queueLength,`,
+            "and, if present, a positive integer fleetLimit and embeddingLimit",
+        ].join(" "));
+    }
+    return config;
+}
+/**
+ * Create `audio/` and `episodes/`, and write `poddrill.json` atomically when it changes.
+ * `ownerAlive` comes from the caller's socket probe.
+ */
+export async function setupWorkspace(input) {
+    if (!isProviderId(input.provider)) {
+        throw new Error(`unknown provider ${input.provider}. Use one of ${providerIds.join(", ")}.`);
+    }
+    if (!isCount(input.queueLength)) {
+        throw new Error(`queue length must be a positive integer, got ${input.queueLength}`);
+    }
+    if (input.fleetLimit !== undefined && !isCount(input.fleetLimit)) {
+        throw new Error(`fleet limit must be a positive integer, got ${input.fleetLimit}`);
+    }
+    if (input.embeddingLimit !== undefined && !isCount(input.embeddingLimit)) {
+        throw new Error(`embedding limit must be a positive integer, got ${input.embeddingLimit}`);
+    }
+    const requested = resolve(input.root);
+    const current = parseConfig(await readJsonFile(join(requested, CONFIG_FILE)));
+    const modelFleet = input.modelFleet ?? current?.modelFleet ?? {};
+    const modelWeights = input.modelWeights ?? current?.modelWeights ?? {};
+    assertModelFleet(modelFleet);
+    assertModelWeights(modelWeights);
+    const config = {
+        provider: input.provider,
+        queueLength: input.queueLength,
+        fleetLimit: input.fleetLimit ?? current?.fleetLimit ?? 1,
+        modelFleet,
+        modelWeights,
+        embeddingLimit: input.embeddingLimit ?? current?.embeddingLimit ?? 1,
+    };
+    const unchanged = current !== undefined && sameConfig(current, config);
+    if (input.ownerAlive && current !== undefined && !unchanged) {
+        return {
+            kind: "rejected",
+            reason: "owner-running",
+            current,
+            message: [
+                `poddrill serve is running with provider ${current.provider},`,
+                `queue length ${current.queueLength}, fleet limit ${current.fleetLimit},`,
+                `and embedding limit ${current.embeddingLimit}.`,
+                "Stop it before changing the setup.",
+            ].join(" "),
+        };
+    }
+    await mkdir(join(requested, AUDIO_DIR), { recursive: true });
+    await mkdir(join(requested, EPISODES_DIR), { recursive: true });
+    const dirs = workspaceDirs(asWorkspaceRoot(await realpath(requested)));
+    if (!unchanged) {
+        const temp = `${dirs.config}.${randomBytes(4).toString("hex")}.tmp`;
+        await writeFile(temp, writtenConfig(config));
+        await rename(temp, dirs.config);
+    }
+    return {
+        kind: "ready",
+        report: {
+            dirs,
+            config,
+        },
+    };
+}
+export function workspaceDirs(root) {
+    return {
+        root,
+        config: join(root, CONFIG_FILE),
+        audioDir: join(root, AUDIO_DIR),
+        episodesDir: join(root, EPISODES_DIR),
+    };
+}
+export function episodeLayout(root, id) {
+    const dir = join(root, EPISODES_DIR, id);
+    const artifact = (kind) => {
+        const artifactDir = join(dir, kind);
+        return {
+            dir: artifactDir,
+            output: join(artifactDir, ARTIFACT_OUTPUTS[kind]),
+            failed: join(artifactDir, "failed.json"),
+            workDir: join(artifactDir, "work"),
+        };
+    };
+    return {
+        dir,
+        manifest: join(dir, "episode.json"),
+        groups: join(dir, "groups.json"),
+        labels: join(dir, "labels.json"),
+        summaries: join(dir, "summaries"),
+        audit: join(dir, "audit.jsonl"),
+        artifacts: { transcript: artifact("transcript"), embedding: artifact("embedding") },
+        source: (extension) => join(dir, `source${extension}`),
+        summaryFile: (summaryId) => join(dir, "summaries", `${summaryId}.yaml`),
+    };
+}
+/** `transcript/work/0007.wav` and `transcript/work/0007.whistle.json` for index 7. */
+export function sliceFiles(layout, index) {
+    const name = workName(index);
+    const workDir = layout.artifacts.transcript.workDir;
+    return {
+        wavPath: join(workDir, `${name}.wav`),
+        whistlePath: join(workDir, `${name}.whistle.json`),
+    };
+}
+/** `embedding/work/0007.wav` for stretch 7. */
+export function stretchWav(layout, index) {
+    return join(layout.artifacts.embedding.workDir, `${workName(index)}.wav`);
+}
+export function episodeIdFromDigest(sha256Hex) {
+    if (!/^[0-9a-f]{64}$/.test(sha256Hex)) {
+        throw new Error("an episode id needs a sha256 digest of 64 lowercase hex digits");
+    }
+    return asEpisodeId(`ep_${sha256Hex.slice(0, 16)}`);
+}
+/** Anything but `^ep_[0-9a-f]{16}$` is `undefined`, which the episode read reports as `unknown`. */
+export function parseEpisodeId(value) {
+    return /^ep_[0-9a-f]{16}$/.test(value) ? asEpisodeId(value) : undefined;
+}
+function workName(index) {
+    return String(index).padStart(4, "0");
+}
+function parseConfig(value) {
+    const record = asRecord(value);
+    if (record === undefined) {
+        return undefined;
+    }
+    const provider = record["provider"];
+    const queueLength = record["queueLength"];
+    if (typeof provider !== "string" || !isProviderId(provider)) {
+        return undefined;
+    }
+    if (!isCount(queueLength)) {
+        return undefined;
+    }
+    const fleetLimit = readFleetLimit(record);
+    const embeddingLimit = optionalCount(record["embeddingLimit"]);
+    if (fleetLimit === undefined || embeddingLimit === undefined) {
+        return undefined;
+    }
+    return {
+        provider,
+        queueLength,
+        fleetLimit,
+        modelFleet: readModelFleet(record["modelFleet"]),
+        modelWeights: readModelWeights(record["modelWeights"]),
+        embeddingLimit,
+    };
+}
+/**
+ * `fleetLimit` wins. An old file that only has `whistleLimit` uses that number.
+ * Neither field reads as 1. A present field that is not a positive integer does not parse.
+ */
+function readFleetLimit(record) {
+    if ("fleetLimit" in record) {
+        return optionalCount(record["fleetLimit"]);
+    }
+    if ("whistleLimit" in record) {
+        return optionalCount(record["whistleLimit"]);
+    }
+    return 1;
+}
+function readModelFleet(value) {
+    if (value === undefined) {
+        return {};
+    }
+    const record = asRecord(value);
+    if (record === undefined) {
+        throw new Error("modelFleet must be an object");
+    }
+    const fleet = {};
+    for (const [key, count] of Object.entries(record)) {
+        if (!isModelId(key)) {
+            throw new Error(`unknown model ${key}`);
+        }
+        if (!isCount(count)) {
+            throw new Error(`fleet size for ${key} must be a positive integer, got ${String(count)}`);
+        }
+        fleet[key] = count;
+    }
+    return fleet;
+}
+function readModelWeights(value) {
+    if (value === undefined) {
+        return {};
+    }
+    const record = asRecord(value);
+    if (record === undefined) {
+        throw new Error("modelWeights must be an object");
+    }
+    const weights = {};
+    for (const [key, path] of Object.entries(record)) {
+        if (!isModelId(key)) {
+            throw new Error(`unknown model ${key}`);
+        }
+        if (typeof path !== "string" || !isAbsolute(path)) {
+            throw new Error(`model weights path for ${key} must be absolute, got ${String(path)}`);
+        }
+        weights[key] = path;
+    }
+    return weights;
+}
+function assertModelFleet(fleet) {
+    for (const id of modelIds) {
+        const count = fleet[id];
+        if (count !== undefined && !isCount(count)) {
+            throw new Error(`fleet size for ${id} must be a positive integer, got ${String(count)}`);
+        }
+    }
+}
+function assertModelWeights(weights) {
+    for (const id of modelIds) {
+        const path = weights[id];
+        if (path !== undefined && !isAbsolute(path)) {
+            throw new Error(`model weights path for ${id} must be absolute, got ${path}`);
+        }
+    }
+}
+function writtenConfig(config) {
+    const body = {
+        provider: config.provider,
+        queueLength: config.queueLength,
+        fleetLimit: config.fleetLimit,
+    };
+    if (modelIds.some((id) => config.modelFleet[id] !== undefined)) {
+        body["modelFleet"] = config.modelFleet;
+    }
+    if (modelIds.some((id) => config.modelWeights[id] !== undefined)) {
+        body["modelWeights"] = config.modelWeights;
+    }
+    body["embeddingLimit"] = config.embeddingLimit;
+    return `${JSON.stringify(body, null, 2)}\n`;
+}
+/** Absent reads as 1. Present and not a positive integer is `undefined`. */
+function optionalCount(value) {
+    if (value === undefined) {
+        return 1;
+    }
+    return isCount(value) ? value : undefined;
+}
+function sameConfig(a, b) {
+    return (a.provider === b.provider &&
+        a.queueLength === b.queueLength &&
+        a.fleetLimit === b.fleetLimit &&
+        a.embeddingLimit === b.embeddingLimit &&
+        sameById(a.modelFleet, b.modelFleet) &&
+        sameById(a.modelWeights, b.modelWeights));
+}
+function sameById(left, right) {
+    return modelIds.every((id) => left[id] === right[id]);
+}
+function isCount(value) {
+    return typeof value === "number" && Number.isInteger(value) && value >= 1;
+}
+function asWorkspaceRoot(path) {
+    return path;
+}
+function asEpisodeId(value) {
+    return value;
+}

package/package.json

@@ -1,6 +1,33 @@
 {
   "name": "poddrill",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "0.1.0",
+  "description": "Transcribe episode audio into timed text, group the voices, and store a summary.",
+  "license": "MIT",
+  "author": "Jianyu Zhao",
+  "type": "module",
+  "bin": {
+    "poddrill": "dist/cli.js"
+  },
+  "files": [
+    "dist",
+    ".agents/skills/poddrill"
+  ],
+  "engines": {
+    "node": ">=22.18.0"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/JeromeZhao/PodDrill.git"
+  },
+  "cactusNeedle": "3.2.0",
+  "scripts": {
+    "build": "node scripts/build-js.mjs",
+    "test": "node --experimental-strip-types --test test/episode.test.ts test/fleet.test.ts test/idle-fleet.test.ts test/jobs.test.ts test/cues.test.ts test/summary.test.ts test/cli.test.ts test/audit.test.ts test/needle.test.ts test/publish.test.ts test/client.test.ts test/harness/run.test.ts"
+  },
+  "dependencies": {
+    "yaml": "2.9.1"
+  },
+  "devDependencies": {
+    "typescript": "5.9.3"
+  }
+}