poddrill 0.1.1 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/skills/poddrill/SKILL.md +34 -38
- package/.agents/skills/poddrill/references/fetch-audio.md +4 -4
- package/.agents/skills/poddrill/references/select-models.md +6 -4
- package/README.md +19 -4
- package/dist/cli.js +163 -10
- package/dist/client.js +222 -36
- package/dist/embed.js +11 -6
- package/dist/fleet.js +1 -1
- package/dist/moonshine.js +3 -2
- package/dist/needle.js +8 -1
- package/dist/node-version.js +59 -0
- package/dist/owner.js +73 -60
- package/dist/provider.js +27 -23
- package/dist/python-env.js +222 -0
- package/dist/weights.js +88 -0
- package/dist/worker.js +6 -3
- package/dist/workspace.js +37 -5
- package/package.json +6 -4
- package/requirements.txt +3 -0
package/dist/owner.js
CHANGED
|
@@ -10,7 +10,12 @@ import { embedWeightsPath, openEmbedHost } from "./embed.js";
|
|
|
10
10
|
import { readPackageIdentity } from "./package-identity.js";
|
|
11
11
|
import { JobQueue } from "./jobs.js";
|
|
12
12
|
import { openWhistleCall } from "./provider.js";
|
|
13
|
+
import { resolvePython } from "./python-env.js";
|
|
13
14
|
import { readConfig } from "./workspace.js";
|
|
15
|
+
export async function pythonForOwner(env, root) {
|
|
16
|
+
const config = await readConfig(root);
|
|
17
|
+
return resolvePython(env, config.python);
|
|
18
|
+
}
|
|
14
19
|
/**
|
|
15
20
|
* macOS rejects a unix socket path longer than 104 bytes.
|
|
16
21
|
* A workspace under /var/folders exceeds that, so the socket lives in the temp dir.
|
|
@@ -53,6 +58,7 @@ export async function serve(root, mcpOnStdio) {
|
|
|
53
58
|
const sock = socketPath(root);
|
|
54
59
|
const waiting = [];
|
|
55
60
|
let queue;
|
|
61
|
+
let requestStop = () => { };
|
|
56
62
|
// allowHalfOpen so a client FIN cannot drop the reply while start is still reading audio.
|
|
57
63
|
const server = createServer({ allowHalfOpen: true }, (connection) => {
|
|
58
64
|
const opened = queue;
|
|
@@ -62,7 +68,7 @@ export async function serve(root, mcpOnStdio) {
|
|
|
62
68
|
waiting.push(connection);
|
|
63
69
|
return;
|
|
64
70
|
}
|
|
65
|
-
accept(connection, opened, root);
|
|
71
|
+
accept(connection, opened, root, requestStop);
|
|
66
72
|
});
|
|
67
73
|
let claimed = false;
|
|
68
74
|
try {
|
|
@@ -77,6 +83,8 @@ export async function serve(root, mcpOnStdio) {
|
|
|
77
83
|
return;
|
|
78
84
|
}
|
|
79
85
|
const config = await readConfig(root);
|
|
86
|
+
const python = await pythonForOwner(process.env, root);
|
|
87
|
+
process.env["PODDRILL_PYTHON"] = python;
|
|
80
88
|
const hosts = {
|
|
81
89
|
whistle: openWhistleCall(config.provider, {
|
|
82
90
|
fleetLimit: config.fleetLimit,
|
|
@@ -113,13 +121,18 @@ export async function serve(root, mcpOnStdio) {
|
|
|
113
121
|
watcher.stop();
|
|
114
122
|
}
|
|
115
123
|
};
|
|
124
|
+
const gate = bindStop(server, sock, opened, hosts, stopIdle);
|
|
125
|
+
requestStop = gate.stop;
|
|
116
126
|
for (const connection of waiting.splice(0, waiting.length)) {
|
|
117
|
-
accept(connection, opened, root);
|
|
127
|
+
accept(connection, opened, root, requestStop);
|
|
118
128
|
}
|
|
119
129
|
const version = mcpOnStdio ? (await readPackageIdentity(import.meta.url)).version : undefined;
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
130
|
+
if (version !== undefined) {
|
|
131
|
+
void speakMcp(version, async (name, args) => callTool(opened, root, name, args)).then(() => {
|
|
132
|
+
gate.stop();
|
|
133
|
+
});
|
|
134
|
+
}
|
|
135
|
+
await gate.done;
|
|
123
136
|
}
|
|
124
137
|
export async function proxyMcp(root) {
|
|
125
138
|
const identity = await readPackageIdentity(import.meta.url);
|
|
@@ -157,7 +170,7 @@ export async function proxyMcp(root) {
|
|
|
157
170
|
return render({ kind: "error", message: `unknown tool ${name}` });
|
|
158
171
|
});
|
|
159
172
|
}
|
|
160
|
-
function accept(socket, queue, root) {
|
|
173
|
+
function accept(socket, queue, root, requestStop) {
|
|
161
174
|
let buffer = "";
|
|
162
175
|
socket.setEncoding("utf8");
|
|
163
176
|
socket.on("data", (chunk) => {
|
|
@@ -170,7 +183,11 @@ function accept(socket, queue, root) {
|
|
|
170
183
|
buffer = "";
|
|
171
184
|
void handleLine(line, queue, root)
|
|
172
185
|
.then((response) => {
|
|
173
|
-
socket.end(response)
|
|
186
|
+
socket.end(response.text, () => {
|
|
187
|
+
if (response.stop) {
|
|
188
|
+
requestStop();
|
|
189
|
+
}
|
|
190
|
+
});
|
|
174
191
|
})
|
|
175
192
|
.catch((error) => {
|
|
176
193
|
const message = error instanceof Error ? error.message : String(error);
|
|
@@ -179,49 +196,54 @@ function accept(socket, queue, root) {
|
|
|
179
196
|
});
|
|
180
197
|
socket.resume();
|
|
181
198
|
}
|
|
199
|
+
function answered(text, stop = false) {
|
|
200
|
+
return { text, stop };
|
|
201
|
+
}
|
|
182
202
|
async function handleLine(line, queue, root) {
|
|
183
203
|
let parsed;
|
|
184
204
|
try {
|
|
185
205
|
parsed = JSON.parse(line);
|
|
186
206
|
}
|
|
187
207
|
catch {
|
|
188
|
-
return render({ kind: "error", message: "malformed request" });
|
|
208
|
+
return answered(render({ kind: "error", message: "malformed request" }));
|
|
189
209
|
}
|
|
190
210
|
const request = parseRequest(parsed);
|
|
191
211
|
if (request === undefined) {
|
|
192
|
-
return render({ kind: "error", message: "unknown request" });
|
|
212
|
+
return answered(render({ kind: "error", message: "unknown request" }));
|
|
193
213
|
}
|
|
194
214
|
switch (request.op) {
|
|
195
215
|
case "ping":
|
|
196
|
-
return "{\"op\":\"pong\"}\n";
|
|
216
|
+
return answered("{\"op\":\"pong\"}\n");
|
|
217
|
+
case "stop":
|
|
218
|
+
return answered(render({ kind: "ready", state: "stopped" }), true);
|
|
197
219
|
case "audit":
|
|
198
|
-
return render(await readAudit(root, request.episodeId));
|
|
220
|
+
return answered(render(await readAudit(root, request.episodeId)));
|
|
199
221
|
case "start":
|
|
200
|
-
return finish(root, request, () => queue.start(request.audioPath, request.language ?? "en"));
|
|
222
|
+
return answered(await finish(root, request, () => queue.start(request.audioPath, request.language ?? "en")));
|
|
201
223
|
case "episode":
|
|
202
|
-
return finish(root, request, () => queue.episode(request.episodeId));
|
|
224
|
+
return answered(await finish(root, request, () => queue.episode(request.episodeId)));
|
|
203
225
|
case "cues":
|
|
204
|
-
return finish(root, request, () => queue.cues(request.episodeId, {
|
|
226
|
+
return answered(await finish(root, request, () => queue.cues(request.episodeId, {
|
|
205
227
|
...(request.limit === undefined ? {} : { limit: request.limit }),
|
|
206
228
|
...(request.after === undefined ? {} : { after: request.after }),
|
|
207
229
|
...(request.groupId === undefined ? {} : { groupId: request.groupId }),
|
|
208
230
|
...(request.hosts === undefined ? {} : { hosts: request.hosts }),
|
|
209
|
-
}));
|
|
231
|
+
})));
|
|
210
232
|
case "labelGroups":
|
|
211
|
-
return finish(root, request, () => queue.labelGroups(request.episodeId, request.groups));
|
|
233
|
+
return answered(await finish(root, request, () => queue.labelGroups(request.episodeId, request.groups)));
|
|
212
234
|
case "startSummary":
|
|
213
|
-
return finish(root, request, () => queue.startSummary(request.episodeId));
|
|
235
|
+
return answered(await finish(root, request, () => queue.startSummary(request.episodeId)));
|
|
214
236
|
case "setSummary":
|
|
215
|
-
return finish(root, request, () => queue.setSummary(request.episodeId, request.summaryId, request.summary));
|
|
237
|
+
return answered(await finish(root, request, () => queue.setSummary(request.episodeId, request.summaryId, request.summary)));
|
|
216
238
|
case "putParagraph":
|
|
217
|
-
return finish(root, request, () => queue.putParagraph(request.episodeId, request.summaryId, request.body));
|
|
239
|
+
return answered(await finish(root, request, () => queue.putParagraph(request.episodeId, request.summaryId, request.body)));
|
|
218
240
|
case "postParagraph":
|
|
219
|
-
return finish(root, request, () => queue.postParagraph(request.episodeId, request.summaryId, request.body));
|
|
241
|
+
return answered(await finish(root, request, () => queue.postParagraph(request.episodeId, request.summaryId, request.body)));
|
|
220
242
|
case "summary":
|
|
221
|
-
return finish(root, request, () => queue.summary(request.episodeId, request.summaryId));
|
|
243
|
+
return answered(await finish(root, request, () => queue.summary(request.episodeId, request.summaryId)));
|
|
222
244
|
default: {
|
|
223
245
|
const leftover = request;
|
|
224
|
-
return render({ kind: "error", message: `unexpected ${JSON.stringify(leftover)}` });
|
|
246
|
+
return answered(render({ kind: "error", message: `unexpected ${JSON.stringify(leftover)}` }));
|
|
225
247
|
}
|
|
226
248
|
}
|
|
227
249
|
}
|
|
@@ -251,6 +273,7 @@ function auditEpisodeId(request, response) {
|
|
|
251
273
|
switch (request.op) {
|
|
252
274
|
case "ping":
|
|
253
275
|
case "audit":
|
|
276
|
+
case "stop":
|
|
254
277
|
return undefined;
|
|
255
278
|
case "start":
|
|
256
279
|
return startedEpisodeId(response);
|
|
@@ -344,6 +367,9 @@ function parseRequest(value) {
|
|
|
344
367
|
if (op === "ping") {
|
|
345
368
|
return { op: "ping" };
|
|
346
369
|
}
|
|
370
|
+
if (op === "stop") {
|
|
371
|
+
return { op: "stop" };
|
|
372
|
+
}
|
|
347
373
|
if (op === "start" && typeof record["audioPath"] === "string") {
|
|
348
374
|
const language = record["language"];
|
|
349
375
|
if (language !== undefined && typeof language !== "string") {
|
|
@@ -474,24 +500,24 @@ function listen(server, sock) {
|
|
|
474
500
|
});
|
|
475
501
|
});
|
|
476
502
|
}
|
|
477
|
-
function
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
return;
|
|
483
|
-
}
|
|
484
|
-
stopping = true;
|
|
485
|
-
void closeOwner(server, sock, queue, hosts, stopIdle).then(() => {
|
|
486
|
-
resolve();
|
|
487
|
-
});
|
|
488
|
-
};
|
|
489
|
-
process.on("SIGTERM", stop);
|
|
490
|
-
process.on("SIGINT", stop);
|
|
491
|
-
if (mcp !== undefined) {
|
|
492
|
-
void mcp.then(stop);
|
|
493
|
-
}
|
|
503
|
+
function bindStop(server, sock, queue, hosts, stopIdle) {
|
|
504
|
+
let stopping = false;
|
|
505
|
+
let resolveDone = () => { };
|
|
506
|
+
const done = new Promise((resolve) => {
|
|
507
|
+
resolveDone = resolve;
|
|
494
508
|
});
|
|
509
|
+
const stop = () => {
|
|
510
|
+
if (stopping) {
|
|
511
|
+
return;
|
|
512
|
+
}
|
|
513
|
+
stopping = true;
|
|
514
|
+
void closeOwner(server, sock, queue, hosts, stopIdle).then(() => {
|
|
515
|
+
resolveDone();
|
|
516
|
+
});
|
|
517
|
+
};
|
|
518
|
+
process.on("SIGTERM", stop);
|
|
519
|
+
process.on("SIGINT", stop);
|
|
520
|
+
return { done, stop };
|
|
495
521
|
}
|
|
496
522
|
/** Stop the idle watchers, then the queue, so no lane takes new work. Then the hosts, so worker pipes close before exit. */
|
|
497
523
|
async function closeOwner(server, sock, queue, hosts, stopIdle) {
|
|
@@ -545,28 +571,17 @@ function takeMessages(buffer) {
|
|
|
545
571
|
const bodies = [];
|
|
546
572
|
let rest = buffer;
|
|
547
573
|
for (;;) {
|
|
548
|
-
const
|
|
549
|
-
if (
|
|
574
|
+
const newline = rest.indexOf("\n");
|
|
575
|
+
if (newline === -1) {
|
|
550
576
|
return { bodies, rest };
|
|
551
577
|
}
|
|
552
|
-
const
|
|
553
|
-
|
|
554
|
-
if (
|
|
555
|
-
|
|
556
|
-
}
|
|
557
|
-
const lengthText = match[1];
|
|
558
|
-
if (lengthText === undefined) {
|
|
559
|
-
return { bodies, rest: Buffer.alloc(0) };
|
|
560
|
-
}
|
|
561
|
-
const length = Number(lengthText);
|
|
562
|
-
const start = headerEnd + 4;
|
|
563
|
-
if (!Number.isInteger(length) || rest.length < start + length) {
|
|
564
|
-
return { bodies, rest };
|
|
578
|
+
const line = rest.subarray(0, newline).toString("utf8");
|
|
579
|
+
rest = Buffer.from(rest.subarray(newline + 1));
|
|
580
|
+
if (line.trim() === "") {
|
|
581
|
+
continue;
|
|
565
582
|
}
|
|
566
|
-
const body = rest.subarray(start, start + length).toString("utf8");
|
|
567
|
-
rest = Buffer.from(rest.subarray(start + length));
|
|
568
583
|
try {
|
|
569
|
-
bodies.push(JSON.parse(
|
|
584
|
+
bodies.push(JSON.parse(line));
|
|
570
585
|
}
|
|
571
586
|
catch {
|
|
572
587
|
bodies.push(undefined);
|
|
@@ -758,9 +773,7 @@ function toolList() {
|
|
|
758
773
|
];
|
|
759
774
|
}
|
|
760
775
|
function writeMessage(message) {
|
|
761
|
-
|
|
762
|
-
const header = Buffer.from(`Content-Length: ${body.length}\r\n\r\n`, "utf8");
|
|
763
|
-
process.stdout.write(Buffer.concat([header, body]));
|
|
776
|
+
process.stdout.write(`${JSON.stringify(message)}\n`);
|
|
764
777
|
}
|
|
765
778
|
function asRecord(value) {
|
|
766
779
|
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
package/dist/provider.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Hosts that run a transcript model. The episode module never imports this file.
|
|
3
|
-
* A new host is a new id in `providerIds` and a `
|
|
3
|
+
* A new host is a new id in `providerIds` and a `providerTransport` case.
|
|
4
|
+
* An `"unwired"` transport is refused by setup and by `openWhistleCall`.
|
|
4
5
|
*/
|
|
5
6
|
import { openFleet } from "./fleet.js";
|
|
6
7
|
import { fleetSize, modelForLanguage } from "./models.js";
|
|
@@ -26,23 +27,37 @@ export function transcriptRefusal(model, weights) {
|
|
|
26
27
|
}
|
|
27
28
|
}
|
|
28
29
|
}
|
|
29
|
-
/**
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
30
|
+
/** `localhost`, `grok-bot`, and `cursor-cloud` all run the local Python host. */
|
|
31
|
+
export function providerTransport(id) {
|
|
32
|
+
switch (id) {
|
|
33
|
+
case "localhost":
|
|
34
|
+
case "grok-bot":
|
|
35
|
+
case "cursor-cloud":
|
|
36
|
+
return "local";
|
|
37
|
+
default: {
|
|
38
|
+
const leftover = id;
|
|
39
|
+
throw new Error(`unexpected provider ${JSON.stringify(leftover)}`);
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
export function unwiredProviderMessage(id) {
|
|
44
|
+
return `${id} provider transport is not wired. poddrill setup refuses a provider that cannot transcribe.`;
|
|
45
|
+
}
|
|
46
|
+
/** Setup and `openWhistleCall` both call this before a worker can start. */
|
|
47
|
+
export function refuseUnwiredTransport(id, transport) {
|
|
48
|
+
if (transport === "unwired") {
|
|
49
|
+
throw new Error(unwiredProviderMessage(id));
|
|
50
|
+
}
|
|
51
|
+
}
|
|
38
52
|
/** Open the host selected at setup. Spawns nothing until the first slice of a model. */
|
|
39
53
|
export function openWhistleCall(providerId, config, whistleSpawn = spawnNeedleWorker, moonshineSpawn = spawnMoonshineWorker) {
|
|
54
|
+
refuseUnwiredTransport(providerId, providerTransport(providerId));
|
|
40
55
|
if (!Number.isInteger(config.fleetLimit) || config.fleetLimit < 1) {
|
|
41
56
|
throw new Error(`fleet limit must be a positive integer, got ${config.fleetLimit}`);
|
|
42
57
|
}
|
|
43
|
-
return
|
|
58
|
+
return openLocalHost(config, whistleSpawn, moonshineSpawn);
|
|
44
59
|
}
|
|
45
|
-
function
|
|
60
|
+
function openLocalHost(config, whistleSpawn, moonshineSpawn) {
|
|
46
61
|
const pools = new Map();
|
|
47
62
|
const poolFor = (model) => {
|
|
48
63
|
const existing = pools.get(model.id);
|
|
@@ -110,14 +125,3 @@ function spawnFor(model, weights, whistleSpawn, moonshineSpawn) {
|
|
|
110
125
|
}
|
|
111
126
|
}
|
|
112
127
|
}
|
|
113
|
-
function unwiredHost(providerId) {
|
|
114
|
-
return {
|
|
115
|
-
async transcribeSlice() {
|
|
116
|
-
throw new Error(`${providerId} provider transport is not wired`);
|
|
117
|
-
},
|
|
118
|
-
async close() { },
|
|
119
|
-
release() {
|
|
120
|
-
return true;
|
|
121
|
-
},
|
|
122
|
-
};
|
|
123
|
-
}
|
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The interpreter PodDrill spawns, and the venv that holds its pinned imports.
|
|
3
|
+
* `PODDRILL_PYTHON` wins, then `poddrill.json`, then `~/.cache/poddrill/venv`, then `python3`.
|
|
4
|
+
*/
|
|
5
|
+
import { spawn } from "node:child_process";
|
|
6
|
+
import { homedir } from "node:os";
|
|
7
|
+
import { isAbsolute, join } from "node:path";
|
|
8
|
+
import { errorCode, pathExists } from "./disk.js";
|
|
9
|
+
import { readPackageIdentity } from "./package-identity.js";
|
|
10
|
+
export const NUMPY_PIN = "2.5.4";
|
|
11
|
+
export const ONNXRUNTIME_PIN = "1.31.0";
|
|
12
|
+
export const INSTALL_PYTHON_FIX = "poddrill install-python";
|
|
13
|
+
/** `HOME` from the env when it is absolute, otherwise the process home. */
|
|
14
|
+
export function homeFromEnv(env) {
|
|
15
|
+
const home = env["HOME"];
|
|
16
|
+
if (home !== undefined && home !== "" && isAbsolute(home)) {
|
|
17
|
+
return home;
|
|
18
|
+
}
|
|
19
|
+
return homedir();
|
|
20
|
+
}
|
|
21
|
+
export function defaultVenvDir(env) {
|
|
22
|
+
return join(homeFromEnv(env), ".cache", "poddrill", "venv");
|
|
23
|
+
}
|
|
24
|
+
export function defaultVenvPython(env) {
|
|
25
|
+
return join(defaultVenvDir(env), "bin", "python");
|
|
26
|
+
}
|
|
27
|
+
/** Sync lookup for workers. The owner has already resolved a venv into `PODDRILL_PYTHON`. */
|
|
28
|
+
export function pythonFromEnv(env) {
|
|
29
|
+
const configured = env["PODDRILL_PYTHON"];
|
|
30
|
+
if (configured !== undefined && configured !== "") {
|
|
31
|
+
return configured;
|
|
32
|
+
}
|
|
33
|
+
return "python3";
|
|
34
|
+
}
|
|
35
|
+
export async function resolvePython(env, configured) {
|
|
36
|
+
const fromEnv = env["PODDRILL_PYTHON"];
|
|
37
|
+
if (fromEnv !== undefined && fromEnv !== "") {
|
|
38
|
+
return fromEnv;
|
|
39
|
+
}
|
|
40
|
+
if (configured !== undefined && configured !== "") {
|
|
41
|
+
return configured;
|
|
42
|
+
}
|
|
43
|
+
const venvPython = defaultVenvPython(env);
|
|
44
|
+
if (await pathExists(venvPython)) {
|
|
45
|
+
return venvPython;
|
|
46
|
+
}
|
|
47
|
+
return "python3";
|
|
48
|
+
}
|
|
49
|
+
export async function pythonPins(moduleUrl) {
|
|
50
|
+
const identity = await readPackageIdentity(moduleUrl);
|
|
51
|
+
return {
|
|
52
|
+
"cactus-needle": identity.cactusNeedle,
|
|
53
|
+
numpy: NUMPY_PIN,
|
|
54
|
+
onnxruntime: ONNXRUNTIME_PIN,
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
export function ytDlpFix(python) {
|
|
58
|
+
return `${shellWord(python)} -m pip install --upgrade yt-dlp`;
|
|
59
|
+
}
|
|
60
|
+
export function moonshineInstallLine() {
|
|
61
|
+
return 'Install moonshine-voice into the PodDrill Python: "$PODDRILL_PYTHON" -m pip install moonshine-voice';
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* Create `~/.cache/poddrill/venv` when the pins are missing and install them there.
|
|
65
|
+
* Uses `uv` when it is on PATH. Otherwise `python3 -m venv`. A second call with the pins present does nothing.
|
|
66
|
+
*/
|
|
67
|
+
export async function installPython(env, moduleUrl) {
|
|
68
|
+
const pins = await pythonPins(moduleUrl);
|
|
69
|
+
const python = defaultVenvPython(env);
|
|
70
|
+
if (await pinsMatch(python, pins, env)) {
|
|
71
|
+
return { python, created: false, installer: "cached", pins };
|
|
72
|
+
}
|
|
73
|
+
const spawnEnv = commandEnv(env);
|
|
74
|
+
const uv = await commandWorks("uv", ["--version"], spawnEnv);
|
|
75
|
+
if (uv) {
|
|
76
|
+
await installWithUv(python, defaultVenvDir(env), pins, spawnEnv);
|
|
77
|
+
await assertPins(python, pins, env);
|
|
78
|
+
return { python, created: true, installer: "uv", pins };
|
|
79
|
+
}
|
|
80
|
+
await installWithVenv(python, defaultVenvDir(env), pins, env, spawnEnv);
|
|
81
|
+
await assertPins(python, pins, env);
|
|
82
|
+
return { python, created: true, installer: "venv", pins };
|
|
83
|
+
}
|
|
84
|
+
export function chooseInstaller(uvAvailable) {
|
|
85
|
+
return uvAvailable ? "uv" : "venv";
|
|
86
|
+
}
|
|
87
|
+
async function installWithUv(python, dest, pins, spawnEnv) {
|
|
88
|
+
if (!(await pathExists(python))) {
|
|
89
|
+
const created = await runProcess("uv", ["venv", dest], spawnEnv);
|
|
90
|
+
if (created.code !== 0) {
|
|
91
|
+
throw new Error(`uv venv failed: ${tail(created.stderr)}`);
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
const installed = await runProcess("uv", ["pip", "install", "--python", python, ...pinArgs(pins)], spawnEnv);
|
|
95
|
+
if (installed.code !== 0) {
|
|
96
|
+
throw new Error(`uv pip install failed: ${tail(`${installed.stdout}\n${installed.stderr}`)}`);
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
async function installWithVenv(python, dest, pins, env, spawnEnv) {
|
|
100
|
+
if (!(await pathExists(python))) {
|
|
101
|
+
const base = venvBase(env, dest);
|
|
102
|
+
const created = await runProcess(base, ["-m", "venv", dest], spawnEnv);
|
|
103
|
+
if (created.code !== 0) {
|
|
104
|
+
throw new Error([
|
|
105
|
+
`python -m venv failed: ${tail(`${created.stdout}\n${created.stderr}`)}`,
|
|
106
|
+
"Install the python3-venv package, or install uv and run poddrill install-python again.",
|
|
107
|
+
].join(" "));
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
const installed = await runProcess(python, ["-m", "pip", "install", "--disable-pip-version-check", ...pinArgs(pins)], spawnEnv);
|
|
111
|
+
if (installed.code !== 0) {
|
|
112
|
+
throw new Error(`pip install failed: ${tail(`${installed.stdout}\n${installed.stderr}`)}`);
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
function venvBase(env, dest) {
|
|
116
|
+
const configured = pythonFromEnv(env);
|
|
117
|
+
if (configured === "python3" || configured.startsWith(`${dest}/`)) {
|
|
118
|
+
return "python3";
|
|
119
|
+
}
|
|
120
|
+
return configured;
|
|
121
|
+
}
|
|
122
|
+
function pinArgs(pins) {
|
|
123
|
+
return [
|
|
124
|
+
`cactus-needle==${pins["cactus-needle"]}`,
|
|
125
|
+
`numpy==${pins.numpy}`,
|
|
126
|
+
`onnxruntime==${pins.onnxruntime}`,
|
|
127
|
+
];
|
|
128
|
+
}
|
|
129
|
+
async function assertPins(python, pins, env) {
|
|
130
|
+
if (!(await pinsMatch(python, pins, env))) {
|
|
131
|
+
throw new Error(`PodDrill Python at ${python} does not import the pinned packages`);
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
async function pinsMatch(python, pins, env) {
|
|
135
|
+
if (!(await pathExists(python))) {
|
|
136
|
+
return false;
|
|
137
|
+
}
|
|
138
|
+
const result = await runProcess(python, ["-c", PIN_PROBE], commandEnv(env));
|
|
139
|
+
if (result.code !== 0) {
|
|
140
|
+
return false;
|
|
141
|
+
}
|
|
142
|
+
const found = new Map();
|
|
143
|
+
for (const line of result.stdout.split("\n")) {
|
|
144
|
+
const index = line.indexOf("==");
|
|
145
|
+
if (index <= 0) {
|
|
146
|
+
continue;
|
|
147
|
+
}
|
|
148
|
+
found.set(line.slice(0, index), line.slice(index + 2).trim());
|
|
149
|
+
}
|
|
150
|
+
return (found.get("cactus-needle") === pins["cactus-needle"] &&
|
|
151
|
+
found.get("numpy") === pins.numpy &&
|
|
152
|
+
found.get("onnxruntime") === pins.onnxruntime);
|
|
153
|
+
}
|
|
154
|
+
const PIN_PROBE = [
|
|
155
|
+
"import importlib.metadata as meta",
|
|
156
|
+
"for name in ('cactus-needle', 'numpy', 'onnxruntime'):",
|
|
157
|
+
" print(name + '==' + meta.version(name))",
|
|
158
|
+
].join("\n");
|
|
159
|
+
function commandEnv(env) {
|
|
160
|
+
const copy = {};
|
|
161
|
+
for (const [key, value] of Object.entries(env)) {
|
|
162
|
+
if (value !== undefined) {
|
|
163
|
+
copy[key] = value;
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
if (copy["PATH"] === undefined && process.env["PATH"] !== undefined) {
|
|
167
|
+
copy["PATH"] = process.env["PATH"];
|
|
168
|
+
}
|
|
169
|
+
return copy;
|
|
170
|
+
}
|
|
171
|
+
async function commandWorks(command, args, env) {
|
|
172
|
+
const result = await runProcess(command, args, env);
|
|
173
|
+
return result.code === 0;
|
|
174
|
+
}
|
|
175
|
+
function runProcess(command, args, env) {
|
|
176
|
+
return new Promise((resolve, reject) => {
|
|
177
|
+
const child = spawn(command, [...args], { env, stdio: ["ignore", "pipe", "pipe"] });
|
|
178
|
+
const out = [];
|
|
179
|
+
const err = [];
|
|
180
|
+
let settled = false;
|
|
181
|
+
const finish = (result) => {
|
|
182
|
+
if (settled) {
|
|
183
|
+
return;
|
|
184
|
+
}
|
|
185
|
+
settled = true;
|
|
186
|
+
resolve(result);
|
|
187
|
+
};
|
|
188
|
+
child.stdout.on("data", (chunk) => {
|
|
189
|
+
out.push(chunk);
|
|
190
|
+
});
|
|
191
|
+
child.stderr.on("data", (chunk) => {
|
|
192
|
+
err.push(chunk);
|
|
193
|
+
});
|
|
194
|
+
child.on("error", (error) => {
|
|
195
|
+
if (errorCode(error) === "ENOENT") {
|
|
196
|
+
finish({ code: null, stdout: "", stderr: error.message });
|
|
197
|
+
return;
|
|
198
|
+
}
|
|
199
|
+
reject(error);
|
|
200
|
+
});
|
|
201
|
+
child.on("close", (code) => {
|
|
202
|
+
finish({
|
|
203
|
+
code,
|
|
204
|
+
stdout: Buffer.concat(out).toString("utf8"),
|
|
205
|
+
stderr: Buffer.concat(err).toString("utf8"),
|
|
206
|
+
});
|
|
207
|
+
});
|
|
208
|
+
});
|
|
209
|
+
}
|
|
210
|
+
function tail(text) {
|
|
211
|
+
const trimmed = text.trim();
|
|
212
|
+
if (trimmed.length <= 1500) {
|
|
213
|
+
return trimmed === "" ? "no output" : trimmed;
|
|
214
|
+
}
|
|
215
|
+
return trimmed.slice(-1500);
|
|
216
|
+
}
|
|
217
|
+
function shellWord(value) {
|
|
218
|
+
if (/^[A-Za-z0-9_./:-]+$/.test(value)) {
|
|
219
|
+
return value;
|
|
220
|
+
}
|
|
221
|
+
return `"${value.replaceAll("\"", "\\\"")}"`;
|
|
222
|
+
}
|
package/dist/weights.js
ADDED
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pinned WeSpeaker ECAPA-TDNN512 weights.
|
|
3
|
+
* The model card has no license field. WeSpeaker docs say pretrained models follow
|
|
4
|
+
* the dataset license, and VoxCeleb is CC BY 4.0, so attribution stays with the file.
|
|
5
|
+
*/
|
|
6
|
+
import { createHash } from "node:crypto";
|
|
7
|
+
import { createReadStream, createWriteStream } from "node:fs";
|
|
8
|
+
import { mkdir, rename, rm, stat } from "node:fs/promises";
|
|
9
|
+
import { dirname } from "node:path";
|
|
10
|
+
import { Readable } from "node:stream";
|
|
11
|
+
import { pipeline } from "node:stream/promises";
|
|
12
|
+
import { errorCode } from "./disk.js";
|
|
13
|
+
export const ECAPA_FILE = "voxceleb_ECAPA512.onnx";
|
|
14
|
+
export const ECAPA_URL = "https://huggingface.co/Wespeaker/wespeaker-voxceleb-ecapa-tdnn512/resolve/89426e111abc9ffcc40279baf808e087693c6e9b/voxceleb_ECAPA512.onnx";
|
|
15
|
+
export const ECAPA_BYTES = 24_861_931;
|
|
16
|
+
export const ECAPA_SHA256 = "f901ae3571912bad4e11e157744cb8b7d77a0b46923b83c410bdbb6c77d69736";
|
|
17
|
+
export const ECAPA_LICENSE = "CC BY 4.0";
|
|
18
|
+
export const ECAPA_ATTRIBUTION = "WeSpeaker ECAPA-TDNN512 pretrained on VoxCeleb. VoxCeleb is used under CC BY 4.0.";
|
|
19
|
+
export const FETCH_WEIGHTS_FIX = "poddrill fetch-weights";
|
|
20
|
+
export const ecapaSpec = { bytes: ECAPA_BYTES, sha256: ECAPA_SHA256 };
|
|
21
|
+
export async function digestStatus(path, spec) {
|
|
22
|
+
let info;
|
|
23
|
+
try {
|
|
24
|
+
info = await stat(path);
|
|
25
|
+
}
|
|
26
|
+
catch (error) {
|
|
27
|
+
if (errorCode(error) === "ENOENT") {
|
|
28
|
+
return "missing";
|
|
29
|
+
}
|
|
30
|
+
throw error;
|
|
31
|
+
}
|
|
32
|
+
if (!info.isFile() || info.size !== spec.bytes) {
|
|
33
|
+
return "mismatch";
|
|
34
|
+
}
|
|
35
|
+
const sha256 = await sha256File(path);
|
|
36
|
+
return sha256 === spec.sha256 ? "match" : "mismatch";
|
|
37
|
+
}
|
|
38
|
+
export async function sha256File(path) {
|
|
39
|
+
const hash = createHash("sha256");
|
|
40
|
+
const stream = createReadStream(path);
|
|
41
|
+
for await (const chunk of stream) {
|
|
42
|
+
hash.update(chunk);
|
|
43
|
+
}
|
|
44
|
+
return hash.digest("hex");
|
|
45
|
+
}
|
|
46
|
+
/** Download the pinned file when the destination is missing or does not match. Idempotent. */
|
|
47
|
+
export async function fetchEcapa(destination) {
|
|
48
|
+
const status = await digestStatus(destination, ecapaSpec);
|
|
49
|
+
if (status === "match") {
|
|
50
|
+
return weightsResult(destination, false);
|
|
51
|
+
}
|
|
52
|
+
await mkdir(dirname(destination), { recursive: true });
|
|
53
|
+
const partial = `${destination}.partial`;
|
|
54
|
+
try {
|
|
55
|
+
const response = await fetch(ECAPA_URL, { headers: { "user-agent": "poddrill" } });
|
|
56
|
+
if (!response.ok) {
|
|
57
|
+
throw new Error(`ECAPA download failed with HTTP ${response.status}`);
|
|
58
|
+
}
|
|
59
|
+
if (response.body === null) {
|
|
60
|
+
throw new Error("ECAPA download failed with an empty body");
|
|
61
|
+
}
|
|
62
|
+
await pipeline(Readable.fromWeb(response.body), createWriteStream(partial));
|
|
63
|
+
const written = await stat(partial);
|
|
64
|
+
if (written.size !== ECAPA_BYTES) {
|
|
65
|
+
throw new Error(`ECAPA download expected ${ECAPA_BYTES} bytes, got ${written.size}`);
|
|
66
|
+
}
|
|
67
|
+
const sha256 = await sha256File(partial);
|
|
68
|
+
if (sha256 !== ECAPA_SHA256) {
|
|
69
|
+
throw new Error(`ECAPA download sha256 ${sha256} does not match ${ECAPA_SHA256}`);
|
|
70
|
+
}
|
|
71
|
+
await rename(partial, destination);
|
|
72
|
+
}
|
|
73
|
+
catch (error) {
|
|
74
|
+
await rm(partial, { force: true });
|
|
75
|
+
throw error;
|
|
76
|
+
}
|
|
77
|
+
return weightsResult(destination, true);
|
|
78
|
+
}
|
|
79
|
+
function weightsResult(path, created) {
|
|
80
|
+
return {
|
|
81
|
+
path,
|
|
82
|
+
created,
|
|
83
|
+
bytes: ECAPA_BYTES,
|
|
84
|
+
sha256: ECAPA_SHA256,
|
|
85
|
+
license: ECAPA_LICENSE,
|
|
86
|
+
attribution: ECAPA_ATTRIBUTION,
|
|
87
|
+
};
|
|
88
|
+
}
|
package/dist/worker.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* One long-lived
|
|
2
|
+
* One long-lived Python process that answers one framed request at a time on stdin/stdout.
|
|
3
3
|
* Frame: 8-byte big-endian body length, then UTF-8 JSON.
|
|
4
4
|
* Shared request: { "wavPath": string }. A Whistle request also sends "language".
|
|
5
5
|
* Replies: { "kind": "ready", ... } once, then per request
|
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
*/
|
|
9
9
|
import { spawn } from "node:child_process";
|
|
10
10
|
import { asRecord } from "./disk.js";
|
|
11
|
+
import { pythonFromEnv } from "./python-env.js";
|
|
11
12
|
const MAX_FRAME_BYTES = 64 * 1024 * 1024;
|
|
12
13
|
const STDERR_TAIL_BYTES = 8 * 1024;
|
|
13
14
|
/**
|
|
@@ -48,7 +49,8 @@ export const PYTHON_FRAMES = [
|
|
|
48
49
|
].join("\n");
|
|
49
50
|
export function spawnFramedWorker(worker, events) {
|
|
50
51
|
const label = worker.label;
|
|
51
|
-
const
|
|
52
|
+
const command = worker.command ?? pythonFromEnv(process.env);
|
|
53
|
+
const child = spawn(command, [...worker.args], { stdio: ["pipe", "pipe", "pipe"] });
|
|
52
54
|
let state = "booting";
|
|
53
55
|
let pending;
|
|
54
56
|
let stdout = Buffer.alloc(0);
|
|
@@ -139,7 +141,8 @@ export function spawnFramedWorker(worker, events) {
|
|
|
139
141
|
});
|
|
140
142
|
child.on("close", (code) => {
|
|
141
143
|
const tail = stderrTail.toString("utf8").trim();
|
|
142
|
-
|
|
144
|
+
const raw = tail === "" ? `${label} exited with code ${code === null ? "null" : String(code)}` : tail;
|
|
145
|
+
exitOnce(worker.explainExit === undefined ? raw : worker.explainExit(raw));
|
|
143
146
|
});
|
|
144
147
|
return {
|
|
145
148
|
request(frame) {
|