pi-ui-extend 1.0.42 → 1.0.45
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +41 -12
- package/dist/app/app.d.ts +10 -0
- package/dist/app/app.js +132 -8
- package/dist/app/cli/install.js +2 -2
- package/dist/app/commands/command-controller.d.ts +4 -0
- package/dist/app/commands/command-controller.js +11 -0
- package/dist/app/commands/command-git-actions.d.ts +25 -0
- package/dist/app/commands/command-git-actions.js +381 -0
- package/dist/app/commands/command-host.d.ts +3 -0
- package/dist/app/commands/command-host.js +5 -2
- package/dist/app/commands/command-model-actions.d.ts +1 -0
- package/dist/app/commands/command-model-actions.js +51 -34
- package/dist/app/commands/command-navigation-actions.d.ts +1 -0
- package/dist/app/commands/command-navigation-actions.js +16 -3
- package/dist/app/commands/command-registry.d.ts +2 -0
- package/dist/app/commands/command-registry.js +14 -0
- package/dist/app/commands/reload-context-inventory.js +12 -1
- package/dist/app/input/autocomplete-controller.js +61 -34
- package/dist/app/input/input-action-controller.d.ts +2 -0
- package/dist/app/input/input-action-controller.js +8 -1
- package/dist/app/input/input-controller.d.ts +2 -1
- package/dist/app/input/input-controller.js +7 -2
- package/dist/app/input/prompt-enhancer-controller.js +53 -35
- package/dist/app/input/voice-controller.d.ts +41 -45
- package/dist/app/input/voice-controller.js +351 -419
- package/dist/app/model/model-usage-controller.js +17 -7
- package/dist/app/model/model-usage-status.d.ts +4 -1
- package/dist/app/model/model-usage-status.js +270 -46
- package/dist/app/popup/menu-items-controller.d.ts +13 -3
- package/dist/app/popup/menu-items-controller.js +37 -21
- package/dist/app/popup/popup-action-controller.d.ts +12 -2
- package/dist/app/popup/popup-action-controller.js +77 -24
- package/dist/app/popup/popup-menu-controller.d.ts +36 -14
- package/dist/app/popup/popup-menu-controller.js +239 -69
- package/dist/app/rendering/dcp-stats.d.ts +9 -4
- package/dist/app/rendering/dcp-stats.js +40 -425
- package/dist/app/rendering/popup-menu-renderer.d.ts +3 -5
- package/dist/app/rendering/popup-menu-renderer.js +42 -37
- package/dist/app/rendering/render-controller.js +23 -2
- package/dist/app/rendering/status-line-renderer.d.ts +5 -0
- package/dist/app/rendering/status-line-renderer.js +41 -5
- package/dist/app/rendering/tab-line-renderer.js +26 -20
- package/dist/app/runtime.d.ts +12 -1
- package/dist/app/runtime.js +120 -12
- package/dist/app/screen/mouse-controller.d.ts +2 -0
- package/dist/app/screen/mouse-controller.js +17 -7
- package/dist/app/screen/status-controller.d.ts +4 -0
- package/dist/app/screen/status-controller.js +5 -0
- package/dist/app/session/lazy-session-manager.js +34 -0
- package/dist/app/session/session-event-controller.d.ts +1 -0
- package/dist/app/session/session-event-controller.js +10 -1
- package/dist/app/session/session-history.d.ts +1 -0
- package/dist/app/session/session-history.js +12 -1
- package/dist/app/session/session-lifecycle-controller.d.ts +6 -1
- package/dist/app/session/session-lifecycle-controller.js +10 -1
- package/dist/app/session/tabs-controller.d.ts +25 -1
- package/dist/app/session/tabs-controller.js +442 -153
- package/dist/app/subagents/subagents-widget-controller.d.ts +1 -0
- package/dist/app/subagents/subagents-widget-controller.js +6 -0
- package/dist/app/types.d.ts +11 -1
- package/dist/app/workspace/workspace-actions-controller.js +10 -2
- package/dist/app/workspace/workspace-undo.d.ts +1 -0
- package/dist/app/workspace/workspace-undo.js +1 -0
- package/dist/bundled-extensions/question/index.js +9 -1
- package/dist/bundled-extensions/question/remote.d.ts +4 -0
- package/dist/bundled-extensions/question/remote.js +33 -0
- package/dist/bundled-extensions/telegram-connector/bot.d.ts +43 -0
- package/dist/bundled-extensions/telegram-connector/bot.js +166 -0
- package/dist/bundled-extensions/telegram-connector/config.d.ts +8 -0
- package/dist/bundled-extensions/telegram-connector/config.js +87 -0
- package/dist/bundled-extensions/telegram-connector/coordinator.d.ts +66 -0
- package/dist/bundled-extensions/telegram-connector/coordinator.js +413 -0
- package/dist/bundled-extensions/telegram-connector/index.d.ts +3 -0
- package/dist/bundled-extensions/telegram-connector/index.js +195 -0
- package/dist/bundled-extensions/terminal-bell/index.d.ts +0 -8
- package/dist/bundled-extensions/terminal-bell/index.js +0 -76
- package/dist/bundled-extensions/workspace-undo/index.d.ts +26 -0
- package/dist/bundled-extensions/workspace-undo/index.js +191 -0
- package/dist/config.d.ts +21 -2
- package/dist/config.js +159 -32
- package/dist/default-pix-config.js +20 -10
- package/dist/schemas/pi-tools-suite-schema.d.ts +8 -4
- package/dist/schemas/pi-tools-suite-schema.js +11 -5
- package/dist/schemas/pix-schema.d.ts +12 -2
- package/dist/schemas/pix-schema.js +37 -9
- package/dist/schemas/tasks-schema.js +1 -1
- package/docs/concurrency.md +3 -1
- package/docs/desktop-mvp.md +10 -5
- package/docs/desktop-task-manager.md +81 -28
- package/docs/release.md +30 -6
- package/external/pi-tools-suite/README.md +165 -39
- package/external/pi-tools-suite/docs/session-recovery.md +7 -0
- package/external/pi-tools-suite/docs/subagent-model-pools.md +12 -5
- package/external/pi-tools-suite/docs/ui-qa-subagent.md +440 -0
- package/external/pi-tools-suite/package.json +6 -1
- package/external/pi-tools-suite/src/antigravity-auth/auth-store.ts +2 -1
- package/external/pi-tools-suite/src/antigravity-auth/constants.ts +9 -3
- package/external/pi-tools-suite/src/antigravity-auth/headers.ts +41 -3
- package/external/pi-tools-suite/src/antigravity-auth/models.ts +94 -21
- package/external/pi-tools-suite/src/antigravity-auth/oauth.ts +5 -17
- package/external/pi-tools-suite/src/antigravity-auth/payload.ts +72 -7
- package/external/pi-tools-suite/src/antigravity-auth/stream.ts +13 -1
- package/external/pi-tools-suite/src/async-subagents/agents/implement.md +1 -1
- package/external/pi-tools-suite/src/async-subagents/agents/ui-qa/backends/browser.mjs +346 -0
- package/external/pi-tools-suite/src/async-subagents/agents/ui-qa/backends/desktop.mjs +1038 -0
- package/external/pi-tools-suite/src/async-subagents/agents/ui-qa/backends/tui.mjs +759 -0
- package/external/pi-tools-suite/src/async-subagents/agents/{browser-qa → ui-qa/browser}/scripts/browser-qa-runner.mjs +32 -23
- package/external/pi-tools-suite/src/async-subagents/agents/ui-qa/drivers/chrome-devtools/chrome-devtools-provider.mjs +895 -0
- package/external/pi-tools-suite/src/async-subagents/agents/ui-qa/drivers/linux/linux-atspi.py +494 -0
- package/external/pi-tools-suite/src/async-subagents/agents/ui-qa/drivers/macos/macos-accessibility.swift +1085 -0
- package/external/pi-tools-suite/src/async-subagents/agents/ui-qa/drivers/native-terminal/bridge-client.mjs +50 -0
- package/external/pi-tools-suite/src/async-subagents/agents/ui-qa/drivers/native-terminal/native-terminal-host.mjs +501 -0
- package/external/pi-tools-suite/src/async-subagents/agents/ui-qa/drivers/windows/windows-uia.ps1 +454 -0
- package/external/pi-tools-suite/src/async-subagents/agents/ui-qa/guides/browser-auth.md +114 -0
- package/external/pi-tools-suite/src/async-subagents/agents/ui-qa/guides/browser-chrome-devtools.md +91 -0
- package/external/pi-tools-suite/src/async-subagents/agents/ui-qa/guides/browser-playwright.md +145 -0
- package/external/pi-tools-suite/src/async-subagents/agents/ui-qa/guides/browser.md +82 -0
- package/external/pi-tools-suite/src/async-subagents/agents/ui-qa/guides/desktop-linux.md +26 -0
- package/external/pi-tools-suite/src/async-subagents/agents/ui-qa/guides/desktop-macos.md +29 -0
- package/external/pi-tools-suite/src/async-subagents/agents/ui-qa/guides/desktop-windows.md +23 -0
- package/external/pi-tools-suite/src/async-subagents/agents/ui-qa/guides/desktop.md +54 -0
- package/external/pi-tools-suite/src/async-subagents/agents/ui-qa/guides/tui-native-terminal.md +59 -0
- package/external/pi-tools-suite/src/async-subagents/agents/ui-qa/guides/tui-pty.md +37 -0
- package/external/pi-tools-suite/src/async-subagents/agents/ui-qa/guides/tui.md +60 -0
- package/external/pi-tools-suite/src/async-subagents/agents/ui-qa/scripts/ui-qa-runner.mjs +535 -0
- package/external/pi-tools-suite/src/async-subagents/agents/ui-qa.md +75 -0
- package/external/pi-tools-suite/src/async-subagents/commands.ts +1 -1
- package/external/pi-tools-suite/src/async-subagents/core/agent-strategy.ts +1 -1
- package/external/pi-tools-suite/src/async-subagents/core/agents-dir.ts +0 -2
- package/external/pi-tools-suite/src/async-subagents/core/browser-qa.ts +16 -2
- package/external/pi-tools-suite/src/async-subagents/core/config.ts +40 -17
- package/external/pi-tools-suite/src/async-subagents/core/model-selection.ts +2 -1
- package/external/pi-tools-suite/src/async-subagents/core/registry.ts +158 -33
- package/external/pi-tools-suite/src/async-subagents/core/routing.ts +7 -4
- package/external/pi-tools-suite/src/async-subagents/core/spawn.ts +34 -24
- package/external/pi-tools-suite/src/async-subagents/core/ultrawork-auto.ts +51 -45
- package/external/pi-tools-suite/src/async-subagents/lib.ts +9 -1
- package/external/pi-tools-suite/src/async-subagents/tools/spawn.ts +0 -1
- package/external/pi-tools-suite/src/async-subagents/tools/subagents.ts +2 -2
- package/external/pi-tools-suite/src/coding-discipline/index.ts +90 -59
- package/external/pi-tools-suite/src/config.ts +14 -0
- package/external/pi-tools-suite/src/context-gateway/accounting-log.ts +282 -0
- package/external/pi-tools-suite/src/context-gateway/config.ts +66 -1
- package/external/pi-tools-suite/src/context-gateway/efficiency.ts +414 -0
- package/external/pi-tools-suite/src/context-gateway/enforcement.ts +123 -3
- package/external/pi-tools-suite/src/context-gateway/index.ts +137 -4
- package/external/pi-tools-suite/src/context-gateway/storeless-capabilities.ts +5 -5
- package/external/pi-tools-suite/src/context-gateway/telemetry.ts +2 -2
- package/external/pi-tools-suite/src/context-gateway/types.ts +8 -1
- package/external/pi-tools-suite/src/dcp/auto-compress-budget.ts +39 -4
- package/external/pi-tools-suite/src/dcp/auto-compress.ts +41 -14
- package/external/pi-tools-suite/src/dcp/commands.ts +14 -156
- package/external/pi-tools-suite/src/dcp/compress-tool.ts +8 -2
- package/external/pi-tools-suite/src/dcp/compression-progress.ts +22 -0
- package/external/pi-tools-suite/src/dcp/config.ts +58 -4
- package/external/pi-tools-suite/src/dcp/conversation-index.ts +36 -9
- package/external/pi-tools-suite/src/dcp/diagnostics.ts +41 -0
- package/external/pi-tools-suite/src/dcp/fresh-tool-results.ts +38 -0
- package/external/pi-tools-suite/src/dcp/index.ts +338 -66
- package/external/pi-tools-suite/src/dcp/journal.ts +126 -4
- package/external/pi-tools-suite/src/dcp/progress-controller.ts +2 -1
- package/external/pi-tools-suite/src/dcp/pruner-candidates.ts +92 -2
- package/external/pi-tools-suite/src/dcp/pruner-nudge.ts +36 -33
- package/external/pi-tools-suite/src/dcp/pruner.ts +1 -0
- package/external/pi-tools-suite/src/dcp/routine-pressure.ts +86 -0
- package/external/pi-tools-suite/src/dcp/state.ts +9 -0
- package/external/pi-tools-suite/src/dcp/statistics.d.ts +8 -0
- package/external/pi-tools-suite/src/dcp/statistics.js +156 -0
- package/external/pi-tools-suite/src/default-pi-tools-suite-config.ts +68 -16
- package/external/pi-tools-suite/src/lib/project.ts +36 -1
- package/external/pi-tools-suite/src/model-tools/index.ts +10 -7
- package/external/pi-tools-suite/src/repo-discovery/index.ts +304 -4
- package/external/pi-tools-suite/src/resource-registry/index.ts +282 -21
- package/external/pi-tools-suite/src/session-recovery/index.ts +17 -0
- package/external/pi-tools-suite/src/todo/index.ts +21 -0
- package/external/pi-tools-suite/src/todo/todo.ts +19 -1
- package/external/pi-tools-suite/src/tool-descriptions.ts +44 -15
- package/package.json +7 -8
- package/schemas/pi-tools-suite.json +44 -12
- package/schemas/pix.json +109 -15
- package/schemas/tasks.json +1 -2
- package/skills/simplify/SKILL.md +33 -5
- package/external/pi-tools-suite/docs/browser-qa-subagent.md +0 -196
- package/external/pi-tools-suite/src/async-subagents/agents/browser-qa.md +0 -598
- package/skills/playwright-cli/SKILL.md +0 -420
- package/skills/playwright-cli/references/element-attributes.md +0 -23
- package/skills/playwright-cli/references/playwright-tests.md +0 -50
- package/skills/playwright-cli/references/request-mocking.md +0 -87
- package/skills/playwright-cli/references/running-code.md +0 -241
- package/skills/playwright-cli/references/session-management.md +0 -273
- package/skills/playwright-cli/references/spec-driven-testing.md +0 -311
- package/skills/playwright-cli/references/storage-state.md +0 -290
- package/skills/playwright-cli/references/test-generation.md +0 -142
- package/skills/playwright-cli/references/tracing.md +0 -154
- package/skills/playwright-cli/references/video-recording.md +0 -147
- package/skills/spec-wiki/SKILL.md +0 -55
- package/skills/spec-wiki/evals/evals.json +0 -132
- package/skills/spec-wiki/evals/files/bootstrap/architecture/rfc-007.md +0 -14
- package/skills/spec-wiki/evals/files/bootstrap/docs/how-to-pay.md +0 -4
- package/skills/spec-wiki/evals/files/bootstrap/docs/overview.md +0 -9
- package/skills/spec-wiki/evals/files/bootstrap/requirements/payment-behavior.md +0 -17
- package/skills/spec-wiki/evals/files/bootstrap/src/payments.ts +0 -12
- package/skills/spec-wiki/evals/files/bootstrap/tests/payments.test.ts +0 -11
- package/skills/spec-wiki/evals/files/focused/src/session.ts +0 -5
- package/skills/spec-wiki/evals/files/focused/tests/session.test.ts +0 -17
- package/skills/spec-wiki/evals/files/maintenance/low-signal/incoming/capture.txt +0 -2
- package/skills/spec-wiki/evals/files/maintenance/low-signal/src/capture.ts +0 -3
- package/skills/spec-wiki/evals/files/maintenance/move/docs/old-job-contract.md +0 -18
- package/skills/spec-wiki/evals/files/maintenance/move/src/job.ts +0 -5
- package/skills/spec-wiki/evals/files/maintenance/new-relation/docs/session-refresh.md +0 -24
- package/skills/spec-wiki/evals/files/maintenance/new-relation/incoming/refresh-worker.ts +0 -7
- package/skills/spec-wiki/evals/files/maintenance/new-relation/src/session.ts +0 -7
- package/skills/spec-wiki/evals/files/maintenance/no-impact/docs/auth-contract.md +0 -17
- package/skills/spec-wiki/evals/files/maintenance/no-impact/incoming/telemetry-format.ts +0 -3
- package/skills/spec-wiki/evals/files/maintenance/no-impact/src/auth.ts +0 -3
- package/skills/spec-wiki/evals/files/query/docs/how-to-run-jobs.md +0 -4
- package/skills/spec-wiki/evals/files/query/docs/spec-overview.md +0 -9
- package/skills/spec-wiki/evals/files/query/features/jobs/cancellation-contract.md +0 -15
- package/skills/spec-wiki/evals/files/query/features/jobs/retry-spec.md +0 -6
- package/skills/spec-wiki/evals/maintenance_regression.py +0 -311
- package/skills/spec-wiki/evals/retrieval-eval.md +0 -37
- package/skills/spec-wiki/evals/retrieval-evals.json +0 -37
- package/skills/spec-wiki/evals/trigger-evals.json +0 -94
- package/skills/spec-wiki/references/bootstrap.md +0 -100
- package/skills/spec-wiki/references/focused-spec.md +0 -117
- package/skills/spec-wiki/references/index-schema.md +0 -330
- package/skills/spec-wiki/references/maintenance.md +0 -134
- package/skills/spec-wiki/references/query.md +0 -74
- package/skills/spec-wiki/scripts/spec_wiki.py +0 -2076
- /package/external/pi-tools-suite/src/async-subagents/agents/{browser-qa → ui-qa/browser}/examples/qa-auth.example.jsonc +0 -0
- /package/external/pi-tools-suite/src/async-subagents/agents/{browser-qa → ui-qa/browser}/examples/qa-flow.example.jsonc +0 -0
- /package/external/pi-tools-suite/src/async-subagents/agents/{browser-qa → ui-qa/browser}/vendor/fflate.LICENSE +0 -0
- /package/external/pi-tools-suite/src/async-subagents/agents/{browser-qa → ui-qa/browser}/vendor/fflate.mjs +0 -0
|
@@ -1,29 +1,24 @@
|
|
|
1
1
|
import { spawn } from "node:child_process";
|
|
2
|
-
import {
|
|
3
|
-
import { access, mkdir, readFile, rename, rm, writeFile } from "node:fs/promises";
|
|
4
|
-
import http from "node:http";
|
|
5
|
-
import https from "node:https";
|
|
6
|
-
import { createRequire } from "node:module";
|
|
7
|
-
import { join } from "node:path";
|
|
8
|
-
import { pipeline } from "node:stream/promises";
|
|
9
|
-
import { fileURLToPath } from "node:url";
|
|
10
|
-
import { savePixDictationLanguage } from "../../config.js";
|
|
2
|
+
import { savePixDictationLanguage, } from "../../config.js";
|
|
11
3
|
import { APP_ICONS } from "../icons.js";
|
|
12
4
|
import { commandExists } from "../process.js";
|
|
13
5
|
const SAMPLE_RATE = 16_000;
|
|
14
|
-
const
|
|
15
|
-
const
|
|
16
|
-
const
|
|
17
|
-
const
|
|
6
|
+
const DEEPGRAM_SOCKET_OPEN = 1;
|
|
7
|
+
const DEEPGRAM_SOCKET_CONNECT_TIMEOUT_MS = 10_000;
|
|
8
|
+
const RECORDER_STOP_GRACE_MS = 150;
|
|
9
|
+
const RECORDER_FORCE_STOP_GRACE_MS = 50;
|
|
10
|
+
const DEEPGRAM_FINALIZE_GRACE_MS = 300;
|
|
11
|
+
const DEFAULT_DEEPGRAM_MODEL = "nova-3";
|
|
18
12
|
const VOICE_SPINNER_FRAMES = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"];
|
|
19
13
|
const VOICE_PARTIAL_TRANSCRIPT_THROTTLE_MS = 100;
|
|
20
|
-
let voskInstallPromise;
|
|
21
14
|
const defaultVoiceControllerDeps = {
|
|
22
|
-
|
|
23
|
-
|
|
15
|
+
deepgramApiKey: () => process.env.DEEPGRAM_API_KEY?.trim() || undefined,
|
|
16
|
+
createDeepgramSocket,
|
|
17
|
+
waitForSocketOpen,
|
|
24
18
|
selectRecorderCommand,
|
|
25
19
|
spawn,
|
|
26
20
|
savePixDictationLanguage,
|
|
21
|
+
delay,
|
|
27
22
|
};
|
|
28
23
|
let voiceControllerDeps = defaultVoiceControllerDeps;
|
|
29
24
|
export function setVoiceControllerTestDeps(overrides) {
|
|
@@ -31,13 +26,14 @@ export function setVoiceControllerTestDeps(overrides) {
|
|
|
31
26
|
}
|
|
32
27
|
export class AppVoiceController {
|
|
33
28
|
host;
|
|
34
|
-
|
|
29
|
+
languageDefinitions;
|
|
35
30
|
languages;
|
|
36
31
|
language;
|
|
32
|
+
deepgramModel;
|
|
33
|
+
deepgramApiKey;
|
|
37
34
|
state = "idle";
|
|
38
|
-
modelCache = new Map();
|
|
39
35
|
audioProcess;
|
|
40
|
-
|
|
36
|
+
socket;
|
|
41
37
|
progressMessage;
|
|
42
38
|
progressFrame = 0;
|
|
43
39
|
progressTimer;
|
|
@@ -46,35 +42,28 @@ export class AppVoiceController {
|
|
|
46
42
|
partialTranscriptTimer;
|
|
47
43
|
startGeneration = 0;
|
|
48
44
|
recordingScope;
|
|
45
|
+
stopPromise;
|
|
46
|
+
disposed = false;
|
|
49
47
|
constructor(host, dictationConfig) {
|
|
50
48
|
this.host = host;
|
|
51
|
-
this.
|
|
52
|
-
this.languages = Object.keys(this.
|
|
49
|
+
this.languageDefinitions = dictationConfig.languages;
|
|
50
|
+
this.languages = Object.keys(this.languageDefinitions);
|
|
53
51
|
this.language = this.initialLanguage(dictationConfig.language);
|
|
52
|
+
this.deepgramModel = dictationConfig.model?.trim() || DEFAULT_DEEPGRAM_MODEL;
|
|
53
|
+
this.deepgramApiKey = dictationConfig.apiKey?.trim() || undefined;
|
|
54
54
|
}
|
|
55
55
|
updateDictationConfig(dictationConfig) {
|
|
56
|
-
this.
|
|
57
|
-
this.languages = Object.keys(this.
|
|
56
|
+
this.languageDefinitions = dictationConfig.languages;
|
|
57
|
+
this.languages = Object.keys(this.languageDefinitions);
|
|
58
58
|
this.language = this.initialLanguage(dictationConfig.language ?? this.language);
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
this.modelCache.delete(language);
|
|
62
|
-
}
|
|
59
|
+
this.deepgramModel = dictationConfig.model?.trim() || DEFAULT_DEEPGRAM_MODEL;
|
|
60
|
+
this.deepgramApiKey = dictationConfig.apiKey?.trim() || undefined;
|
|
63
61
|
}
|
|
64
62
|
statusWidgetText() {
|
|
65
63
|
const languageLabel = this.showLanguageSwitcher() ? ` ${this.language.toUpperCase()}` : "";
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
case "downloading":
|
|
70
|
-
return `${APP_ICONS.microphone}${languageLabel} ${APP_ICONS.timerSand}`;
|
|
71
|
-
case "loading":
|
|
72
|
-
return `${APP_ICONS.microphone}${languageLabel} ${APP_ICONS.timerSand}`;
|
|
73
|
-
case "listening":
|
|
74
|
-
return `${APP_ICONS.microphone}${languageLabel}`;
|
|
75
|
-
case "idle":
|
|
76
|
-
return `${APP_ICONS.microphone}${languageLabel}`;
|
|
77
|
-
}
|
|
64
|
+
return this.state === "connecting"
|
|
65
|
+
? `${APP_ICONS.microphone}${languageLabel} ${APP_ICONS.timerSand}`
|
|
66
|
+
: `${APP_ICONS.microphone}${languageLabel}`;
|
|
78
67
|
}
|
|
79
68
|
showLanguageSwitcher() {
|
|
80
69
|
return this.languages.length > 1;
|
|
@@ -89,6 +78,12 @@ export class AppVoiceController {
|
|
|
89
78
|
return `${spinner} ${this.progressMessage}`;
|
|
90
79
|
}
|
|
91
80
|
async toggleRecording() {
|
|
81
|
+
if (this.disposed)
|
|
82
|
+
return;
|
|
83
|
+
if (this.stopPromise) {
|
|
84
|
+
await this.stopPromise;
|
|
85
|
+
return;
|
|
86
|
+
}
|
|
92
87
|
if (this.state !== "idle") {
|
|
93
88
|
await this.stopRecording();
|
|
94
89
|
return;
|
|
@@ -96,152 +91,197 @@ export class AppVoiceController {
|
|
|
96
91
|
await this.startRecording();
|
|
97
92
|
}
|
|
98
93
|
async toggleLanguage() {
|
|
99
|
-
if (!this.showLanguageSwitcher())
|
|
94
|
+
if (this.disposed || !this.showLanguageSwitcher())
|
|
95
|
+
return;
|
|
96
|
+
if (this.stopPromise) {
|
|
97
|
+
await this.stopPromise;
|
|
100
98
|
return;
|
|
99
|
+
}
|
|
101
100
|
const scope = this.host.activeInputScope();
|
|
102
|
-
const
|
|
103
|
-
if (
|
|
101
|
+
const shouldRestart = this.state !== "idle";
|
|
102
|
+
if (shouldRestart)
|
|
104
103
|
await this.stopRecording();
|
|
105
|
-
if (!this.isScopeActive(scope))
|
|
104
|
+
if (this.disposed || !this.isScopeActive(scope))
|
|
106
105
|
return;
|
|
107
106
|
this.language = this.nextLanguage();
|
|
108
107
|
this.saveLanguageSelection(this.language);
|
|
109
|
-
this.host.showToast(`Voice language: ${this.
|
|
108
|
+
this.host.showToast(`Voice language: ${this.languageDefinition(this.language).label}`, "info");
|
|
110
109
|
this.host.render();
|
|
111
|
-
if (
|
|
110
|
+
if (shouldRestart)
|
|
112
111
|
void this.startRecording();
|
|
113
112
|
}
|
|
114
113
|
async stopRecording() {
|
|
114
|
+
if (this.disposed)
|
|
115
|
+
return;
|
|
116
|
+
if (this.stopPromise)
|
|
117
|
+
return await this.stopPromise;
|
|
118
|
+
const stopping = Promise.resolve().then(() => this.stopRecordingNow());
|
|
119
|
+
this.stopPromise = stopping;
|
|
120
|
+
try {
|
|
121
|
+
await stopping;
|
|
122
|
+
}
|
|
123
|
+
finally {
|
|
124
|
+
if (this.stopPromise === stopping)
|
|
125
|
+
this.stopPromise = undefined;
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
async dispose() {
|
|
129
|
+
if (this.disposed)
|
|
130
|
+
return;
|
|
131
|
+
this.disposed = true;
|
|
115
132
|
this.startGeneration += 1;
|
|
116
|
-
const scope = this.recordingScope;
|
|
117
133
|
const audioProcess = this.audioProcess;
|
|
118
|
-
const
|
|
134
|
+
const socket = this.socket;
|
|
119
135
|
this.audioProcess = undefined;
|
|
120
|
-
this.
|
|
136
|
+
this.socket = undefined;
|
|
121
137
|
this.recordingScope = undefined;
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
if (
|
|
125
|
-
this.
|
|
126
|
-
this.
|
|
127
|
-
recognizer.free?.();
|
|
128
|
-
}
|
|
129
|
-
if (this.state !== "idle") {
|
|
130
|
-
this.clearProgressMessage();
|
|
131
|
-
this.state = "idle";
|
|
132
|
-
if (this.isScopeActive(scope))
|
|
133
|
-
this.host.render();
|
|
138
|
+
this.clearProgressMessage();
|
|
139
|
+
this.partialTranscript = undefined;
|
|
140
|
+
if (this.partialTranscriptTimer) {
|
|
141
|
+
clearTimeout(this.partialTranscriptTimer);
|
|
142
|
+
this.partialTranscriptTimer = undefined;
|
|
134
143
|
}
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
this.modelCache.clear();
|
|
144
|
+
this.state = "idle";
|
|
145
|
+
if (audioProcess?.exitCode === null)
|
|
146
|
+
safeKillProcess(audioProcess, "SIGKILL");
|
|
147
|
+
if (socket)
|
|
148
|
+
safeCloseSocket(socket);
|
|
141
149
|
}
|
|
142
150
|
async startRecording() {
|
|
151
|
+
if (this.disposed)
|
|
152
|
+
return;
|
|
143
153
|
const language = this.language;
|
|
144
154
|
const scope = this.host.activeInputScope();
|
|
145
155
|
const generation = this.startGeneration + 1;
|
|
146
156
|
this.startGeneration = generation;
|
|
147
157
|
this.recordingScope = scope;
|
|
158
|
+
this.state = "connecting";
|
|
159
|
+
this.setProgressMessage("Connecting to Deepgram...", generation, scope);
|
|
160
|
+
let socket;
|
|
148
161
|
try {
|
|
149
|
-
const
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
if (!this.continueStart(generation, scope))
|
|
154
|
-
return;
|
|
155
|
-
vosk.setLogLevel?.(-1);
|
|
156
|
-
this.state = "downloading";
|
|
157
|
-
this.host.render();
|
|
158
|
-
const modelPath = await voiceControllerDeps.ensureModel(language, this.modelDefinition(language));
|
|
159
|
-
if (!this.continueStart(generation, scope))
|
|
160
|
-
return;
|
|
161
|
-
this.state = "loading";
|
|
162
|
-
this.host.render();
|
|
163
|
-
const model = this.cachedModel(language, modelPath, vosk);
|
|
162
|
+
const apiKey = this.deepgramApiKey ?? voiceControllerDeps.deepgramApiKey();
|
|
163
|
+
if (!apiKey) {
|
|
164
|
+
throw new Error("Deepgram API key is not configured; set dictation.apiKey in ~/.config/pi/pix.jsonc or DEEPGRAM_API_KEY");
|
|
165
|
+
}
|
|
164
166
|
const recorder = await voiceControllerDeps.selectRecorderCommand();
|
|
165
167
|
if (!this.continueStart(generation, scope))
|
|
166
168
|
return;
|
|
167
|
-
|
|
169
|
+
socket = voiceControllerDeps.createDeepgramSocket(buildDeepgramUrl(this.deepgramModel, this.deepgramLanguage(language), true), ["token", apiKey]);
|
|
170
|
+
this.socket = socket;
|
|
171
|
+
await voiceControllerDeps.waitForSocketOpen(socket);
|
|
172
|
+
if (!this.continueStart(generation, scope) || this.socket !== socket) {
|
|
173
|
+
safeCloseSocket(socket);
|
|
174
|
+
return;
|
|
175
|
+
}
|
|
176
|
+
this.bindSocket(socket, generation, scope);
|
|
168
177
|
const audioProcess = voiceControllerDeps.spawn(recorder.command, recorder.args, { stdio: ["ignore", "pipe", "pipe"] });
|
|
169
|
-
this.recognizer = recognizer;
|
|
170
178
|
this.audioProcess = audioProcess;
|
|
179
|
+
this.clearProgressMessage();
|
|
171
180
|
this.state = "listening";
|
|
172
181
|
this.host.render();
|
|
173
|
-
this.host.showToast(`Voice input on (${this.
|
|
174
|
-
this.bindAudioProcess(audioProcess,
|
|
182
|
+
this.host.showToast(`Voice input on (${this.languageDefinition(language).label}, Deepgram ${this.deepgramModel})`, "info");
|
|
183
|
+
this.bindAudioProcess(audioProcess, socket, generation, scope);
|
|
175
184
|
}
|
|
176
185
|
catch (error) {
|
|
177
|
-
if (!this.
|
|
186
|
+
if (!this.isCurrentStart(generation, scope)) {
|
|
187
|
+
if (socket)
|
|
188
|
+
safeCloseSocket(socket);
|
|
178
189
|
return;
|
|
190
|
+
}
|
|
191
|
+
this.startGeneration += 1;
|
|
192
|
+
this.recordingScope = undefined;
|
|
193
|
+
this.audioProcess = undefined;
|
|
194
|
+
this.socket = undefined;
|
|
195
|
+
if (socket)
|
|
196
|
+
safeCloseSocket(socket);
|
|
197
|
+
this.clearPartialTranscript(scope);
|
|
179
198
|
this.clearProgressMessage();
|
|
180
|
-
this.cleanupRecognizer();
|
|
181
199
|
this.state = "idle";
|
|
182
200
|
this.addProgressSystemMessage(`Unavailable: ${errorMessage(error)}`);
|
|
183
201
|
this.host.showToast(`Voice input unavailable: ${errorMessage(error)}`, "error");
|
|
184
202
|
this.host.render();
|
|
185
203
|
}
|
|
186
204
|
}
|
|
187
|
-
|
|
188
|
-
const
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
const
|
|
192
|
-
this.
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
205
|
+
async stopRecordingNow() {
|
|
206
|
+
const scope = this.recordingScope;
|
|
207
|
+
const generation = this.startGeneration;
|
|
208
|
+
const audioProcess = this.audioProcess;
|
|
209
|
+
const socket = this.socket;
|
|
210
|
+
const wasListening = this.state === "listening";
|
|
211
|
+
this.clearProgressMessage();
|
|
212
|
+
this.state = "idle";
|
|
213
|
+
if (this.isScopeActive(scope))
|
|
214
|
+
this.host.render();
|
|
215
|
+
if (!wasListening || !socket || socket.readyState !== DEEPGRAM_SOCKET_OPEN) {
|
|
216
|
+
if (audioProcess?.exitCode === null)
|
|
217
|
+
safeKillProcess(audioProcess, "SIGKILL");
|
|
218
|
+
if (this.audioProcess === audioProcess)
|
|
219
|
+
this.audioProcess = undefined;
|
|
220
|
+
this.startGeneration += 1;
|
|
221
|
+
this.socket = undefined;
|
|
222
|
+
this.recordingScope = undefined;
|
|
223
|
+
this.clearPartialTranscript(scope);
|
|
224
|
+
if (socket)
|
|
225
|
+
safeCloseSocket(socket);
|
|
226
|
+
return;
|
|
227
|
+
}
|
|
228
|
+
if (audioProcess)
|
|
229
|
+
await stopRecorderProcess(audioProcess);
|
|
230
|
+
if (this.audioProcess === audioProcess)
|
|
231
|
+
this.audioProcess = undefined;
|
|
206
232
|
try {
|
|
207
|
-
|
|
233
|
+
socket.send(JSON.stringify({ type: "Finalize" }));
|
|
208
234
|
}
|
|
209
|
-
catch
|
|
210
|
-
|
|
235
|
+
catch {
|
|
236
|
+
// Closing below still commits the most recent interim result.
|
|
211
237
|
}
|
|
238
|
+
await voiceControllerDeps.delay(DEEPGRAM_FINALIZE_GRACE_MS);
|
|
239
|
+
if (this.startGeneration === generation && this.recordingScope === scope && this.socket === socket) {
|
|
240
|
+
this.commitPartialTranscript(scope);
|
|
241
|
+
this.startGeneration += 1;
|
|
242
|
+
this.socket = undefined;
|
|
243
|
+
this.recordingScope = undefined;
|
|
244
|
+
}
|
|
245
|
+
safeCloseSocket(socket);
|
|
212
246
|
}
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
247
|
+
bindSocket(socket, generation, scope) {
|
|
248
|
+
socket.addEventListener("message", (event) => {
|
|
249
|
+
if (!this.isCurrentSocket(socket, generation, scope))
|
|
250
|
+
return;
|
|
251
|
+
const transcript = parseDeepgramTranscript(event.data);
|
|
252
|
+
if (!transcript)
|
|
253
|
+
return;
|
|
254
|
+
if (transcript.isFinal) {
|
|
255
|
+
this.clearPartialTranscript(scope);
|
|
256
|
+
this.host.insertTranscript(transcript.text);
|
|
257
|
+
return;
|
|
258
|
+
}
|
|
259
|
+
this.emitPartialTranscript(transcript.text, socket, generation, scope);
|
|
260
|
+
});
|
|
261
|
+
socket.addEventListener("error", () => {
|
|
262
|
+
if (!this.isCurrentSocket(socket, generation, scope) || this.state === "idle")
|
|
263
|
+
return;
|
|
264
|
+
this.host.showToast("Voice recognition failed: Deepgram WebSocket error", "error");
|
|
265
|
+
void this.stopRecording();
|
|
266
|
+
});
|
|
267
|
+
socket.addEventListener("close", (event) => {
|
|
268
|
+
if (!this.isCurrentSocket(socket, generation, scope) || this.state === "idle")
|
|
269
|
+
return;
|
|
270
|
+
const suffix = event.code && event.code !== 1000
|
|
271
|
+
? ` (${event.code}${event.reason ? `: ${event.reason}` : ""})`
|
|
272
|
+
: "";
|
|
273
|
+
this.host.showToast(`Voice recognition stopped: Deepgram connection closed${suffix}`, "warning");
|
|
274
|
+
void this.stopRecording();
|
|
225
275
|
});
|
|
226
|
-
if (this.isCurrentStart(generation, scope))
|
|
227
|
-
this.addProgressSystemMessage("Vosk voice bindings are ready.");
|
|
228
|
-
if (this.isCurrentStart(generation, scope))
|
|
229
|
-
this.clearProgressMessage();
|
|
230
|
-
return vosk;
|
|
231
276
|
}
|
|
232
|
-
bindAudioProcess(audioProcess,
|
|
277
|
+
bindAudioProcess(audioProcess, socket, generation, scope) {
|
|
233
278
|
let stderr = "";
|
|
234
279
|
audioProcess.stdout.on("data", (chunk) => {
|
|
235
|
-
if (!this.isCurrentAudioProcess(audioProcess,
|
|
280
|
+
if (!this.isCurrentAudioProcess(audioProcess, socket, generation, scope))
|
|
236
281
|
return;
|
|
237
282
|
try {
|
|
238
|
-
if (
|
|
239
|
-
|
|
240
|
-
this.emitTranscript(recognizer.result(), scope);
|
|
241
|
-
}
|
|
242
|
-
else {
|
|
243
|
-
this.emitPartialTranscript(recognizer.partialResult?.(), audioProcess, recognizer, generation, scope);
|
|
244
|
-
}
|
|
283
|
+
if (socket.readyState === DEEPGRAM_SOCKET_OPEN)
|
|
284
|
+
socket.send(chunk);
|
|
245
285
|
}
|
|
246
286
|
catch (error) {
|
|
247
287
|
this.host.showToast(`Voice recognition failed: ${errorMessage(error)}`, "error");
|
|
@@ -252,33 +292,51 @@ export class AppVoiceController {
|
|
|
252
292
|
stderr = `${stderr}${chunk.toString("utf8")}`.slice(-600);
|
|
253
293
|
});
|
|
254
294
|
audioProcess.once("error", (error) => {
|
|
255
|
-
if (!this.isCurrentAudioProcess(audioProcess,
|
|
295
|
+
if (!this.isCurrentAudioProcess(audioProcess, socket, generation, scope))
|
|
296
|
+
return;
|
|
297
|
+
if (this.state === "idle")
|
|
256
298
|
return;
|
|
257
|
-
this.clearPartialTranscript(scope);
|
|
258
|
-
this.cleanupRecognizer();
|
|
259
|
-
this.audioProcess = undefined;
|
|
260
|
-
this.state = "idle";
|
|
261
299
|
this.host.showToast(`Voice recorder failed: ${errorMessage(error)}`, "error");
|
|
262
|
-
this.
|
|
300
|
+
void this.stopRecording();
|
|
263
301
|
});
|
|
264
302
|
audioProcess.once("close", (code, signal) => {
|
|
265
|
-
if (!this.isCurrentAudioProcess(audioProcess,
|
|
303
|
+
if (!this.isCurrentAudioProcess(audioProcess, socket, generation, scope))
|
|
304
|
+
return;
|
|
305
|
+
if (this.state === "idle")
|
|
266
306
|
return;
|
|
267
|
-
this.clearPartialTranscript(scope);
|
|
268
|
-
this.emitTranscript(recognizer.finalResult(), scope);
|
|
269
|
-
this.cleanupRecognizer();
|
|
270
|
-
this.audioProcess = undefined;
|
|
271
|
-
this.state = "idle";
|
|
272
307
|
if (code && code !== 0) {
|
|
273
308
|
const details = stderr.trim() || signal || `exit code ${code}`;
|
|
274
309
|
this.host.showToast(`Voice recorder stopped: ${details}`, "warning");
|
|
275
310
|
}
|
|
276
|
-
this.
|
|
311
|
+
void this.stopRecording();
|
|
277
312
|
});
|
|
278
313
|
}
|
|
279
|
-
|
|
280
|
-
this.
|
|
281
|
-
this.
|
|
314
|
+
nextLanguage() {
|
|
315
|
+
const currentIndex = this.languages.indexOf(this.language);
|
|
316
|
+
const nextIndex = currentIndex >= 0 ? (currentIndex + 1) % this.languages.length : 0;
|
|
317
|
+
return this.languages[nextIndex] ?? this.language;
|
|
318
|
+
}
|
|
319
|
+
initialLanguage(configuredLanguage) {
|
|
320
|
+
if (configuredLanguage && this.languages.includes(configuredLanguage))
|
|
321
|
+
return configuredLanguage;
|
|
322
|
+
return this.languages.includes("en") ? "en" : this.languages[0] ?? "en";
|
|
323
|
+
}
|
|
324
|
+
saveLanguageSelection(language) {
|
|
325
|
+
try {
|
|
326
|
+
voiceControllerDeps.savePixDictationLanguage(language);
|
|
327
|
+
}
|
|
328
|
+
catch (error) {
|
|
329
|
+
this.host.showToast(`Could not save voice language: ${errorMessage(error)}`, "warning");
|
|
330
|
+
}
|
|
331
|
+
}
|
|
332
|
+
languageDefinition(language) {
|
|
333
|
+
const definition = this.languageDefinitions[language];
|
|
334
|
+
if (!definition)
|
|
335
|
+
throw new Error(`dictation language is not configured: ${language}`);
|
|
336
|
+
return definition;
|
|
337
|
+
}
|
|
338
|
+
deepgramLanguage(language) {
|
|
339
|
+
return this.languageDefinition(language).deepgramLanguage?.trim() || language;
|
|
282
340
|
}
|
|
283
341
|
setProgressMessage(message, generation = this.startGeneration, scope = this.recordingScope) {
|
|
284
342
|
this.progressMessage = message;
|
|
@@ -309,24 +367,13 @@ export class AppVoiceController {
|
|
|
309
367
|
this.lastSystemProgressMessage = text;
|
|
310
368
|
this.host.addSystemMessage(text);
|
|
311
369
|
}
|
|
312
|
-
|
|
313
|
-
if (!this.
|
|
314
|
-
return;
|
|
315
|
-
const text = transcriptText(result);
|
|
316
|
-
if (!text)
|
|
370
|
+
emitPartialTranscript(text, socket = this.socket, generation = this.startGeneration, scope = this.recordingScope) {
|
|
371
|
+
if (socket && !this.isCurrentSocket(socket, generation, scope))
|
|
317
372
|
return;
|
|
318
|
-
this.host.insertTranscript(text);
|
|
319
|
-
}
|
|
320
|
-
emitPartialTranscript(result, audioProcess = this.audioProcess, recognizer = this.recognizer, generation = this.startGeneration, scope = this.recordingScope) {
|
|
321
|
-
if (audioProcess || recognizer) {
|
|
322
|
-
if (!audioProcess || !recognizer || !this.isCurrentAudioProcess(audioProcess, recognizer, generation, scope))
|
|
323
|
-
return;
|
|
324
|
-
}
|
|
325
|
-
const text = partialTranscriptText(result);
|
|
326
373
|
if (text === this.partialTranscript)
|
|
327
374
|
return;
|
|
328
375
|
this.partialTranscript = text;
|
|
329
|
-
this.schedulePartialTranscriptEmit(
|
|
376
|
+
this.schedulePartialTranscriptEmit(socket, generation, scope);
|
|
330
377
|
}
|
|
331
378
|
clearPartialTranscript(scope = this.recordingScope) {
|
|
332
379
|
if (!this.partialTranscript)
|
|
@@ -339,18 +386,21 @@ export class AppVoiceController {
|
|
|
339
386
|
if (this.isScopeActive(scope))
|
|
340
387
|
this.host.setPartialTranscript(undefined);
|
|
341
388
|
}
|
|
342
|
-
|
|
389
|
+
commitPartialTranscript(scope = this.recordingScope) {
|
|
390
|
+
const text = this.partialTranscript;
|
|
391
|
+
this.clearPartialTranscript(scope);
|
|
392
|
+
if (text && this.isScopeActive(scope))
|
|
393
|
+
this.host.insertTranscript(text);
|
|
394
|
+
}
|
|
395
|
+
schedulePartialTranscriptEmit(socket, generation, scope) {
|
|
343
396
|
if (this.partialTranscriptTimer)
|
|
344
397
|
return;
|
|
345
398
|
this.partialTranscriptTimer = setTimeout(() => {
|
|
346
399
|
this.partialTranscriptTimer = undefined;
|
|
347
|
-
if (
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
}
|
|
351
|
-
else if (!this.isScopeActive(scope)) {
|
|
400
|
+
if (socket && !this.isCurrentSocket(socket, generation, scope))
|
|
401
|
+
return;
|
|
402
|
+
if (!this.isScopeActive(scope))
|
|
352
403
|
return;
|
|
353
|
-
}
|
|
354
404
|
this.host.setPartialTranscript(this.partialTranscript);
|
|
355
405
|
}, VOICE_PARTIAL_TRANSCRIPT_THROTTLE_MS);
|
|
356
406
|
this.partialTranscriptTimer.unref?.();
|
|
@@ -365,256 +415,66 @@ export class AppVoiceController {
|
|
|
365
415
|
return true;
|
|
366
416
|
this.startGeneration += 1;
|
|
367
417
|
this.recordingScope = undefined;
|
|
418
|
+
this.socket = undefined;
|
|
368
419
|
this.clearProgressMessage();
|
|
369
420
|
this.state = "idle";
|
|
370
421
|
return false;
|
|
371
422
|
}
|
|
372
|
-
|
|
373
|
-
return this.
|
|
423
|
+
isCurrentSocket(socket, generation, scope = this.recordingScope) {
|
|
424
|
+
return this.startGeneration === generation
|
|
425
|
+
&& this.recordingScope === scope
|
|
426
|
+
&& this.socket === socket
|
|
427
|
+
&& this.isScopeActive(scope);
|
|
428
|
+
}
|
|
429
|
+
isCurrentAudioProcess(audioProcess, socket, generation, scope = this.recordingScope) {
|
|
430
|
+
return this.audioProcess === audioProcess && this.isCurrentSocket(socket, generation, scope);
|
|
374
431
|
}
|
|
375
432
|
isScopeActive(scope) {
|
|
376
433
|
return this.host.activeInputScope() === scope;
|
|
377
434
|
}
|
|
378
435
|
}
|
|
379
|
-
|
|
380
|
-
const
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
throw new Error(`downloaded ${definition.label} (${language}) model did not contain a valid Vosk model`);
|
|
396
|
-
}
|
|
397
|
-
await rm(modelPath, { recursive: true, force: true });
|
|
398
|
-
await rename(extractedPath, modelPath);
|
|
399
|
-
await rm(tempPath, { recursive: true, force: true });
|
|
400
|
-
await rm(zipPath, { force: true });
|
|
401
|
-
return modelPath;
|
|
402
|
-
}
|
|
403
|
-
async function looksLikeVoskModel(modelPath) {
|
|
404
|
-
return (await pathExists(join(modelPath, "conf", "model.conf"))) && (await pathExists(join(modelPath, "am", "final.mdl")));
|
|
436
|
+
export function buildDeepgramUrl(model, language, rawLinear16) {
|
|
437
|
+
const url = new URL("wss://api.deepgram.com/v1/listen");
|
|
438
|
+
url.searchParams.set("model", model.trim() || DEFAULT_DEEPGRAM_MODEL);
|
|
439
|
+
url.searchParams.set("language", language.trim() || "en");
|
|
440
|
+
url.searchParams.set("interim_results", "true");
|
|
441
|
+
url.searchParams.set("punctuate", "true");
|
|
442
|
+
url.searchParams.set("smart_format", "true");
|
|
443
|
+
url.searchParams.set("vad_events", "true");
|
|
444
|
+
url.searchParams.set("endpointing", "300");
|
|
445
|
+
url.searchParams.set("utterance_end_ms", "1000");
|
|
446
|
+
if (rawLinear16) {
|
|
447
|
+
url.searchParams.set("encoding", "linear16");
|
|
448
|
+
url.searchParams.set("sample_rate", String(SAMPLE_RATE));
|
|
449
|
+
url.searchParams.set("channels", "1");
|
|
450
|
+
}
|
|
451
|
+
return url.toString();
|
|
405
452
|
}
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
async function downloadFile(url, destination, redirects = 3) {
|
|
416
|
-
await new Promise((resolve, reject) => {
|
|
417
|
-
const client = url.startsWith("https:") ? https : http;
|
|
418
|
-
let settled = false;
|
|
419
|
-
const finish = (callback) => {
|
|
420
|
-
if (settled)
|
|
421
|
-
return;
|
|
422
|
-
settled = true;
|
|
423
|
-
callback();
|
|
424
|
-
};
|
|
425
|
-
const request = client.get(url, (response) => {
|
|
426
|
-
const statusCode = response.statusCode ?? 0;
|
|
427
|
-
const location = response.headers.location;
|
|
428
|
-
if ([301, 302, 303, 307, 308].includes(statusCode) && location && redirects > 0) {
|
|
429
|
-
response.resume();
|
|
430
|
-
const redirectedUrl = new URL(location, url).toString();
|
|
431
|
-
downloadFile(redirectedUrl, destination, redirects - 1).then(() => finish(resolve), (error) => finish(() => reject(error)));
|
|
432
|
-
return;
|
|
433
|
-
}
|
|
434
|
-
if (statusCode !== 200) {
|
|
435
|
-
response.resume();
|
|
436
|
-
finish(() => reject(new Error(`download failed with HTTP ${statusCode}`)));
|
|
437
|
-
return;
|
|
438
|
-
}
|
|
439
|
-
const file = createWriteStream(destination);
|
|
440
|
-
response.on("error", (error) => {
|
|
441
|
-
finish(() => reject(error));
|
|
442
|
-
});
|
|
443
|
-
pipeline(response, file).then(() => finish(resolve), (error) => finish(() => reject(error)));
|
|
444
|
-
});
|
|
445
|
-
request.on("error", (error) => {
|
|
446
|
-
finish(() => reject(error));
|
|
447
|
-
});
|
|
448
|
-
});
|
|
449
|
-
}
|
|
450
|
-
async function extractZip(zipPath, destination) {
|
|
451
|
-
if (await commandExists("unzip")) {
|
|
452
|
-
await runCommand("unzip", ["-q", zipPath, "-d", destination]);
|
|
453
|
-
return;
|
|
454
|
-
}
|
|
455
|
-
if (process.platform === "darwin" && await commandExists("ditto")) {
|
|
456
|
-
await runCommand("ditto", ["-x", "-k", zipPath, destination]);
|
|
457
|
-
return;
|
|
458
|
-
}
|
|
459
|
-
throw new Error("cannot extract Vosk model: install `unzip` (or `ditto` on macOS)");
|
|
460
|
-
}
|
|
461
|
-
async function runCommand(command, args) {
|
|
462
|
-
await new Promise((resolve, reject) => {
|
|
463
|
-
const child = spawn(command, args, { stdio: ["ignore", "ignore", "pipe"] });
|
|
464
|
-
let stderr = "";
|
|
465
|
-
child.stderr.on("data", (chunk) => {
|
|
466
|
-
stderr = `${stderr}${chunk.toString("utf8")}`.slice(-600);
|
|
467
|
-
});
|
|
468
|
-
child.once("error", reject);
|
|
469
|
-
child.once("close", (code) => {
|
|
470
|
-
if (code === 0)
|
|
471
|
-
resolve();
|
|
472
|
-
else
|
|
473
|
-
reject(new Error(`${command} failed${stderr.trim() ? `: ${stderr.trim()}` : ""}`));
|
|
474
|
-
});
|
|
475
|
-
});
|
|
476
|
-
}
|
|
477
|
-
async function loadVoskWithAutoInstall(initialError, progress) {
|
|
478
|
-
const installOutput = await ensureVoskInstalled(progress, initialError);
|
|
479
|
-
const attempt = tryLoadVosk();
|
|
480
|
-
if (attempt.ok)
|
|
481
|
-
return attempt.module;
|
|
482
|
-
throw new Error(`automatic Vosk install/build finished, but bindings still cannot load: ${errorMessage(attempt.error)}${installDiagnosticSuffix(installOutput)}`);
|
|
483
|
-
}
|
|
484
|
-
async function ensureVoskInstalled(progress, initialError) {
|
|
485
|
-
if (voskInstallPromise) {
|
|
486
|
-
progress("Waiting for Vosk install already in progress...");
|
|
487
|
-
await voskInstallPromise;
|
|
488
|
-
return "";
|
|
489
|
-
}
|
|
490
|
-
voskInstallPromise = installVoskBindings(progress, initialError).finally(() => {
|
|
491
|
-
voskInstallPromise = undefined;
|
|
492
|
-
});
|
|
493
|
-
return await voskInstallPromise;
|
|
494
|
-
}
|
|
495
|
-
async function installVoskBindings(progress, initialError) {
|
|
496
|
-
progress(`Installing Vosk bindings (${VOSK_PACKAGE_SPEC})...`);
|
|
497
|
-
let installOutput = "";
|
|
498
|
-
try {
|
|
499
|
-
installOutput = await runNpmCommand(["install", "--no-save", "--package-lock=false", VOSK_PACKAGE_SPEC, "--ignore-scripts"], "Installing Vosk", progress);
|
|
500
|
-
await patchVoskNativeDependencies(progress);
|
|
501
|
-
progress("Building Vosk native dependencies...");
|
|
502
|
-
const rebuildOutput = await runNpmCommand(["rebuild", "ffi-napi", "ref-napi", "--foreground-scripts", "--ignore-scripts=false"], "Building Vosk", progress);
|
|
503
|
-
const installedAttempt = tryLoadVosk();
|
|
504
|
-
if (installedAttempt.ok)
|
|
505
|
-
return `${installOutput}\n${rebuildOutput}`;
|
|
506
|
-
if (isMissingModuleError(installedAttempt.error)) {
|
|
507
|
-
throw new Error(`npm install finished without a loadable Vosk package: ${errorMessage(installedAttempt.error)}${installDiagnosticSuffix(`${installOutput}\n${rebuildOutput}`)}`);
|
|
508
|
-
}
|
|
509
|
-
progress("Rebuilding Vosk package...");
|
|
510
|
-
const voskRebuildOutput = await runNpmCommand(["rebuild", "vosk", "--foreground-scripts", "--ignore-scripts=false"], "Building Vosk", progress);
|
|
511
|
-
return `${installOutput}\n${rebuildOutput}\n${voskRebuildOutput}`;
|
|
512
|
-
}
|
|
513
|
-
catch (error) {
|
|
514
|
-
throw new Error(`automatic Vosk install/build failed: ${errorMessage(error)} (initial load error: ${errorMessage(initialError)})`);
|
|
515
|
-
}
|
|
516
|
-
}
|
|
517
|
-
async function patchVoskNativeDependencies(progress) {
|
|
518
|
-
const headerPath = join(projectRoot, "node_modules", "get-uv-event-loop-napi-h", "include", "get-uv-event-loop-napi.h");
|
|
519
|
-
let source;
|
|
453
|
+
export function parseDeepgramTranscript(data) {
|
|
454
|
+
const source = typeof data === "string"
|
|
455
|
+
? data
|
|
456
|
+
: Buffer.isBuffer(data)
|
|
457
|
+
? data.toString("utf8")
|
|
458
|
+
: undefined;
|
|
459
|
+
if (!source)
|
|
460
|
+
return undefined;
|
|
461
|
+
let parsed;
|
|
520
462
|
try {
|
|
521
|
-
|
|
463
|
+
parsed = JSON.parse(source);
|
|
522
464
|
}
|
|
523
465
|
catch {
|
|
524
|
-
return;
|
|
525
|
-
}
|
|
526
|
-
const oldLine = "napi_get_uv_event_loop__ = &napi_get_uv_event_loop;";
|
|
527
|
-
const newLine = "napi_get_uv_event_loop__ = (get_uv_event_loop_fn)&napi_get_uv_event_loop;";
|
|
528
|
-
if (source.includes(newLine))
|
|
529
|
-
return;
|
|
530
|
-
if (!source.includes(oldLine))
|
|
531
|
-
return;
|
|
532
|
-
await writeFile(headerPath, source.replace(oldLine, newLine));
|
|
533
|
-
progress("Patched Vosk native headers for Node 24...");
|
|
534
|
-
}
|
|
535
|
-
async function runNpmCommand(args, label, progress) {
|
|
536
|
-
return await new Promise((resolve, reject) => {
|
|
537
|
-
const child = spawn(npmCommand(), args, {
|
|
538
|
-
cwd: projectRoot,
|
|
539
|
-
env: { ...process.env, npm_config_ignore_scripts: "false" },
|
|
540
|
-
stdio: ["ignore", "pipe", "pipe"],
|
|
541
|
-
});
|
|
542
|
-
let output = "";
|
|
543
|
-
const observeOutput = (chunk) => {
|
|
544
|
-
const text = chunk.toString("utf8");
|
|
545
|
-
output = `${output}${text}`.slice(-1200);
|
|
546
|
-
const line = lastUsefulLine(text);
|
|
547
|
-
if (line)
|
|
548
|
-
progress(`${label}: ${line}`);
|
|
549
|
-
};
|
|
550
|
-
child.stdout.on("data", observeOutput);
|
|
551
|
-
child.stderr.on("data", observeOutput);
|
|
552
|
-
child.once("error", reject);
|
|
553
|
-
child.once("close", (code) => {
|
|
554
|
-
if (code === 0)
|
|
555
|
-
resolve(output);
|
|
556
|
-
else
|
|
557
|
-
reject(new Error(`${npmCommand()} ${args.join(" ")} failed${output.trim() ? `: ${output.trim()}` : ""}`));
|
|
558
|
-
});
|
|
559
|
-
});
|
|
560
|
-
}
|
|
561
|
-
function isMissingModuleError(error) {
|
|
562
|
-
return /Cannot find module ['"]vosk['"]/u.test(errorMessage(error));
|
|
563
|
-
}
|
|
564
|
-
function installDiagnosticSuffix(output) {
|
|
565
|
-
const summary = compactOutputSummary(output);
|
|
566
|
-
const nodeHint = nodeVersionCompatibilityHint(output);
|
|
567
|
-
return `${summary ? ` Last npm output: ${summary}` : ""}${nodeHint ? ` ${nodeHint}` : ""}`;
|
|
568
|
-
}
|
|
569
|
-
function compactOutputSummary(output) {
|
|
570
|
-
const lines = output.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean);
|
|
571
|
-
const interesting = lines.filter((line) => /\b(error|ERR!|failed|not ok|incompatible|Cannot find|node-gyp|make:)\b/iu.test(line));
|
|
572
|
-
const selected = (interesting.length > 0 ? interesting : lines).slice(-8);
|
|
573
|
-
const summary = selected.join(" | ");
|
|
574
|
-
return summary.length > 900 ? `${summary.slice(0, 897)}...` : summary;
|
|
575
|
-
}
|
|
576
|
-
function nodeVersionCompatibilityHint(output) {
|
|
577
|
-
const nodeMajor = Number.parseInt(process.versions.node.split(".")[0] ?? "", 10);
|
|
578
|
-
if (nodeMajor >= 25 && /ffi-napi|node-gyp|napi_add_finalizer|get_uv_event_loop/iu.test(output)) {
|
|
579
|
-
return `Detected Node ${process.versions.node}; Vosk npm bindings depend on ffi-napi, which may not build on this Node version. Try running Pix with Node 22 or 24.`;
|
|
580
|
-
}
|
|
581
|
-
return undefined;
|
|
582
|
-
}
|
|
583
|
-
function lastUsefulLine(text) {
|
|
584
|
-
const lines = text.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean);
|
|
585
|
-
const line = lines.at(-1);
|
|
586
|
-
if (!line)
|
|
587
466
|
return undefined;
|
|
588
|
-
return line.length > 96 ? `${line.slice(0, 93)}...` : line;
|
|
589
|
-
}
|
|
590
|
-
function npmCommand() {
|
|
591
|
-
return process.platform === "win32" ? "npm.cmd" : "npm";
|
|
592
|
-
}
|
|
593
|
-
function tryLoadVosk() {
|
|
594
|
-
try {
|
|
595
|
-
return { ok: true, module: loadVosk() };
|
|
596
|
-
}
|
|
597
|
-
catch (error) {
|
|
598
|
-
return { ok: false, error };
|
|
599
|
-
}
|
|
600
|
-
}
|
|
601
|
-
function loadVosk() {
|
|
602
|
-
let moduleValue;
|
|
603
|
-
try {
|
|
604
|
-
moduleValue = require("vosk");
|
|
605
467
|
}
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
if (!
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
const record = value;
|
|
617
|
-
return typeof record.Model === "function" && typeof record.Recognizer === "function";
|
|
468
|
+
if (!isRecord(parsed) || parsed.type !== "Results" || !isRecord(parsed.channel))
|
|
469
|
+
return undefined;
|
|
470
|
+
const alternatives = parsed.channel.alternatives;
|
|
471
|
+
if (!Array.isArray(alternatives) || !isRecord(alternatives[0]))
|
|
472
|
+
return undefined;
|
|
473
|
+
const transcript = alternatives[0].transcript;
|
|
474
|
+
if (typeof transcript !== "string")
|
|
475
|
+
return undefined;
|
|
476
|
+
const text = normalizeTranscript(transcript);
|
|
477
|
+
return text ? { text, isFinal: parsed.is_final === true } : undefined;
|
|
618
478
|
}
|
|
619
479
|
async function selectRecorderCommand() {
|
|
620
480
|
const commands = [
|
|
@@ -653,25 +513,97 @@ async function selectRecorderCommand() {
|
|
|
653
513
|
}
|
|
654
514
|
throw new Error("audio recorder not found: install SoX (`rec`/`sox`), ffmpeg, or arecord");
|
|
655
515
|
}
|
|
656
|
-
function
|
|
657
|
-
const
|
|
658
|
-
|
|
659
|
-
|
|
516
|
+
function createDeepgramSocket(url, protocols) {
|
|
517
|
+
const WebSocketConstructor = globalThis.WebSocket;
|
|
518
|
+
if (!WebSocketConstructor)
|
|
519
|
+
throw new Error("this Node runtime does not provide WebSocket support");
|
|
520
|
+
return new WebSocketConstructor(url, protocols);
|
|
521
|
+
}
|
|
522
|
+
async function waitForSocketOpen(socket) {
|
|
523
|
+
if (socket.readyState === DEEPGRAM_SOCKET_OPEN)
|
|
524
|
+
return;
|
|
525
|
+
await new Promise((resolve, reject) => {
|
|
526
|
+
let settled = false;
|
|
527
|
+
const finish = (error) => {
|
|
528
|
+
if (settled)
|
|
529
|
+
return;
|
|
530
|
+
settled = true;
|
|
531
|
+
clearTimeout(timer);
|
|
532
|
+
socket.removeEventListener("open", onOpen);
|
|
533
|
+
socket.removeEventListener("error", onError);
|
|
534
|
+
socket.removeEventListener("close", onClose);
|
|
535
|
+
if (error)
|
|
536
|
+
reject(error);
|
|
537
|
+
else
|
|
538
|
+
resolve();
|
|
539
|
+
};
|
|
540
|
+
const onOpen = () => finish();
|
|
541
|
+
const onError = () => finish(new Error("Deepgram WebSocket connection failed"));
|
|
542
|
+
const onClose = (event) => finish(new Error(`Deepgram WebSocket closed before connecting${event.code ? ` (${event.code}${event.reason ? `: ${event.reason}` : ""})` : ""}`));
|
|
543
|
+
const timer = setTimeout(() => finish(new Error("Deepgram WebSocket connection timed out")), DEEPGRAM_SOCKET_CONNECT_TIMEOUT_MS);
|
|
544
|
+
timer.unref?.();
|
|
545
|
+
socket.addEventListener("open", onOpen);
|
|
546
|
+
socket.addEventListener("error", onError);
|
|
547
|
+
socket.addEventListener("close", onClose);
|
|
548
|
+
});
|
|
549
|
+
}
|
|
550
|
+
function safeCloseSocket(socket) {
|
|
551
|
+
try {
|
|
552
|
+
socket.close(1000, "voice input stopped");
|
|
553
|
+
}
|
|
554
|
+
catch {
|
|
555
|
+
// Best-effort cleanup.
|
|
556
|
+
}
|
|
557
|
+
}
|
|
558
|
+
async function stopRecorderProcess(audioProcess) {
|
|
559
|
+
if (audioProcess.exitCode !== null)
|
|
560
|
+
return;
|
|
561
|
+
const gracefulClose = waitForProcessClose(audioProcess, RECORDER_STOP_GRACE_MS);
|
|
562
|
+
if (!audioProcess.killed)
|
|
563
|
+
safeKillProcess(audioProcess, "SIGTERM");
|
|
564
|
+
if (await gracefulClose)
|
|
565
|
+
return;
|
|
566
|
+
if (audioProcess.exitCode === null)
|
|
567
|
+
safeKillProcess(audioProcess, "SIGKILL");
|
|
568
|
+
await waitForProcessClose(audioProcess, RECORDER_FORCE_STOP_GRACE_MS);
|
|
660
569
|
}
|
|
661
|
-
function
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
return
|
|
570
|
+
function waitForProcessClose(audioProcess, timeoutMs) {
|
|
571
|
+
if (audioProcess.exitCode !== null)
|
|
572
|
+
return Promise.resolve(true);
|
|
573
|
+
return new Promise((resolve) => {
|
|
574
|
+
let settled = false;
|
|
575
|
+
const finish = (closed) => {
|
|
576
|
+
if (settled)
|
|
577
|
+
return;
|
|
578
|
+
settled = true;
|
|
579
|
+
clearTimeout(timer);
|
|
580
|
+
audioProcess.removeListener("close", onClose);
|
|
581
|
+
resolve(closed);
|
|
582
|
+
};
|
|
583
|
+
const onClose = () => finish(true);
|
|
584
|
+
const timer = setTimeout(() => finish(false), timeoutMs);
|
|
585
|
+
timer.unref?.();
|
|
586
|
+
audioProcess.once("close", onClose);
|
|
587
|
+
});
|
|
665
588
|
}
|
|
666
|
-
function
|
|
589
|
+
function safeKillProcess(audioProcess, signal) {
|
|
667
590
|
try {
|
|
668
|
-
|
|
669
|
-
return parsed && typeof parsed === "object" ? parsed : undefined;
|
|
591
|
+
audioProcess.kill(signal);
|
|
670
592
|
}
|
|
671
593
|
catch {
|
|
672
|
-
|
|
594
|
+
// Best-effort cleanup; shutdown has its own outer deadline.
|
|
673
595
|
}
|
|
674
596
|
}
|
|
597
|
+
function normalizeTranscript(text) {
|
|
598
|
+
const normalized = text.trim().replace(/\s+/gu, " ");
|
|
599
|
+
return normalized || undefined;
|
|
600
|
+
}
|
|
601
|
+
function isRecord(value) {
|
|
602
|
+
return !!value && typeof value === "object" && !Array.isArray(value);
|
|
603
|
+
}
|
|
604
|
+
async function delay(ms) {
|
|
605
|
+
await new Promise((resolve) => setTimeout(resolve, ms));
|
|
606
|
+
}
|
|
675
607
|
function errorMessage(error) {
|
|
676
608
|
return error instanceof Error ? error.message : String(error);
|
|
677
609
|
}
|