pi-voicekit 0.2.3 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +46 -3
- package/extensions/voice/local.ts +31 -23
- package/extensions/voice/post-process-queue.ts +532 -0
- package/extensions/voice/post-process.ts +42 -21
- package/extensions/voice/sherpa-engine.ts +41 -17
- package/extensions/voice.ts +367 -55
- package/package.json +1 -1
|
@@ -432,33 +432,57 @@ export function segmentPcmForLongAudio(samples: Float32Array, sampleRate: number
|
|
|
432
432
|
return segments.length > 0 ? segments : [samples];
|
|
433
433
|
}
|
|
434
434
|
|
|
435
|
+
/**
|
|
436
|
+
* Decode VAD segments in order, calling `onSegment(text, index)` as soon as a segment's text
|
|
437
|
+
* is available and before the next segment starts decoding. Returns the non-empty trimmed
|
|
438
|
+
* texts in order — the parts `transcribeBufferSegmented` joins.
|
|
439
|
+
*
|
|
440
|
+
* The recognizer is already a parameter, so this seam can be driven with a stub in tests.
|
|
441
|
+
* `onSegment` is observational: a throwing callback is swallowed, because an observer must
|
|
442
|
+
* never cost the user their transcript.
|
|
443
|
+
*/
|
|
444
|
+
export async function decodeSegmentsInOrder(
|
|
445
|
+
recognizer: SherpaRecognizer,
|
|
446
|
+
segments: readonly Float32Array[],
|
|
447
|
+
onSegment?: (text: string, index: number) => void
|
|
448
|
+
): Promise<string[]> {
|
|
449
|
+
const parts: string[] = [];
|
|
450
|
+
for (const [index, segment] of segments.entries()) {
|
|
451
|
+
const stream = recognizer.createStream();
|
|
452
|
+
stream.acceptWaveform({ sampleRate: 16000, samples: segment });
|
|
453
|
+
await recognizer.decodeAsync(stream);
|
|
454
|
+
const r = recognizer.getResult(stream);
|
|
455
|
+
const t = (r?.text || "").trim();
|
|
456
|
+
if (onSegment) {
|
|
457
|
+
try {
|
|
458
|
+
onSegment(t, index);
|
|
459
|
+
} catch {}
|
|
460
|
+
}
|
|
461
|
+
if (t) parts.push(t);
|
|
462
|
+
}
|
|
463
|
+
return parts;
|
|
464
|
+
}
|
|
465
|
+
|
|
435
466
|
/**
|
|
436
467
|
* Transcribe PCM with automatic VAD segmentation for long recordings.
|
|
437
468
|
* Byte-identical fast path (single decode) for audio ≤ thresholdSecs.
|
|
469
|
+
*
|
|
470
|
+
* `onSegment` is called for every decoded segment — the fast path's single segment included —
|
|
471
|
+
* as soon as it decodes and before the next decode starts. Leaving it out keeps today's exact
|
|
472
|
+
* return value (docs/superpowers/specs/2026-09-26-polish-pipeline-design.md §4.1).
|
|
438
473
|
*/
|
|
439
474
|
export async function transcribeBufferSegmented(
|
|
440
475
|
pcmData: Buffer,
|
|
441
476
|
recognizer: SherpaRecognizer,
|
|
442
|
-
thresholdSecs = 10
|
|
477
|
+
thresholdSecs = 10,
|
|
478
|
+
onSegment?: (text: string, index: number) => void
|
|
443
479
|
): Promise<string> {
|
|
444
480
|
getSherpaModule();
|
|
445
481
|
const samples = pcmToFloat32(pcmData);
|
|
482
|
+
// The fast path is one segment too, so a pipelining caller sees exactly one segment and a
|
|
483
|
+
// short dictation keeps today's one-call behaviour.
|
|
446
484
|
if (samples.length / 16000 <= thresholdSecs) {
|
|
447
|
-
|
|
448
|
-
stream.acceptWaveform({ sampleRate: 16000, samples });
|
|
449
|
-
await recognizer.decodeAsync(stream);
|
|
450
|
-
const r = recognizer.getResult(stream);
|
|
451
|
-
return (r?.text || "").trim();
|
|
452
|
-
}
|
|
453
|
-
|
|
454
|
-
const parts: string[] = [];
|
|
455
|
-
for (const seg of segmentPcmForLongAudio(samples, 16000)) {
|
|
456
|
-
const stream = recognizer.createStream();
|
|
457
|
-
stream.acceptWaveform({ sampleRate: 16000, samples: seg });
|
|
458
|
-
await recognizer.decodeAsync(stream);
|
|
459
|
-
const r = recognizer.getResult(stream);
|
|
460
|
-
const t = (r?.text || "").trim();
|
|
461
|
-
if (t) parts.push(t);
|
|
485
|
+
return (await decodeSegmentsInOrder(recognizer, [samples], onSegment)).join(" ");
|
|
462
486
|
}
|
|
463
|
-
return
|
|
487
|
+
return (await decodeSegmentsInOrder(recognizer, segmentPcmForLongAudio(samples, 16000), onSegment)).join(" ");
|
|
464
488
|
}
|
package/extensions/voice.ts
CHANGED
|
@@ -94,6 +94,7 @@ import {
|
|
|
94
94
|
checkLocalServer,
|
|
95
95
|
LOCAL_MODELS,
|
|
96
96
|
DEFAULT_LOCAL_ENDPOINT,
|
|
97
|
+
DEFAULT_LOCAL_MODEL,
|
|
97
98
|
getLanguagesForLocalModel,
|
|
98
99
|
isLanguageSupportedByModel,
|
|
99
100
|
localLanguageDisplayName,
|
|
@@ -112,8 +113,11 @@ import {
|
|
|
112
113
|
polishModelOptions,
|
|
113
114
|
polishTranscript,
|
|
114
115
|
resolveModelChoice,
|
|
116
|
+
type EditorRead,
|
|
117
|
+
type PolishAuditSegments,
|
|
115
118
|
} from "./voice/post-process";
|
|
116
|
-
import {
|
|
119
|
+
import { createPolishQueue, type PolishQueue } from "./voice/post-process-queue";
|
|
120
|
+
import { DEFAULT_CONTEXT_LIMITS, type EntryLike } from "./voice/post-process-context";
|
|
117
121
|
import { polishMaxTokens } from "./voice/post-process-prompt";
|
|
118
122
|
|
|
119
123
|
/** Adapter for the real event loop — lets GapTimer run under the real setTimeout. */
|
|
@@ -774,7 +778,14 @@ export default function (pi: ExtensionAPI) {
|
|
|
774
778
|
// ─── Transcript polish (post-processing) ─────────────────────────────────
|
|
775
779
|
// One bounded pass between the final transcript and the editor write. The
|
|
776
780
|
// token below is what makes a late result inert (spec §4.1.1).
|
|
777
|
-
|
|
781
|
+
/**
|
|
782
|
+
* Why the pending pass lost ownership:
|
|
783
|
+
* - `discard` — the result must not be used, and the editor keeps the user's text.
|
|
784
|
+
* - `abort` — a newer recording or session owns the flow; touch nothing.
|
|
785
|
+
* - `raw` — `discard`'s ownership rules, but the pass may still write the recogniser's
|
|
786
|
+
* text (used when the user turns polish off mid-dictation).
|
|
787
|
+
*/
|
|
788
|
+
type PolishPassToken = { invalidated: "discard" | "abort" | "raw" | null };
|
|
778
789
|
let activePolishPass: PolishPassToken | null = null;
|
|
779
790
|
/**
|
|
780
791
|
* Editor value the pending pass started from, or null when no pass is pending. The
|
|
@@ -783,20 +794,86 @@ export default function (pi: ExtensionAPI) {
|
|
|
783
794
|
*/
|
|
784
795
|
let polishPassEditorSnapshot: string | null = null;
|
|
785
796
|
|
|
786
|
-
function invalidatePolishPass(reason: string, cleanup: "complete" | "relinquish" = "relinquish"): void {
|
|
797
|
+
function invalidatePolishPass(reason: string, cleanup: "complete" | "relinquish" | "raw" = "relinquish"): void {
|
|
787
798
|
const pass = activePolishPass;
|
|
788
799
|
if (!pass) return;
|
|
789
|
-
const disposition = cleanup === "complete" ? "discard" : "abort";
|
|
800
|
+
const disposition = cleanup === "complete" ? "discard" : cleanup === "raw" ? "raw" : "abort";
|
|
790
801
|
if (pass.invalidated !== disposition) voiceDebug("polish pass invalidated", { reason });
|
|
791
802
|
pass.invalidated = disposition;
|
|
792
|
-
// Retain a discarded pass until its tail completes, or a later teardown takes
|
|
793
|
-
// ownership
|
|
803
|
+
// Retain a discarded or raw pass until its tail completes, or a later teardown takes
|
|
804
|
+
// ownership: `raw` still has to decide the write and record the audit. The awaiting
|
|
805
|
+
// callback retains this token even after relinquishing.
|
|
794
806
|
if (cleanup === "relinquish") {
|
|
795
807
|
activePolishPass = null;
|
|
796
808
|
polishPassEditorSnapshot = null;
|
|
797
809
|
}
|
|
798
810
|
}
|
|
799
811
|
|
|
812
|
+
/**
|
|
813
|
+
* One local dictation's segmented pass: the token every segment shares, the bounded queue
|
|
814
|
+
* and the request facts the audit entry summarises. Created on the first recogniser
|
|
815
|
+
* segment, so a dictation with no segments never departs from the single-call behaviour.
|
|
816
|
+
*/
|
|
817
|
+
interface PolishQueuePass {
|
|
818
|
+
token: PolishPassToken;
|
|
819
|
+
queue: PolishQueue;
|
|
820
|
+
/** Wall clock when the first segment's polish work started. */
|
|
821
|
+
startedAt: number;
|
|
822
|
+
/** The model that actually ran, as `provider/id`, for telemetry. */
|
|
823
|
+
modelLabel: string;
|
|
824
|
+
configured: string;
|
|
825
|
+
/** What the requests carried; the queue decides sampling per segment. */
|
|
826
|
+
stats: { thinkingOff: boolean; maxTokens?: number };
|
|
827
|
+
/**
|
|
828
|
+
* The editor when the pass was created. The final write compares against this snapshot
|
|
829
|
+
* only — a fresh read at the end would mistake text the user typed while later segments
|
|
830
|
+
* were still being recognised for an unchanged editor and overwrite it.
|
|
831
|
+
*/
|
|
832
|
+
editorSnapshot: string | typeof EDITOR_READ_FAILED;
|
|
833
|
+
/** The UI captured with the pass, so a reassigned `ctx` cannot retarget its writes. */
|
|
834
|
+
ui: ExtensionContext["ui"];
|
|
835
|
+
}
|
|
836
|
+
|
|
837
|
+
/**
|
|
838
|
+
* D5 one-time disclosure and the model choice, shared by the single-call pass and the
|
|
839
|
+
* segment queue. The notice fires before any transcript can leave, and resolution never
|
|
840
|
+
* falls back (D8). The caller owns the failure notice, because only the caller knows
|
|
841
|
+
* whether the dictation still has a write path.
|
|
842
|
+
*/
|
|
843
|
+
function resolvePolishModel(): ReturnType<typeof resolveModelChoice> {
|
|
844
|
+
if (!config.postProcessNoticeShown && ctx?.hasUI) {
|
|
845
|
+
// D5: one-time disclosure. The flag is set in memory first (an assignment cannot
|
|
846
|
+
// throw); the write and the notification then sit in guards of their own. A
|
|
847
|
+
// read-only config directory must cost the flag's persistence, never the
|
|
848
|
+
// disclosure — the in-memory flag still suppresses a repeat in this process.
|
|
849
|
+
config.postProcessNoticeShown = true;
|
|
850
|
+
try {
|
|
851
|
+
// R26: field-level global write — `config` also carries this project's values, so
|
|
852
|
+
// a whole-block write would reset unrelated machine-global settings. Persist the
|
|
853
|
+
// flag BEFORE notifying, per the house rule in tts-onboarding.
|
|
854
|
+
saveGlobalVoiceFields({ postProcessNoticeShown: true });
|
|
855
|
+
} catch (error) {
|
|
856
|
+
voiceDebug("polish notice setting write failed", String(error));
|
|
857
|
+
}
|
|
858
|
+
try {
|
|
859
|
+
const turns = config.postProcessContextTurns ?? DEFAULT_CONTEXT_LIMITS.turns;
|
|
860
|
+
ctx.ui.notify(
|
|
861
|
+
[
|
|
862
|
+
"Voice polish is on: every dictation makes one extra model call,",
|
|
863
|
+
turns > 0
|
|
864
|
+
? `and the last ${turns} conversation turns are sent with it.`
|
|
865
|
+
: "and no conversation context is sent with it.",
|
|
866
|
+
"Turn it off with /voice-polish off.",
|
|
867
|
+
].join(" "),
|
|
868
|
+
"info"
|
|
869
|
+
);
|
|
870
|
+
} catch (error) {
|
|
871
|
+
voiceDebug("polish notice notification failed", String(error));
|
|
872
|
+
}
|
|
873
|
+
}
|
|
874
|
+
return resolveModelChoice(parseModelRef(config.postProcessModel), polishModelLookup, ctx?.model);
|
|
875
|
+
}
|
|
876
|
+
|
|
800
877
|
/**
|
|
801
878
|
* Ownership reads go through here: a throwing editor read is a failed read and
|
|
802
879
|
* never an unchanged editor (spec invariant 2).
|
|
@@ -887,48 +964,20 @@ export default function (pi: ExtensionAPI) {
|
|
|
887
964
|
/** Recorded so a slow or truncated pass can be diagnosed without reading code. */
|
|
888
965
|
thinkingOff?: boolean;
|
|
889
966
|
maxTokens?: number;
|
|
967
|
+
/** Per-segment outcome when the queue produced this dictation; absent for a single call. */
|
|
968
|
+
segments?: PolishAuditSegments;
|
|
890
969
|
reason?: string;
|
|
891
970
|
error?: string;
|
|
892
971
|
};
|
|
893
|
-
async function runPolishPass(raw: string, editorSnapshot:
|
|
972
|
+
async function runPolishPass(raw: string, editorSnapshot: EditorRead): Promise<PolishOutcome> {
|
|
894
973
|
const id: PolishPassToken = { invalidated: null };
|
|
895
974
|
activePolishPass = id;
|
|
896
|
-
polishPassEditorSnapshot = editorSnapshot;
|
|
897
|
-
if (!config.postProcessNoticeShown && ctx?.hasUI) {
|
|
898
|
-
// D5: one-time disclosure. The flag is set in memory first (an assignment cannot
|
|
899
|
-
// throw); the write and the notification then sit in guards of their own. A
|
|
900
|
-
// read-only config directory must cost the flag's persistence, never the
|
|
901
|
-
// disclosure — the in-memory flag still suppresses a repeat in this process.
|
|
902
|
-
config.postProcessNoticeShown = true;
|
|
903
|
-
try {
|
|
904
|
-
// R26: field-level global write — `config` also carries this project's values, so
|
|
905
|
-
// a whole-block write would reset unrelated machine-global settings. Persist the
|
|
906
|
-
// flag BEFORE notifying, per the house rule in tts-onboarding.
|
|
907
|
-
saveGlobalVoiceFields({ postProcessNoticeShown: true });
|
|
908
|
-
} catch (error) {
|
|
909
|
-
voiceDebug("polish notice setting write failed", String(error));
|
|
910
|
-
}
|
|
911
|
-
try {
|
|
912
|
-
const turns = config.postProcessContextTurns ?? DEFAULT_CONTEXT_LIMITS.turns;
|
|
913
|
-
ctx.ui.notify(
|
|
914
|
-
[
|
|
915
|
-
"Voice polish is on: every dictation makes one extra model call,",
|
|
916
|
-
turns > 0
|
|
917
|
-
? `and the last ${turns} conversation turns are sent with it.`
|
|
918
|
-
: "and no conversation context is sent with it.",
|
|
919
|
-
"Turn it off with /voice-polish off.",
|
|
920
|
-
].join(" "),
|
|
921
|
-
"info"
|
|
922
|
-
);
|
|
923
|
-
} catch (error) {
|
|
924
|
-
voiceDebug("polish notice notification failed", String(error));
|
|
925
|
-
}
|
|
926
|
-
}
|
|
975
|
+
polishPassEditorSnapshot = editorSnapshot === EDITOR_READ_FAILED ? null : editorSnapshot;
|
|
927
976
|
// R19: everything before the model call is fail-open too — a throw here
|
|
928
977
|
// (registry lookup, notify, status) must not cost the user their dictation.
|
|
929
978
|
let choice: ReturnType<typeof resolveModelChoice>;
|
|
930
979
|
try {
|
|
931
|
-
choice =
|
|
980
|
+
choice = resolvePolishModel();
|
|
932
981
|
} catch (err) {
|
|
933
982
|
activePolishPass = null;
|
|
934
983
|
polishPassEditorSnapshot = null;
|
|
@@ -980,10 +1029,9 @@ export default function (pi: ExtensionAPI) {
|
|
|
980
1029
|
{
|
|
981
1030
|
signal,
|
|
982
1031
|
maxTokens: request.maxTokens,
|
|
983
|
-
// Measured 2026-09-26: without this a reasoning model
|
|
984
|
-
//
|
|
985
|
-
|
|
986
|
-
...polishSamplingOptions(model as { reasoning?: boolean }, raw.length),
|
|
1032
|
+
// Measured 2026-09-26: without this a reasoning model can spend the whole budget thinking,
|
|
1033
|
+
// truncating the answer on any dictation — see polishSamplingOptions.
|
|
1034
|
+
...polishSamplingOptions(model as { reasoning?: boolean }),
|
|
987
1035
|
}
|
|
988
1036
|
),
|
|
989
1037
|
debug: (reason, data) => voiceDebug(`polish ${reason}`, data),
|
|
@@ -993,7 +1041,7 @@ export default function (pi: ExtensionAPI) {
|
|
|
993
1041
|
configured: choice.ref,
|
|
994
1042
|
// Pure and cheap, so computing it twice (here and in the call options) is fine, and it
|
|
995
1043
|
// keeps the audit entry honest about what the pass decided.
|
|
996
|
-
thinkingOff: Boolean(polishSamplingOptions(model as { reasoning?: boolean }
|
|
1044
|
+
thinkingOff: Boolean(polishSamplingOptions(model as { reasoning?: boolean }).samplingParams),
|
|
997
1045
|
maxTokens: polishMaxTokens(raw.length),
|
|
998
1046
|
status: result.status,
|
|
999
1047
|
ms: Date.now() - started,
|
|
@@ -1006,19 +1054,27 @@ export default function (pi: ExtensionAPI) {
|
|
|
1006
1054
|
return { action: "abort", text: raw };
|
|
1007
1055
|
}
|
|
1008
1056
|
const decision = decideApply({
|
|
1009
|
-
|
|
1057
|
+
// `raw` is the polish-off disposition: the rewrite is never used, but the dictation
|
|
1058
|
+
// still owns its write decision and keeps the recogniser's text.
|
|
1059
|
+
tokenCurrent: id.invalidated === null || id.invalidated === "raw",
|
|
1010
1060
|
editorSnapshot,
|
|
1011
1061
|
currentEditor: readEditorOrFailed(),
|
|
1012
1062
|
});
|
|
1063
|
+
const rawOnly = id.invalidated === "raw";
|
|
1013
1064
|
// The caller finalizes telemetry after the editor write, not at this decision.
|
|
1014
|
-
const pendingTelemetry = {
|
|
1065
|
+
const pendingTelemetry = {
|
|
1066
|
+
...telemetry,
|
|
1067
|
+
...(rawOnly ? { status: "skipped" } : {}),
|
|
1068
|
+
reason: decision.apply ? (rawOnly ? "polish-off" : result.reason) : decision.reason,
|
|
1069
|
+
};
|
|
1015
1070
|
if (!decision.apply) {
|
|
1016
1071
|
return { action: "discard", text: raw, status: "discarded", telemetry: pendingTelemetry };
|
|
1017
1072
|
}
|
|
1073
|
+
const applied = !rawOnly && result.status === "applied";
|
|
1018
1074
|
return {
|
|
1019
1075
|
action: "apply",
|
|
1020
|
-
text:
|
|
1021
|
-
status:
|
|
1076
|
+
text: applied ? result.text : raw,
|
|
1077
|
+
status: applied ? "applied" : "failed",
|
|
1022
1078
|
telemetry: pendingTelemetry,
|
|
1023
1079
|
};
|
|
1024
1080
|
} catch (err) {
|
|
@@ -1040,7 +1096,8 @@ export default function (pi: ExtensionAPI) {
|
|
|
1040
1096
|
return { action: "abort", text: raw };
|
|
1041
1097
|
}
|
|
1042
1098
|
const decision = decideApply({
|
|
1043
|
-
|
|
1099
|
+
// A polish-off pass still owns the raw-text write; a discarded or relinquished one does not.
|
|
1100
|
+
tokenCurrent: id.invalidated === null || id.invalidated === "raw",
|
|
1044
1101
|
editorSnapshot,
|
|
1045
1102
|
currentEditor: readEditorOrFailed(),
|
|
1046
1103
|
});
|
|
@@ -1069,6 +1126,201 @@ export default function (pi: ExtensionAPI) {
|
|
|
1069
1126
|
}
|
|
1070
1127
|
}
|
|
1071
1128
|
|
|
1129
|
+
/**
|
|
1130
|
+
* Create the bounded segment queue for one local, in-process dictation. Called when the
|
|
1131
|
+
* first non-empty recogniser segment decodes, so a silent or endpoint-backed dictation
|
|
1132
|
+
* never creates a queue and keeps `runPolishPass` exactly as it is.
|
|
1133
|
+
*/
|
|
1134
|
+
function createLocalPolishQueuePass(): PolishQueuePass | null {
|
|
1135
|
+
// Same gate as the onDone polish block: with polish off (or no UI) no call may fire,
|
|
1136
|
+
// and the dictation stays exactly as it was before the pipeline existed.
|
|
1137
|
+
if (!ctx?.hasUI || config.postProcessEnabled === false) return null;
|
|
1138
|
+
let choice: ReturnType<typeof resolveModelChoice>;
|
|
1139
|
+
try {
|
|
1140
|
+
choice = resolvePolishModel();
|
|
1141
|
+
} catch (err) {
|
|
1142
|
+
// The raw fallback, and the notice, are decided once in runPolishPass.
|
|
1143
|
+
voiceDebug("polish model resolution threw — using the raw transcript", { error: String(err) });
|
|
1144
|
+
return null;
|
|
1145
|
+
}
|
|
1146
|
+
if (!choice.model) {
|
|
1147
|
+
voiceDebug("polish skipped", { ref: choice.ref, reason: choice.reason });
|
|
1148
|
+
return null;
|
|
1149
|
+
}
|
|
1150
|
+
// Read the session context before any pass state is claimed: a throw here leaves the
|
|
1151
|
+
// caller's buffer intact, so a later segment can still create the pass.
|
|
1152
|
+
let entries: readonly EntryLike[];
|
|
1153
|
+
try {
|
|
1154
|
+
entries = ctx.sessionManager.buildContextEntries();
|
|
1155
|
+
} catch (err) {
|
|
1156
|
+
voiceDebug("polish context build threw — keeping the segments buffered", { error: String(err) });
|
|
1157
|
+
return null;
|
|
1158
|
+
}
|
|
1159
|
+
const model = choice.model;
|
|
1160
|
+
// Pin everything this pass needs for the whole dictation: `ctx` is reassigned on every
|
|
1161
|
+
// command and session event, and a call queued for this dictation must never land in a
|
|
1162
|
+
// newer context.
|
|
1163
|
+
const modelRegistry = ctx.modelRegistry;
|
|
1164
|
+
const ui = ctx.ui;
|
|
1165
|
+
const id: PolishPassToken = { invalidated: null };
|
|
1166
|
+
activePolishPass = id;
|
|
1167
|
+
let editorSnapshot: string | typeof EDITOR_READ_FAILED;
|
|
1168
|
+
try {
|
|
1169
|
+
editorSnapshot = ui.getEditorText?.() ?? "";
|
|
1170
|
+
} catch (err) {
|
|
1171
|
+
// A failed read is not an unchanged editor: the pass keeps the marker and the write
|
|
1172
|
+
// decision discards, so nothing can overwrite text we could not read (invariant 2).
|
|
1173
|
+
voiceDebug("polish editor snapshot read threw — the pass will discard its write", { error: String(err) });
|
|
1174
|
+
editorSnapshot = EDITOR_READ_FAILED;
|
|
1175
|
+
}
|
|
1176
|
+
polishPassEditorSnapshot = editorSnapshot === EDITOR_READ_FAILED ? null : editorSnapshot;
|
|
1177
|
+
const stats: PolishQueuePass["stats"] = { thinkingOff: false };
|
|
1178
|
+
const queue = createPolishQueue({
|
|
1179
|
+
timeoutMs: config.postProcessTimeoutMs ?? 8000,
|
|
1180
|
+
entries,
|
|
1181
|
+
limits: {
|
|
1182
|
+
turns: config.postProcessContextTurns ?? DEFAULT_CONTEXT_LIMITS.turns,
|
|
1183
|
+
perEntryChars: DEFAULT_CONTEXT_LIMITS.perEntryChars,
|
|
1184
|
+
totalChars: DEFAULT_CONTEXT_LIMITS.totalChars,
|
|
1185
|
+
},
|
|
1186
|
+
model: model as { reasoning?: boolean },
|
|
1187
|
+
isCurrent: () => activePolishPass === id && id.invalidated === null,
|
|
1188
|
+
call: (request, signal) => {
|
|
1189
|
+
// The queue decides this segment's sampling; record what actually left so the one
|
|
1190
|
+
// audit entry per dictation can summarise it.
|
|
1191
|
+
if (request.samplingParams) stats.thinkingOff = true;
|
|
1192
|
+
stats.maxTokens = Math.max(stats.maxTokens ?? 0, request.maxTokens);
|
|
1193
|
+
return modelRegistry.complete(
|
|
1194
|
+
model as never,
|
|
1195
|
+
{ systemPrompt: request.systemPrompt, messages: request.messages as never },
|
|
1196
|
+
{
|
|
1197
|
+
signal,
|
|
1198
|
+
maxTokens: request.maxTokens,
|
|
1199
|
+
// Forward the gate, not the whole request, so the queue's decision reaches the
|
|
1200
|
+
// transport exactly as the single-call path's does.
|
|
1201
|
+
...(request.samplingParams ? { samplingParams: request.samplingParams } : {}),
|
|
1202
|
+
}
|
|
1203
|
+
);
|
|
1204
|
+
},
|
|
1205
|
+
debug: (reason, data) => voiceDebug(`polish ${reason}`, data),
|
|
1206
|
+
});
|
|
1207
|
+
return {
|
|
1208
|
+
token: id,
|
|
1209
|
+
queue,
|
|
1210
|
+
startedAt: Date.now(),
|
|
1211
|
+
modelLabel: polishModelLabel(choice),
|
|
1212
|
+
configured: choice.ref,
|
|
1213
|
+
stats,
|
|
1214
|
+
editorSnapshot,
|
|
1215
|
+
ui,
|
|
1216
|
+
};
|
|
1217
|
+
}
|
|
1218
|
+
|
|
1219
|
+
/**
|
|
1220
|
+
* The ownership decision every queue outcome goes through. The editor must still match the
|
|
1221
|
+
* snapshot the pass started from — never a fresh read, which would mistake text the user
|
|
1222
|
+
* typed while later segments were recognised for an unchanged editor (review finding 1).
|
|
1223
|
+
* A snapshot that could not be read is a deliberate discard (spec invariant 2).
|
|
1224
|
+
*/
|
|
1225
|
+
function decideQueueWrite(pass: PolishQueuePass): { apply: boolean; reason?: string } {
|
|
1226
|
+
const snapshot = pass.editorSnapshot;
|
|
1227
|
+
if (snapshot === EDITOR_READ_FAILED) return { apply: false, reason: "editor-unreadable" };
|
|
1228
|
+
return decideApply({
|
|
1229
|
+
// `raw` keeps the write path alive; a discarded or relinquished pass may not write.
|
|
1230
|
+
tokenCurrent: pass.token.invalidated === null || pass.token.invalidated === "raw",
|
|
1231
|
+
editorSnapshot: snapshot,
|
|
1232
|
+
currentEditor: readEditorOrFailed(),
|
|
1233
|
+
});
|
|
1234
|
+
}
|
|
1235
|
+
|
|
1236
|
+
/**
|
|
1237
|
+
* Finish the segmented pass once recognition is done: await every segment, then decide the
|
|
1238
|
+
* write with the same ownership rules as the single call. Fail-open throughout: a queue
|
|
1239
|
+
* that never received a segment delegates to `runPolishPass`, and a failed segment keeps
|
|
1240
|
+
* its own raw text while its neighbours keep theirs.
|
|
1241
|
+
*/
|
|
1242
|
+
async function finishPolishQueuePass(pass: PolishQueuePass, raw: string): Promise<PolishOutcome> {
|
|
1243
|
+
const id = pass.token;
|
|
1244
|
+
const started = pass.startedAt;
|
|
1245
|
+
if (activePolishPass === id) {
|
|
1246
|
+
try {
|
|
1247
|
+
pass.ui.setStatus("voice", "polishing…");
|
|
1248
|
+
} catch (err) {
|
|
1249
|
+
voiceDebug("polish status write threw", { error: String(err) });
|
|
1250
|
+
}
|
|
1251
|
+
}
|
|
1252
|
+
try {
|
|
1253
|
+
const result = await pass.queue.finish();
|
|
1254
|
+
if (result.segments.length === 0 && id.invalidated === null) {
|
|
1255
|
+
// Defensive: the caller pushes a segment in the same tick the pass is created, so a
|
|
1256
|
+
// queue with no segments should not exist. If it does, keep today's single-call
|
|
1257
|
+
// behaviour; an invalidated pass never opens a new request.
|
|
1258
|
+
if (activePolishPass === id) {
|
|
1259
|
+
activePolishPass = null;
|
|
1260
|
+
polishPassEditorSnapshot = null;
|
|
1261
|
+
}
|
|
1262
|
+
return await runPolishPass(raw, readEditorOrFailed());
|
|
1263
|
+
}
|
|
1264
|
+
// `raw` is the polish-off disposition: queued work stops and the polished text must
|
|
1265
|
+
// not be used, but the dictation still gets its raw transcript and its audit entry.
|
|
1266
|
+
const polishOff = id.invalidated === "raw";
|
|
1267
|
+
const failureReason = polishOff
|
|
1268
|
+
? "polish-off"
|
|
1269
|
+
: result.segments.find((segment) => segment.reason !== undefined)?.reason;
|
|
1270
|
+
const telemetry: PolishTelemetry = {
|
|
1271
|
+
model: pass.modelLabel,
|
|
1272
|
+
configured: pass.configured,
|
|
1273
|
+
status: polishOff
|
|
1274
|
+
? "skipped"
|
|
1275
|
+
: result.polished > 0
|
|
1276
|
+
? "applied"
|
|
1277
|
+
: failureReason === "invalidated"
|
|
1278
|
+
? "skipped"
|
|
1279
|
+
: "rejected",
|
|
1280
|
+
ms: Date.now() - started,
|
|
1281
|
+
thinkingOff: pass.stats.thinkingOff,
|
|
1282
|
+
segments: {
|
|
1283
|
+
count: result.segments.length,
|
|
1284
|
+
polished: result.polished,
|
|
1285
|
+
failed: result.failed,
|
|
1286
|
+
retried: result.retried,
|
|
1287
|
+
},
|
|
1288
|
+
};
|
|
1289
|
+
if (pass.stats.maxTokens !== undefined) telemetry.maxTokens = pass.stats.maxTokens;
|
|
1290
|
+
// A newer recording or session owns the editor now: change nothing at all.
|
|
1291
|
+
if (activePolishPass !== id) {
|
|
1292
|
+
voiceDebug("polish result", { ...telemetry, disposition: "aborted", reason: "invalidated" });
|
|
1293
|
+
return { action: "abort", text: raw };
|
|
1294
|
+
}
|
|
1295
|
+
const decision = decideQueueWrite(pass);
|
|
1296
|
+
const pendingTelemetry = { ...telemetry, reason: decision.apply ? failureReason : decision.reason };
|
|
1297
|
+
if (!decision.apply) {
|
|
1298
|
+
return { action: "discard", text: raw, status: "discarded", telemetry: pendingTelemetry };
|
|
1299
|
+
}
|
|
1300
|
+
const polished = !polishOff && result.polished > 0;
|
|
1301
|
+
return {
|
|
1302
|
+
action: "apply",
|
|
1303
|
+
// One failed segment must not cost its neighbours their polish; when every segment
|
|
1304
|
+
// fell back, or polish was turned off, the recogniser's own join is the raw transcript.
|
|
1305
|
+
text: polished ? result.text : raw,
|
|
1306
|
+
status: polished ? "applied" : "failed",
|
|
1307
|
+
telemetry: pendingTelemetry,
|
|
1308
|
+
};
|
|
1309
|
+
} finally {
|
|
1310
|
+
// R20: a stale pass must not restore its status text over the flow that
|
|
1311
|
+
// replaced it — and a cosmetic status write is never allowed to throw.
|
|
1312
|
+
if (activePolishPass === id) {
|
|
1313
|
+
activePolishPass = null;
|
|
1314
|
+
polishPassEditorSnapshot = null;
|
|
1315
|
+
try {
|
|
1316
|
+
updateVoiceStatus();
|
|
1317
|
+
} catch (err) {
|
|
1318
|
+
voiceDebug("polish status restore threw", { error: String(err) });
|
|
1319
|
+
}
|
|
1320
|
+
}
|
|
1321
|
+
}
|
|
1322
|
+
}
|
|
1323
|
+
|
|
1072
1324
|
let preRecordingSession: StreamingSession | null = null; // Started during warmup, promoted on confirm (Deepgram only)
|
|
1073
1325
|
|
|
1074
1326
|
let lastStopTime = 0; // For Escape-to-clear-editor within 30s of recording
|
|
@@ -1664,6 +1916,16 @@ export default function (pi: ExtensionAPI) {
|
|
|
1664
1916
|
});
|
|
1665
1917
|
setVoiceState("recording");
|
|
1666
1918
|
|
|
1919
|
+
// This dictation's segmented polish state, local in-process only. Both live in this
|
|
1920
|
+
// recording's closure, so a late segment from an aborted session can never land in the
|
|
1921
|
+
// next dictation's queue.
|
|
1922
|
+
let localPolishSession: LocalSession | null = null;
|
|
1923
|
+
let localPolishQueue: PolishQueuePass | null = null;
|
|
1924
|
+
// Every recognised segment, in arrival order, until the queue owns them. The buffer is
|
|
1925
|
+
// what lets a queue created after the first segment cover the whole dictation instead of
|
|
1926
|
+
// only its tail (a model or context failure on an early segment must cost no content).
|
|
1927
|
+
const localSegments: { index: number; text: string }[] = [];
|
|
1928
|
+
|
|
1667
1929
|
// ── Callbacks for the active recording session ──
|
|
1668
1930
|
const recordingCallbacks = {
|
|
1669
1931
|
onTranscript: (interim: string, finals: string[]) => {
|
|
@@ -1671,6 +1933,37 @@ export default function (pi: ExtensionAPI) {
|
|
|
1671
1933
|
updateLiveTranscriptWidget(interim, finals);
|
|
1672
1934
|
updateVoiceStatus();
|
|
1673
1935
|
},
|
|
1936
|
+
onSegment: (text: string, index: number) => {
|
|
1937
|
+
// Local in-process recognition only, and only while the session is live: an aborted
|
|
1938
|
+
// session's late segments must not open calls for a pass nobody owns.
|
|
1939
|
+
if (!localPolishSession || localPolishSession.closed) return;
|
|
1940
|
+
if (!text.trim()) return;
|
|
1941
|
+
localSegments.push({ index, text });
|
|
1942
|
+
const pass = localPolishQueue;
|
|
1943
|
+
if (pass) {
|
|
1944
|
+
// Checked between segments as well, so a cancelled pass stops opening calls it can
|
|
1945
|
+
// never write.
|
|
1946
|
+
if (pass.token.invalidated !== null) return;
|
|
1947
|
+
pass.queue.push(index, text);
|
|
1948
|
+
return;
|
|
1949
|
+
}
|
|
1950
|
+
// First non-empty segment: try to create the pass. A failure (model unavailable, the
|
|
1951
|
+
// context read threw) is retried on the next segment, and the buffer above keeps every
|
|
1952
|
+
// earlier segment so the queue that eventually succeeds still covers all of them.
|
|
1953
|
+
let created: PolishQueuePass | null = null;
|
|
1954
|
+
try {
|
|
1955
|
+
created = createLocalPolishQueuePass();
|
|
1956
|
+
} catch (err) {
|
|
1957
|
+
voiceDebug("polish queue creation threw — keeping the segments buffered", { error: String(err) });
|
|
1958
|
+
return;
|
|
1959
|
+
}
|
|
1960
|
+
if (!created) return;
|
|
1961
|
+
localPolishQueue = created;
|
|
1962
|
+
for (const buffered of localSegments) {
|
|
1963
|
+
if (created.token.invalidated !== null) break;
|
|
1964
|
+
created.queue.push(buffered.index, buffered.text);
|
|
1965
|
+
}
|
|
1966
|
+
},
|
|
1674
1967
|
onDone: async (fullText: string, meta: { hadAudio: boolean; hadSpeech: boolean }) => {
|
|
1675
1968
|
voiceDebug("onDone callback", { fullText: fullText.slice(0, 100), meta, voiceState, spaceConsumed });
|
|
1676
1969
|
activeSession = null;
|
|
@@ -1719,7 +2012,11 @@ export default function (pi: ExtensionAPI) {
|
|
|
1719
2012
|
let skipWrite = false;
|
|
1720
2013
|
let polishOutcome: PolishOutcomeStatus | undefined;
|
|
1721
2014
|
let polishTelemetry: PolishTelemetry | undefined;
|
|
1722
|
-
|
|
2015
|
+
// A queue created before polish was switched off still owes the dictation its
|
|
2016
|
+
// ownership decision and its audit entry: the current switch cannot bypass a pass
|
|
2017
|
+
// that already made (and paid for) requests (review finding 4).
|
|
2018
|
+
const queuePass = localPolishQueue;
|
|
2019
|
+
if (ctx?.hasUI && (queuePass !== null || config.postProcessEnabled !== false)) {
|
|
1723
2020
|
// R18: the streaming transport can finalize itself (ws.onclose /
|
|
1724
2021
|
// finalizeTimer) without going through stopVoiceRecording, so the state
|
|
1725
2022
|
// may still be "recording" here. Hold the pass inside the finalizing
|
|
@@ -1739,13 +2036,14 @@ export default function (pi: ExtensionAPI) {
|
|
|
1739
2036
|
}
|
|
1740
2037
|
let outcome: PolishOutcome;
|
|
1741
2038
|
try {
|
|
1742
|
-
outcome =
|
|
2039
|
+
outcome = queuePass
|
|
2040
|
+
? await finishPolishQueuePass(queuePass, fullText)
|
|
2041
|
+
: await runPolishPass(fullText, readEditorOrFailed());
|
|
1743
2042
|
} catch (err) {
|
|
1744
2043
|
// Item 3: the pass decides its own throws by ownership; only a failure that
|
|
1745
|
-
// never reached that decision lands here
|
|
1746
|
-
//
|
|
1747
|
-
//
|
|
1748
|
-
// The dictation is still recorded and the completion tail still runs.
|
|
2044
|
+
// never reached that decision lands here. Ownership was never established, so the
|
|
2045
|
+
// raw text may not overwrite the editor: discard. The dictation is still recorded
|
|
2046
|
+
// and the completion tail still runs.
|
|
1749
2047
|
invalidatePolishPass("pass-threw");
|
|
1750
2048
|
voiceDebug("polish pass threw before ownership — discarding the editor write", {
|
|
1751
2049
|
error: String(err),
|
|
@@ -1832,6 +2130,11 @@ export default function (pi: ExtensionAPI) {
|
|
|
1832
2130
|
maxTokens: polishTelemetry?.maxTokens,
|
|
1833
2131
|
durationSec: Number(elapsed),
|
|
1834
2132
|
backend: config.backend,
|
|
2133
|
+
// The local model in effect, with the same fallback the transcriber applies; omitted
|
|
2134
|
+
// for cloud backends, which have no local model id to group by.
|
|
2135
|
+
recognizer: config.backend === "local" ? config.localModel || DEFAULT_LOCAL_MODEL : undefined,
|
|
2136
|
+
// One summary of the segmented pass; absent when a single call produced the text.
|
|
2137
|
+
segments: polishTelemetry?.segments,
|
|
1835
2138
|
written: wroteEditor ? finalText : undefined,
|
|
1836
2139
|
status: final.status,
|
|
1837
2140
|
disposition: final.disposition,
|
|
@@ -1960,6 +2263,10 @@ export default function (pi: ExtensionAPI) {
|
|
|
1960
2263
|
}
|
|
1961
2264
|
hideWidget();
|
|
1962
2265
|
|
|
2266
|
+
// A queue can already be in flight when transcription fails: invalidate its shared
|
|
2267
|
+
// pass id so no late segment can write or open further calls.
|
|
2268
|
+
invalidatePolishPass("recording-error");
|
|
2269
|
+
|
|
1963
2270
|
// ── STOP THE LOOP ──
|
|
1964
2271
|
// On error, fully reset ALL hold state AND set a cooldown
|
|
1965
2272
|
// so incoming key-repeat events can't re-trigger activation.
|
|
@@ -1992,6 +2299,7 @@ export default function (pi: ExtensionAPI) {
|
|
|
1992
2299
|
voiceDebug(`${audioTool.name} stderr:`, msg);
|
|
1993
2300
|
});
|
|
1994
2301
|
session = startLocalSession(recProc, recordingCallbacks);
|
|
2302
|
+
localPolishSession = session;
|
|
1995
2303
|
|
|
1996
2304
|
// Feed audio level meter for waveform animation
|
|
1997
2305
|
recProc.stdout?.on("data", (chunk: Buffer) => {
|
|
@@ -4065,6 +4373,10 @@ export default function (pi: ExtensionAPI) {
|
|
|
4065
4373
|
return;
|
|
4066
4374
|
}
|
|
4067
4375
|
config.postProcessEnabled = next;
|
|
4376
|
+
// Stop an in-flight pass now. A queue that already sent segments still finishes its
|
|
4377
|
+
// ownership decision and writes its audit entry (with the raw text), but no further
|
|
4378
|
+
// request may leave once polish is off (review finding 4).
|
|
4379
|
+
if (!next) invalidatePolishPass("polish-off", "raw");
|
|
4068
4380
|
cmdCtx.ui.notify(`Voice polish ${next ? "enabled" : "disabled"}.`, "info");
|
|
4069
4381
|
return;
|
|
4070
4382
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pi-voicekit",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.3.1",
|
|
4
4
|
"description": "Voice in + voice out for Pi CLI — hold-to-talk STT (Deepgram streaming or 21 offline models) plus TTS (Kitten Nano, Piper, Kokoro, or Deepgram Aura)",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"keywords": [
|