pi-meta-oauth 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -3
- package/extensions/voice/windows-audio.cs +281 -0
- package/extensions/voice/windows-audio.ps1 +195 -0
- package/extensions/voice.ts +153 -12
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -11,7 +11,7 @@ Meta Model API OAuth and Muse-style voice input for [pi](https://pi.dev).
|
|
|
11
11
|
- Device authorization against `https://auth.meta.com`
|
|
12
12
|
- Model API-key minting through `POST https://api.meta.ai/muse-code/key`
|
|
13
13
|
- Dynamic Muse model catalog from `GET https://api.meta.ai/v1/models`
|
|
14
|
-
- Toggle-based Meta voice dictation on macOS with a live green input meter
|
|
14
|
+
- Toggle-based Meta voice dictation on macOS and Windows with a live green input meter
|
|
15
15
|
|
|
16
16
|
## Install
|
|
17
17
|
|
|
@@ -41,7 +41,7 @@ The access key is re-minted daily.
|
|
|
41
41
|
|
|
42
42
|
## Voice input
|
|
43
43
|
|
|
44
|
-
Voice mode
|
|
44
|
+
Voice mode runs on **macOS and Windows** (Linux is not yet supported). It uses the same Meta credential managed by this provider; a separate Muse login is not required.
|
|
45
45
|
|
|
46
46
|
1. Press **Alt+V** once to start recording.
|
|
47
47
|
2. Speak while the green microphone meter is visible in Pi's status bar.
|
|
@@ -54,7 +54,11 @@ Commands:
|
|
|
54
54
|
- `/voice-on` — enable the Alt+V shortcut
|
|
55
55
|
- `/voice-off` — disable voice input
|
|
56
56
|
|
|
57
|
-
|
|
57
|
+
Audio is captured only between the two Alt+V presses and streamed as 16 kHz mono PCM to Muse Code's internal Meta ASR endpoint. No local speech-recognition model is downloaded. The endpoint is undocumented and may change in a future Muse release.
|
|
58
|
+
|
|
59
|
+
**macOS:** uses `AVFoundation` via a Swift helper compiled with `xcrun swiftc` (first recording triggers the system microphone permission prompt).
|
|
60
|
+
|
|
61
|
+
**Windows:** uses `winmm.dll` `waveIn*` via a C# helper. On first run pi tries to compile `extensions/voice/windows-audio.cs` with `csc.exe` (in-box .NET Framework) to `~/.pi/agent/bin/pi-meta-oauth-voice-v1.exe` for best performance. If no compiler is found it falls back to `extensions/voice/windows-audio.ps1` executed with `powershell.exe -ExecutionPolicy Bypass` (PowerShell 5.1 inbox, or `pwsh` 7 if present) which compiles the same capture code at runtime via `Add-Type`. No extra install is required. If recording fails, check **Settings → Privacy & security → Microphone** that access is allowed for desktop apps.
|
|
58
62
|
|
|
59
63
|
Optional overrides:
|
|
60
64
|
|
|
@@ -0,0 +1,281 @@
|
|
|
1
|
+
// Windows microphone helper for pi-meta-oauth voice.
|
|
2
|
+
// Captures 16 kHz mono PCM via winmm.dll waveIn* and streams base64 JSON lines
|
|
3
|
+
// Protocol mirrors macos-audio.swift:
|
|
4
|
+
// -> stdout: {"type":"ready","sampleRate":16000,"channels":1,"encoding":"pcm_s16le"}
|
|
5
|
+
// -> stdout: {"type":"audio","data":"<base64 s16le>"} per captured chunk
|
|
6
|
+
// -> stdout: {"type":"stopped"}
|
|
7
|
+
// -> stdout: {"type":"error","message":"..."}
|
|
8
|
+
// <- stdin: "stop\n" to end capture.
|
|
9
|
+
// Built with: csc.exe /nologo /target:exe /out:pi-meta-oauth-voice-v1.exe windows-audio.cs
|
|
10
|
+
// or: dotnet build via csc.dll
|
|
11
|
+
// No external dependencies beyond winmm.dll and System.dll.
|
|
12
|
+
|
|
13
|
+
using System;
|
|
14
|
+
using System.Text;
|
|
15
|
+
using System.Threading;
|
|
16
|
+
using System.Runtime.InteropServices;
|
|
17
|
+
|
|
18
|
+
class WinVoice
|
|
19
|
+
{
|
|
20
|
+
// ----- winmm interop -----
|
|
21
|
+
[StructLayout(LayoutKind.Sequential)]
|
|
22
|
+
struct WAVEFORMATEX
|
|
23
|
+
{
|
|
24
|
+
public ushort wFormatTag;
|
|
25
|
+
public ushort nChannels;
|
|
26
|
+
public uint nSamplesPerSec;
|
|
27
|
+
public uint nAvgBytesPerSec;
|
|
28
|
+
public ushort nBlockAlign;
|
|
29
|
+
public ushort wBitsPerSample;
|
|
30
|
+
public ushort cbSize;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
[StructLayout(LayoutKind.Sequential)]
|
|
34
|
+
struct WAVEHDR
|
|
35
|
+
{
|
|
36
|
+
public IntPtr lpData;
|
|
37
|
+
public uint dwBufferLength;
|
|
38
|
+
public uint dwBytesRecorded;
|
|
39
|
+
public IntPtr dwUser;
|
|
40
|
+
public uint dwFlags;
|
|
41
|
+
public uint dwLoops;
|
|
42
|
+
public IntPtr lpNext;
|
|
43
|
+
public IntPtr reserved;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
delegate void WaveInProc(IntPtr hwi, uint uMsg, IntPtr dwInstance, IntPtr dwParam1, IntPtr dwParam2);
|
|
47
|
+
|
|
48
|
+
[DllImport("winmm.dll")]
|
|
49
|
+
static extern int waveInGetNumDevs();
|
|
50
|
+
|
|
51
|
+
[DllImport("winmm.dll", CharSet = CharSet.Auto)]
|
|
52
|
+
static extern int waveInGetErrorText(int mmrError, StringBuilder pszText, int cchText);
|
|
53
|
+
|
|
54
|
+
[DllImport("winmm.dll")]
|
|
55
|
+
static extern int waveInOpen(out IntPtr phwi, int uDeviceID, ref WAVEFORMATEX pwfx, WaveInProc dwCallback, IntPtr dwInstance, uint fdwOpen);
|
|
56
|
+
|
|
57
|
+
[DllImport("winmm.dll")]
|
|
58
|
+
static extern int waveInPrepareHeader(IntPtr hwi, IntPtr pwh, int cbwh);
|
|
59
|
+
|
|
60
|
+
[DllImport("winmm.dll")]
|
|
61
|
+
static extern int waveInUnprepareHeader(IntPtr hwi, IntPtr pwh, int cbwh);
|
|
62
|
+
|
|
63
|
+
[DllImport("winmm.dll")]
|
|
64
|
+
static extern int waveInAddBuffer(IntPtr hwi, IntPtr pwh, int cbwh);
|
|
65
|
+
|
|
66
|
+
[DllImport("winmm.dll")]
|
|
67
|
+
static extern int waveInStart(IntPtr hwi);
|
|
68
|
+
|
|
69
|
+
[DllImport("winmm.dll")]
|
|
70
|
+
static extern int waveInStop(IntPtr hwi);
|
|
71
|
+
|
|
72
|
+
[DllImport("winmm.dll")]
|
|
73
|
+
static extern int waveInReset(IntPtr hwi);
|
|
74
|
+
|
|
75
|
+
[DllImport("winmm.dll")]
|
|
76
|
+
static extern int waveInClose(IntPtr hwi);
|
|
77
|
+
|
|
78
|
+
const uint CALLBACK_FUNCTION = 0x00030000;
|
|
79
|
+
const int WAVE_MAPPER = -1;
|
|
80
|
+
const ushort WAVE_FORMAT_PCM = 1;
|
|
81
|
+
const uint WIM_DATA = 0x3C0;
|
|
82
|
+
|
|
83
|
+
const int BUFFER_BYTES = 3200; // 100 ms @ 16 kHz mono s16le
|
|
84
|
+
const int BUFFER_COUNT = 3;
|
|
85
|
+
|
|
86
|
+
static readonly object emitLock = new object();
|
|
87
|
+
static WaveInProc callbackDelegate; // keep alive
|
|
88
|
+
static IntPtr hWaveIn = IntPtr.Zero;
|
|
89
|
+
static IntPtr[] headerPtrs = new IntPtr[BUFFER_COUNT];
|
|
90
|
+
static IntPtr[] bufferPtrs = new IntPtr[BUFFER_COUNT];
|
|
91
|
+
static volatile bool stopping = false;
|
|
92
|
+
static volatile bool finished = false;
|
|
93
|
+
static ManualResetEvent done = new ManualResetEvent(false);
|
|
94
|
+
|
|
95
|
+
static void Emit(string type, string extraJson = null)
|
|
96
|
+
{
|
|
97
|
+
string line;
|
|
98
|
+
if (extraJson == null) line = "{\"type\":\"" + type + "\"}";
|
|
99
|
+
else line = "{\"type\":\"" + type + "\"," + extraJson + "}";
|
|
100
|
+
lock (emitLock)
|
|
101
|
+
{
|
|
102
|
+
Console.WriteLine(line);
|
|
103
|
+
Console.Out.Flush();
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
static void EmitError(string msg)
|
|
108
|
+
{
|
|
109
|
+
string esc = msg.Replace("\\", "\\\\").Replace("\"", "\\\"").Replace("\r", "\\r").Replace("\n", "\\n");
|
|
110
|
+
Emit("error", "\"message\":\"" + esc + "\"");
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
static string ErrorText(int code)
|
|
114
|
+
{
|
|
115
|
+
var sb = new StringBuilder(256);
|
|
116
|
+
if (waveInGetErrorText(code, sb, sb.Capacity) == 0 && sb.Length > 0) return sb.ToString();
|
|
117
|
+
return "winmm error " + code;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
static void WaveCallback(IntPtr hwi, uint uMsg, IntPtr dwInstance, IntPtr dwParam1, IntPtr dwParam2)
|
|
121
|
+
{
|
|
122
|
+
if (uMsg != WIM_DATA) return;
|
|
123
|
+
if (stopping || finished) return;
|
|
124
|
+
try
|
|
125
|
+
{
|
|
126
|
+
WAVEHDR hdr = (WAVEHDR)Marshal.PtrToStructure(dwParam1, typeof(WAVEHDR));
|
|
127
|
+
uint recorded = hdr.dwBytesRecorded;
|
|
128
|
+
if (recorded > 0 && recorded <= hdr.dwBufferLength)
|
|
129
|
+
{
|
|
130
|
+
byte[] data = new byte[recorded];
|
|
131
|
+
Marshal.Copy(hdr.lpData, data, 0, (int)recorded);
|
|
132
|
+
string b64 = Convert.ToBase64String(data);
|
|
133
|
+
Emit("audio", "\"data\":\"" + b64 + "\"");
|
|
134
|
+
}
|
|
135
|
+
if (!stopping && !finished)
|
|
136
|
+
{
|
|
137
|
+
// requeue same header (dwBytesRecorded will be reset by driver)
|
|
138
|
+
// Need to reset flags? waveInAddBuffer will reuse.
|
|
139
|
+
int r = waveInAddBuffer(hWaveIn, dwParam1, Marshal.SizeOf(typeof(WAVEHDR)));
|
|
140
|
+
if (r != 0)
|
|
141
|
+
{
|
|
142
|
+
// If requeue fails, surface error but don't crash callback
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
catch (Exception ex)
|
|
147
|
+
{
|
|
148
|
+
// Don't emit from callback if already finishing
|
|
149
|
+
if (!finished) EmitError("wave callback failed: " + ex.Message);
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
static void Cleanup()
|
|
154
|
+
{
|
|
155
|
+
if (hWaveIn != IntPtr.Zero)
|
|
156
|
+
{
|
|
157
|
+
try { waveInReset(hWaveIn); } catch { }
|
|
158
|
+
for (int i = 0; i < BUFFER_COUNT; i++)
|
|
159
|
+
{
|
|
160
|
+
if (headerPtrs[i] != IntPtr.Zero)
|
|
161
|
+
{
|
|
162
|
+
try { waveInUnprepareHeader(hWaveIn, headerPtrs[i], Marshal.SizeOf(typeof(WAVEHDR))); } catch { }
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
try { waveInClose(hWaveIn); } catch { }
|
|
166
|
+
hWaveIn = IntPtr.Zero;
|
|
167
|
+
}
|
|
168
|
+
for (int i = 0; i < BUFFER_COUNT; i++)
|
|
169
|
+
{
|
|
170
|
+
if (bufferPtrs[i] != IntPtr.Zero) { Marshal.FreeHGlobal(bufferPtrs[i]); bufferPtrs[i] = IntPtr.Zero; }
|
|
171
|
+
if (headerPtrs[i] != IntPtr.Zero) { Marshal.FreeHGlobal(headerPtrs[i]); headerPtrs[i] = IntPtr.Zero; }
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
static void StopAndExit(int code)
|
|
176
|
+
{
|
|
177
|
+
if (finished) return;
|
|
178
|
+
finished = true;
|
|
179
|
+
stopping = true;
|
|
180
|
+
Cleanup();
|
|
181
|
+
if (code == 0) Emit("stopped");
|
|
182
|
+
try { Console.Out.Flush(); } catch { }
|
|
183
|
+
done.Set();
|
|
184
|
+
Environment.Exit(code);
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
static int Main(string[] args)
|
|
188
|
+
{
|
|
189
|
+
// Ctrl+C should produce error rather than silent exit
|
|
190
|
+
Console.CancelKeyPress += (s, e) => { e.Cancel = true; StopAndExit(0); };
|
|
191
|
+
|
|
192
|
+
if (waveInGetNumDevs() == 0)
|
|
193
|
+
{
|
|
194
|
+
EmitError("No microphone was found. Check Settings \u2192 Privacy & security \u2192 Microphone and ensure access is allowed.");
|
|
195
|
+
return 2;
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
var fmt = new WAVEFORMATEX
|
|
199
|
+
{
|
|
200
|
+
wFormatTag = WAVE_FORMAT_PCM,
|
|
201
|
+
nChannels = 1,
|
|
202
|
+
nSamplesPerSec = 16000,
|
|
203
|
+
wBitsPerSample = 16,
|
|
204
|
+
nBlockAlign = 2,
|
|
205
|
+
nAvgBytesPerSec = 32000,
|
|
206
|
+
cbSize = 0
|
|
207
|
+
};
|
|
208
|
+
|
|
209
|
+
callbackDelegate = new WaveInProc(WaveCallback);
|
|
210
|
+
|
|
211
|
+
int res = waveInOpen(out hWaveIn, WAVE_MAPPER, ref fmt, callbackDelegate, IntPtr.Zero, CALLBACK_FUNCTION);
|
|
212
|
+
if (res != 0)
|
|
213
|
+
{
|
|
214
|
+
string detail = ErrorText(res);
|
|
215
|
+
if (res == 32) detail += " — Microphone is in use by another app.";
|
|
216
|
+
// 5 = MMSYSERR_NODRIVER etc hints
|
|
217
|
+
EmitError("Could not open microphone: " + detail + " Check microphone privacy settings.");
|
|
218
|
+
return 2;
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
for (int i = 0; i < BUFFER_COUNT; i++)
|
|
222
|
+
{
|
|
223
|
+
bufferPtrs[i] = Marshal.AllocHGlobal(BUFFER_BYTES);
|
|
224
|
+
headerPtrs[i] = Marshal.AllocHGlobal(Marshal.SizeOf(typeof(WAVEHDR)));
|
|
225
|
+
var hdr = new WAVEHDR
|
|
226
|
+
{
|
|
227
|
+
lpData = bufferPtrs[i],
|
|
228
|
+
dwBufferLength = (uint)BUFFER_BYTES,
|
|
229
|
+
dwBytesRecorded = 0,
|
|
230
|
+
dwFlags = 0
|
|
231
|
+
};
|
|
232
|
+
Marshal.StructureToPtr(hdr, headerPtrs[i], false);
|
|
233
|
+
res = waveInPrepareHeader(hWaveIn, headerPtrs[i], Marshal.SizeOf(typeof(WAVEHDR)));
|
|
234
|
+
if (res != 0) { EmitError("Could not prepare audio buffer: " + ErrorText(res)); Cleanup(); return 3; }
|
|
235
|
+
res = waveInAddBuffer(hWaveIn, headerPtrs[i], Marshal.SizeOf(typeof(WAVEHDR)));
|
|
236
|
+
if (res != 0) { EmitError("Could not queue audio buffer: " + ErrorText(res)); Cleanup(); return 3; }
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
res = waveInStart(hWaveIn);
|
|
240
|
+
if (res != 0) { EmitError("Could not start microphone capture: " + ErrorText(res)); Cleanup(); return 4; }
|
|
241
|
+
|
|
242
|
+
Emit("ready", "\"sampleRate\":16000,\"channels\":1,\"encoding\":\"pcm_s16le\"");
|
|
243
|
+
|
|
244
|
+
// Wait for "stop" on stdin in background (Console.ReadLine blocks)
|
|
245
|
+
var stdinThread = new Thread(() =>
|
|
246
|
+
{
|
|
247
|
+
try
|
|
248
|
+
{
|
|
249
|
+
string line;
|
|
250
|
+
while ((line = Console.ReadLine()) != null)
|
|
251
|
+
{
|
|
252
|
+
if (line.Trim() == "stop")
|
|
253
|
+
{
|
|
254
|
+
stopping = true;
|
|
255
|
+
try { waveInStop(hWaveIn); } catch { }
|
|
256
|
+
// Give a brief window for final WIM_DATA to be delivered,
|
|
257
|
+
// then reset and exit. 150 ms is enough for trailing 100 ms buffer.
|
|
258
|
+
Thread.Sleep(180);
|
|
259
|
+
StopAndExit(0);
|
|
260
|
+
break;
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
// stdin closed (parent died) -> stop as well
|
|
264
|
+
if (!finished)
|
|
265
|
+
{
|
|
266
|
+
stopping = true;
|
|
267
|
+
try { waveInStop(hWaveIn); } catch { }
|
|
268
|
+
Thread.Sleep(120);
|
|
269
|
+
StopAndExit(0);
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
catch { StopAndExit(0); }
|
|
273
|
+
});
|
|
274
|
+
stdinThread.IsBackground = true;
|
|
275
|
+
stdinThread.Start();
|
|
276
|
+
|
|
277
|
+
// Block main thread until done signal
|
|
278
|
+
done.WaitOne();
|
|
279
|
+
return 0;
|
|
280
|
+
}
|
|
281
|
+
}
|
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
#Requires -Version 5.1
|
|
2
|
+
# Windows microphone helper for pi-meta-oauth voice (PowerShell fallback).
|
|
3
|
+
# Mirrors windows-audio.cs protocol when a compiled exe is not available.
|
|
4
|
+
# Uses winmm.dll waveIn* via Add-Type compiled C# — no NAudio / ffmpeg required.
|
|
5
|
+
# Protocol:
|
|
6
|
+
# stdout: {"type":"ready",...}
|
|
7
|
+
# stdout: {"type":"audio","data":"<base64>"}
|
|
8
|
+
# stdout: {"type":"stopped"}
|
|
9
|
+
# stdout: {"type":"error","message":"..."}
|
|
10
|
+
# stdin: "stop\n"
|
|
11
|
+
# Run: powershell -NoProfile -ExecutionPolicy Bypass -File windows-audio.ps1
|
|
12
|
+
|
|
13
|
+
$ErrorActionPreference = "Stop"
|
|
14
|
+
|
|
15
|
+
$csCode = @"
|
|
16
|
+
using System;
|
|
17
|
+
using System.Text;
|
|
18
|
+
using System.Threading;
|
|
19
|
+
using System.Runtime.InteropServices;
|
|
20
|
+
|
|
21
|
+
public class PsWinVoice
|
|
22
|
+
{
|
|
23
|
+
[StructLayout(LayoutKind.Sequential)]
|
|
24
|
+
struct WAVEFORMATEX
|
|
25
|
+
{
|
|
26
|
+
public ushort wFormatTag;
|
|
27
|
+
public ushort nChannels;
|
|
28
|
+
public uint nSamplesPerSec;
|
|
29
|
+
public uint nAvgBytesPerSec;
|
|
30
|
+
public ushort nBlockAlign;
|
|
31
|
+
public ushort wBitsPerSample;
|
|
32
|
+
public ushort cbSize;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
[StructLayout(LayoutKind.Sequential)]
|
|
36
|
+
struct WAVEHDR
|
|
37
|
+
{
|
|
38
|
+
public IntPtr lpData;
|
|
39
|
+
public uint dwBufferLength;
|
|
40
|
+
public uint dwBytesRecorded;
|
|
41
|
+
public IntPtr dwUser;
|
|
42
|
+
public uint dwFlags;
|
|
43
|
+
public uint dwLoops;
|
|
44
|
+
public IntPtr lpNext;
|
|
45
|
+
public IntPtr reserved;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
delegate void WaveInProc(IntPtr hwi, uint uMsg, IntPtr dwInstance, IntPtr dwParam1, IntPtr dwParam2);
|
|
49
|
+
|
|
50
|
+
[DllImport("winmm.dll")]
|
|
51
|
+
static extern int waveInGetNumDevs();
|
|
52
|
+
[DllImport("winmm.dll", CharSet = CharSet.Auto)]
|
|
53
|
+
static extern int waveInGetErrorText(int mmrError, StringBuilder pszText, int cchText);
|
|
54
|
+
[DllImport("winmm.dll")]
|
|
55
|
+
static extern int waveInOpen(out IntPtr phwi, int uDeviceID, ref WAVEFORMATEX pwfx, WaveInProc dwCallback, IntPtr dwInstance, uint fdwOpen);
|
|
56
|
+
[DllImport("winmm.dll")]
|
|
57
|
+
static extern int waveInPrepareHeader(IntPtr hwi, IntPtr pwh, int cbwh);
|
|
58
|
+
[DllImport("winmm.dll")]
|
|
59
|
+
static extern int waveInUnprepareHeader(IntPtr hwi, IntPtr pwh, int cbwh);
|
|
60
|
+
[DllImport("winmm.dll")]
|
|
61
|
+
static extern int waveInAddBuffer(IntPtr hwi, IntPtr pwh, int cbwh);
|
|
62
|
+
[DllImport("winmm.dll")]
|
|
63
|
+
static extern int waveInStart(IntPtr hwi);
|
|
64
|
+
[DllImport("winmm.dll")]
|
|
65
|
+
static extern int waveInStop(IntPtr hwi);
|
|
66
|
+
[DllImport("winmm.dll")]
|
|
67
|
+
static extern int waveInReset(IntPtr hwi);
|
|
68
|
+
[DllImport("winmm.dll")]
|
|
69
|
+
static extern int waveInClose(IntPtr hwi);
|
|
70
|
+
|
|
71
|
+
const uint CALLBACK_FUNCTION = 0x00030000;
|
|
72
|
+
const int WAVE_MAPPER = -1;
|
|
73
|
+
const int BUFFER_BYTES = 3200;
|
|
74
|
+
const int BUFFER_COUNT = 3;
|
|
75
|
+
|
|
76
|
+
static object emitLock = new object();
|
|
77
|
+
static WaveInProc cb;
|
|
78
|
+
static IntPtr hWaveIn = IntPtr.Zero;
|
|
79
|
+
static IntPtr[] hdrs = new IntPtr[BUFFER_COUNT];
|
|
80
|
+
static IntPtr[] bufs = new IntPtr[BUFFER_COUNT];
|
|
81
|
+
static volatile bool stopping = false;
|
|
82
|
+
static volatile bool finished = false;
|
|
83
|
+
static ManualResetEvent done = new ManualResetEvent(false);
|
|
84
|
+
|
|
85
|
+
static void Emit(string t, string extra = null)
|
|
86
|
+
{
|
|
87
|
+
string line = extra == null ? "{\"type\":\"" + t + "\"}" : "{\"type\":\"" + t + "\"," + extra + "}";
|
|
88
|
+
lock (emitLock) { Console.WriteLine(line); Console.Out.Flush(); }
|
|
89
|
+
}
|
|
90
|
+
static void EmitErr(string m)
|
|
91
|
+
{
|
|
92
|
+
string e = m.Replace("\\", "\\\\").Replace("\"", "\\\"").Replace("\r", "\\r").Replace("\n", "\\n");
|
|
93
|
+
Emit("error", "\"message\":\"" + e + "\"");
|
|
94
|
+
}
|
|
95
|
+
static string ErrTxt(int c)
|
|
96
|
+
{
|
|
97
|
+
var sb = new StringBuilder(256);
|
|
98
|
+
if (waveInGetErrorText(c, sb, sb.Capacity) == 0 && sb.Length > 0) return sb.ToString();
|
|
99
|
+
return "winmm error " + c;
|
|
100
|
+
}
|
|
101
|
+
static void Callback(IntPtr hwi, uint uMsg, IntPtr a, IntPtr pHdr, IntPtr b)
|
|
102
|
+
{
|
|
103
|
+
if (uMsg != 0x3C0) return;
|
|
104
|
+
if (stopping || finished) return;
|
|
105
|
+
try
|
|
106
|
+
{
|
|
107
|
+
WAVEHDR h = (WAVEHDR)Marshal.PtrToStructure(pHdr, typeof(WAVEHDR));
|
|
108
|
+
uint n = h.dwBytesRecorded;
|
|
109
|
+
if (n > 0 && n <= h.dwBufferLength)
|
|
110
|
+
{
|
|
111
|
+
byte[] d = new byte[n];
|
|
112
|
+
Marshal.Copy(h.lpData, d, 0, (int)n);
|
|
113
|
+
Emit("audio", "\"data\":\"" + Convert.ToBase64String(d) + "\"");
|
|
114
|
+
}
|
|
115
|
+
if (!stopping && !finished) waveInAddBuffer(hWaveIn, pHdr, Marshal.SizeOf(typeof(WAVEHDR)));
|
|
116
|
+
}
|
|
117
|
+
catch (Exception ex) { if (!finished) EmitErr("wave callback failed: " + ex.Message); }
|
|
118
|
+
}
|
|
119
|
+
static void Cleanup()
|
|
120
|
+
{
|
|
121
|
+
if (hWaveIn != IntPtr.Zero)
|
|
122
|
+
{
|
|
123
|
+
try { waveInReset(hWaveIn); } catch {}
|
|
124
|
+
for (int i = 0; i < BUFFER_COUNT; i++) if (hdrs[i] != IntPtr.Zero) try { waveInUnprepareHeader(hWaveIn, hdrs[i], Marshal.SizeOf(typeof(WAVEHDR))); } catch {}
|
|
125
|
+
try { waveInClose(hWaveIn); } catch {}
|
|
126
|
+
hWaveIn = IntPtr.Zero;
|
|
127
|
+
}
|
|
128
|
+
for (int i = 0; i < BUFFER_COUNT; i++)
|
|
129
|
+
{
|
|
130
|
+
if (bufs[i] != IntPtr.Zero) { Marshal.FreeHGlobal(bufs[i]); bufs[i]=IntPtr.Zero; }
|
|
131
|
+
if (hdrs[i] != IntPtr.Zero) { Marshal.FreeHGlobal(hdrs[i]); hdrs[i]=IntPtr.Zero; }
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
static void StopOk(int code)
|
|
135
|
+
{
|
|
136
|
+
if (finished) return;
|
|
137
|
+
finished = true; stopping = true;
|
|
138
|
+
Cleanup();
|
|
139
|
+
if (code == 0) Emit("stopped");
|
|
140
|
+
try { Console.Out.Flush(); } catch {}
|
|
141
|
+
done.Set();
|
|
142
|
+
Environment.Exit(code);
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
public static int Run()
|
|
146
|
+
{
|
|
147
|
+
if (waveInGetNumDevs() == 0) { EmitErr("No microphone was found. Check Settings \u2192 Privacy & security \u2192 Microphone."); return 2; }
|
|
148
|
+
var fmt = new WAVEFORMATEX { wFormatTag = 1, nChannels = 1, nSamplesPerSec = 16000, wBitsPerSample = 16, nBlockAlign = 2, nAvgBytesPerSec = 32000, cbSize = 0 };
|
|
149
|
+
cb = new WaveInProc(Callback);
|
|
150
|
+
int r = waveInOpen(out hWaveIn, WAVE_MAPPER, ref fmt, cb, IntPtr.Zero, CALLBACK_FUNCTION);
|
|
151
|
+
if (r != 0) { EmitErr("Could not open microphone: " + ErrTxt(r) + ". Check microphone privacy settings."); return 2; }
|
|
152
|
+
for (int i=0;i<BUFFER_COUNT;i++)
|
|
153
|
+
{
|
|
154
|
+
bufs[i] = Marshal.AllocHGlobal(BUFFER_BYTES);
|
|
155
|
+
hdrs[i] = Marshal.AllocHGlobal(Marshal.SizeOf(typeof(WAVEHDR)));
|
|
156
|
+
var h = new WAVEHDR { lpData = bufs[i], dwBufferLength = (uint)BUFFER_BYTES };
|
|
157
|
+
Marshal.StructureToPtr(h, hdrs[i], false);
|
|
158
|
+
r = waveInPrepareHeader(hWaveIn, hdrs[i], Marshal.SizeOf(typeof(WAVEHDR)));
|
|
159
|
+
if (r!=0){ EmitErr("Could not prepare buffer: "+ErrTxt(r)); Cleanup(); return 3; }
|
|
160
|
+
r = waveInAddBuffer(hWaveIn, hdrs[i], Marshal.SizeOf(typeof(WAVEHDR)));
|
|
161
|
+
if (r!=0){ EmitErr("Could not queue buffer: "+ErrTxt(r)); Cleanup(); return 3; }
|
|
162
|
+
}
|
|
163
|
+
r = waveInStart(hWaveIn);
|
|
164
|
+
if (r!=0){ EmitErr("Could not start capture: "+ErrTxt(r)); Cleanup(); return 4; }
|
|
165
|
+
Emit("ready", "\"sampleRate\":16000,\"channels\":1,\"encoding\":\"pcm_s16le\"");
|
|
166
|
+
var t = new Thread(() => {
|
|
167
|
+
try {
|
|
168
|
+
string l;
|
|
169
|
+
while ((l = Console.ReadLine()) != null) {
|
|
170
|
+
if (l.Trim() == "stop") { stopping=true; try{ waveInStop(hWaveIn);}catch{} Thread.Sleep(180); StopOk(0); break; }
|
|
171
|
+
}
|
|
172
|
+
if (!finished) { stopping=true; try{ waveInStop(hWaveIn);}catch{} Thread.Sleep(120); StopOk(0); }
|
|
173
|
+
} catch { StopOk(0); }
|
|
174
|
+
});
|
|
175
|
+
t.IsBackground=true; t.Start();
|
|
176
|
+
done.WaitOne();
|
|
177
|
+
return 0;
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
"@
|
|
181
|
+
|
|
182
|
+
try {
|
|
183
|
+
Add-Type -TypeDefinition $csCode -Language CSharp -ErrorAction Stop | Out-Null
|
|
184
|
+
} catch {
|
|
185
|
+
# Emit JSON error expected by TypeScript fail path
|
|
186
|
+
$msg = $_.Exception.Message.Replace('\','\\').Replace('"','\"')
|
|
187
|
+
Write-Output ('{"type":"error","message":"Failed to initialize Windows audio capture: ' + $msg + '"}')
|
|
188
|
+
exit 3
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
# Ensure Console uses UTF8 without BOM for JSON lines
|
|
192
|
+
[Console]::OutputEncoding = [System.Text.Encoding]::UTF8
|
|
193
|
+
|
|
194
|
+
$code = [PsWinVoice]::Run()
|
|
195
|
+
exit $code
|
package/extensions/voice.ts
CHANGED
|
@@ -25,6 +25,10 @@ const HELPER_SOURCE = join(HELPER_DIR, "macos-audio.swift");
|
|
|
25
25
|
const HELPER_PLIST = join(HELPER_DIR, "Info.plist");
|
|
26
26
|
const HELPER_ENTITLEMENTS = join(HELPER_DIR, "Entitlements.plist");
|
|
27
27
|
const HELPER_BINARY = join(AGENT_DIR, "bin/pi-meta-oauth-voice-v1");
|
|
28
|
+
// Windows helpers
|
|
29
|
+
const WINDOWS_HELPER_SOURCE = join(HELPER_DIR, "windows-audio.cs");
|
|
30
|
+
const WINDOWS_HELPER_PS1 = join(HELPER_DIR, "windows-audio.ps1");
|
|
31
|
+
const WINDOWS_HELPER_BINARY = join(AGENT_DIR, "bin/pi-meta-oauth-voice-v1.exe");
|
|
28
32
|
|
|
29
33
|
const DEFAULT_ASR_ENDPOINT =
|
|
30
34
|
"wss://shortwave.facebook.com/voyager/v1/asr/duplex";
|
|
@@ -45,6 +49,24 @@ type AsrTranscript = {
|
|
|
45
49
|
final?: unknown;
|
|
46
50
|
};
|
|
47
51
|
|
|
52
|
+
function isMacOS(): boolean {
|
|
53
|
+
return platform() === "darwin";
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function isWindows(): boolean {
|
|
57
|
+
return platform() === "win32";
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
function isSupportedPlatform(): boolean {
|
|
61
|
+
return isMacOS() || isWindows();
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function supportedPlatformLabel(): string {
|
|
65
|
+
if (isMacOS()) return "macOS";
|
|
66
|
+
if (isWindows()) return "Windows";
|
|
67
|
+
return `${platform()}`;
|
|
68
|
+
}
|
|
69
|
+
|
|
48
70
|
function loadSettings(): Settings {
|
|
49
71
|
if (existsSync(SETTINGS_FILE)) {
|
|
50
72
|
try {
|
|
@@ -56,7 +78,7 @@ function loadSettings(): Settings {
|
|
|
56
78
|
// Fall through to the platform default.
|
|
57
79
|
}
|
|
58
80
|
}
|
|
59
|
-
return { enabled:
|
|
81
|
+
return { enabled: isSupportedPlatform() };
|
|
60
82
|
}
|
|
61
83
|
|
|
62
84
|
function saveSettings(settings: Settings): void {
|
|
@@ -206,10 +228,17 @@ function helperNeedsBuild(): boolean {
|
|
|
206
228
|
);
|
|
207
229
|
}
|
|
208
230
|
|
|
231
|
+
function windowsHelperNeedsBuild(): boolean {
|
|
232
|
+
if (!existsSync(WINDOWS_HELPER_BINARY)) return true;
|
|
233
|
+
if (!existsSync(WINDOWS_HELPER_SOURCE)) return false;
|
|
234
|
+
const binaryTime = statSync(WINDOWS_HELPER_BINARY).mtimeMs;
|
|
235
|
+
return statSync(WINDOWS_HELPER_SOURCE).mtimeMs > binaryTime;
|
|
236
|
+
}
|
|
237
|
+
|
|
209
238
|
async function ensureMacOSHelper(): Promise<string> {
|
|
210
|
-
if (
|
|
239
|
+
if (!isMacOS()) {
|
|
211
240
|
throw new Error(
|
|
212
|
-
"Muse-style voice input is currently available only on macOS",
|
|
241
|
+
"Muse-style voice input is currently available only on macOS and Windows",
|
|
213
242
|
);
|
|
214
243
|
}
|
|
215
244
|
if (
|
|
@@ -249,6 +278,109 @@ async function ensureMacOSHelper(): Promise<string> {
|
|
|
249
278
|
return HELPER_BINARY;
|
|
250
279
|
}
|
|
251
280
|
|
|
281
|
+
async function tryCompileWindowsHelper(): Promise<boolean> {
|
|
282
|
+
if (!existsSync(WINDOWS_HELPER_SOURCE)) return false;
|
|
283
|
+
mkdirSync(join(AGENT_DIR, "bin"), { recursive: true });
|
|
284
|
+
|
|
285
|
+
const candidates: string[] = [
|
|
286
|
+
"csc",
|
|
287
|
+
"csc.exe",
|
|
288
|
+
"C:\\Windows\\Microsoft.NET\\Framework\\v4.0.30319\\csc.exe",
|
|
289
|
+
"C:\\Windows\\Microsoft.NET\\Framework64\\v4.0.30319\\csc.exe",
|
|
290
|
+
];
|
|
291
|
+
// Also try where-found csc via PATH expansion if on Windows
|
|
292
|
+
for (const c of candidates) {
|
|
293
|
+
try {
|
|
294
|
+
await runProcess(c, [
|
|
295
|
+
"/nologo",
|
|
296
|
+
"/target:exe",
|
|
297
|
+
`/out:${WINDOWS_HELPER_BINARY}`,
|
|
298
|
+
WINDOWS_HELPER_SOURCE,
|
|
299
|
+
]);
|
|
300
|
+
if (existsSync(WINDOWS_HELPER_BINARY)) return true;
|
|
301
|
+
} catch {
|
|
302
|
+
// try next candidate
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
return false;
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
function findPowerShellCommand(): string {
|
|
309
|
+
// Prefer pwsh (PowerShell 7) if present, otherwise Windows PowerShell
|
|
310
|
+
// We check filesystem first to avoid spawning during hot path
|
|
311
|
+
if (existsSync("C:\\Program Files\\PowerShell\\7\\pwsh.exe"))
|
|
312
|
+
return "C:\\Program Files\\PowerShell\\7\\pwsh.exe";
|
|
313
|
+
if (existsSync("C:\\Program Files\\PowerShell\\7\\pwsh")) return "pwsh";
|
|
314
|
+
// Windows PowerShell is inbox in System32
|
|
315
|
+
if (
|
|
316
|
+
existsSync("C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe")
|
|
317
|
+
)
|
|
318
|
+
return "C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe";
|
|
319
|
+
return "powershell.exe";
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
async function ensureWindowsHelper(): Promise<{
|
|
323
|
+
command: string;
|
|
324
|
+
args: string[];
|
|
325
|
+
}> {
|
|
326
|
+
if (!isWindows()) {
|
|
327
|
+
throw new Error(
|
|
328
|
+
"Muse-style voice input is currently available only on macOS and Windows",
|
|
329
|
+
);
|
|
330
|
+
}
|
|
331
|
+
if (!existsSync(WINDOWS_HELPER_SOURCE) && !existsSync(WINDOWS_HELPER_PS1)) {
|
|
332
|
+
throw new Error("The Windows microphone helper sources are incomplete");
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
// If compiled exe exists and is fresh, use it
|
|
336
|
+
if (existsSync(WINDOWS_HELPER_BINARY) && !windowsHelperNeedsBuild()) {
|
|
337
|
+
return { command: WINDOWS_HELPER_BINARY, args: [] };
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
// Try to compile exe for better performance
|
|
341
|
+
if (existsSync(WINDOWS_HELPER_SOURCE)) {
|
|
342
|
+
if (!windowsHelperNeedsBuild() && existsSync(WINDOWS_HELPER_BINARY)) {
|
|
343
|
+
return { command: WINDOWS_HELPER_BINARY, args: [] };
|
|
344
|
+
}
|
|
345
|
+
const compiled = await tryCompileWindowsHelper();
|
|
346
|
+
if (compiled && existsSync(WINDOWS_HELPER_BINARY)) {
|
|
347
|
+
return { command: WINDOWS_HELPER_BINARY, args: [] };
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
// Fallback to PowerShell script (no compilation needed)
|
|
352
|
+
if (existsSync(WINDOWS_HELPER_PS1)) {
|
|
353
|
+
const ps = findPowerShellCommand();
|
|
354
|
+
return {
|
|
355
|
+
command: ps,
|
|
356
|
+
args: [
|
|
357
|
+
"-NoProfile",
|
|
358
|
+
"-ExecutionPolicy",
|
|
359
|
+
"Bypass",
|
|
360
|
+
"-File",
|
|
361
|
+
WINDOWS_HELPER_PS1,
|
|
362
|
+
],
|
|
363
|
+
};
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
throw new Error(
|
|
367
|
+
"Windows voice helper could not be prepared — no compiled binary and no PowerShell fallback found",
|
|
368
|
+
);
|
|
369
|
+
}
|
|
370
|
+
|
|
371
|
+
async function ensureHelper(): Promise<{ command: string; args: string[] }> {
|
|
372
|
+
if (isMacOS()) {
|
|
373
|
+
const bin = await ensureMacOSHelper();
|
|
374
|
+
return { command: bin, args: [] };
|
|
375
|
+
}
|
|
376
|
+
if (isWindows()) {
|
|
377
|
+
return ensureWindowsHelper();
|
|
378
|
+
}
|
|
379
|
+
throw new Error(
|
|
380
|
+
`Muse-style voice input is currently available only on macOS and Windows (current: ${supportedPlatformLabel()})`,
|
|
381
|
+
);
|
|
382
|
+
}
|
|
383
|
+
|
|
252
384
|
class MetaVoiceController {
|
|
253
385
|
private readonly pi: ExtensionAPI;
|
|
254
386
|
private readonly settings = loadSettings();
|
|
@@ -339,9 +471,9 @@ class MetaVoiceController {
|
|
|
339
471
|
}
|
|
340
472
|
|
|
341
473
|
private enable(ctx: ExtensionContext): void {
|
|
342
|
-
if (
|
|
474
|
+
if (!isSupportedPlatform()) {
|
|
343
475
|
ctx.ui.notify(
|
|
344
|
-
|
|
476
|
+
`Meta voice input is currently available only on macOS and Windows (current: ${supportedPlatformLabel()})`,
|
|
345
477
|
"error",
|
|
346
478
|
);
|
|
347
479
|
return;
|
|
@@ -407,8 +539,8 @@ class MetaVoiceController {
|
|
|
407
539
|
this.renderTranscript(ctx);
|
|
408
540
|
|
|
409
541
|
try {
|
|
410
|
-
const [
|
|
411
|
-
|
|
542
|
+
const [helperInfo, apiKey] = await Promise.all([
|
|
543
|
+
ensureHelper(),
|
|
412
544
|
ctx.modelRegistry.getApiKeyForProvider(META_PROVIDER_ID),
|
|
413
545
|
]);
|
|
414
546
|
if (currentGeneration !== this.generation) return;
|
|
@@ -421,7 +553,12 @@ class MetaVoiceController {
|
|
|
421
553
|
}
|
|
422
554
|
|
|
423
555
|
this.connectSocket(ctx, apiKey, currentGeneration);
|
|
424
|
-
this.spawnHelper(
|
|
556
|
+
this.spawnHelper(
|
|
557
|
+
ctx,
|
|
558
|
+
helperInfo.command,
|
|
559
|
+
helperInfo.args,
|
|
560
|
+
currentGeneration,
|
|
561
|
+
);
|
|
425
562
|
this.renderInterval = setInterval(() => {
|
|
426
563
|
this.renderTranscript(ctx);
|
|
427
564
|
this.renderRecordingStatus(ctx);
|
|
@@ -558,10 +695,14 @@ class MetaVoiceController {
|
|
|
558
695
|
|
|
559
696
|
private spawnHelper(
|
|
560
697
|
ctx: ExtensionContext,
|
|
561
|
-
|
|
698
|
+
command: string,
|
|
699
|
+
args: string[],
|
|
562
700
|
currentGeneration: number,
|
|
563
701
|
): void {
|
|
564
|
-
const helper = spawn(
|
|
702
|
+
const helper = spawn(command, args, {
|
|
703
|
+
stdio: ["pipe", "pipe", "pipe"],
|
|
704
|
+
windowsHide: true,
|
|
705
|
+
});
|
|
565
706
|
this.helper = helper;
|
|
566
707
|
helper.stdout?.on("data", (chunk: Buffer) =>
|
|
567
708
|
this.parseHelperOutput(ctx, chunk, currentGeneration),
|
|
@@ -603,7 +744,7 @@ class MetaVoiceController {
|
|
|
603
744
|
try {
|
|
604
745
|
this.handleHelperEvent(ctx, JSON.parse(line) as HelperEvent);
|
|
605
746
|
} catch {
|
|
606
|
-
// Ignore non-protocol output from
|
|
747
|
+
// Ignore non-protocol output from audio frameworks.
|
|
607
748
|
}
|
|
608
749
|
}
|
|
609
750
|
}
|
|
@@ -636,7 +777,7 @@ class MetaVoiceController {
|
|
|
636
777
|
new Error(
|
|
637
778
|
typeof event.message === "string"
|
|
638
779
|
? event.message
|
|
639
|
-
: "
|
|
780
|
+
: "microphone capture failed",
|
|
640
781
|
),
|
|
641
782
|
);
|
|
642
783
|
return;
|