whale-code 6.4.0 → 6.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (187) hide show
  1. package/bin/swagmanager-mcp.js +7 -0
  2. package/dist/cli/app.js +30 -2
  3. package/dist/cli/chat/ChatApp.d.ts +4 -4
  4. package/dist/cli/chat/ChatApp.js +114 -44
  5. package/dist/cli/chat/ChatInput.d.ts +13 -6
  6. package/dist/cli/chat/ChatInput.js +433 -89
  7. package/dist/cli/chat/MemoryManager.d.ts +15 -0
  8. package/dist/cli/chat/MemoryManager.js +61 -0
  9. package/dist/cli/chat/MessageList.d.ts +8 -0
  10. package/dist/cli/chat/MessageList.js +1 -1
  11. package/dist/cli/chat/NodeManager.d.ts +30 -0
  12. package/dist/cli/chat/NodeManager.js +89 -0
  13. package/dist/cli/chat/NodeSelector.d.ts +19 -0
  14. package/dist/cli/chat/NodeSelector.js +37 -0
  15. package/dist/cli/chat/PlanApproval.d.ts +17 -0
  16. package/dist/cli/chat/PlanApproval.js +82 -0
  17. package/dist/cli/chat/SessionManager.d.ts +16 -0
  18. package/dist/cli/chat/SessionManager.js +43 -0
  19. package/dist/cli/chat/SlashMenu.d.ts +38 -0
  20. package/dist/cli/chat/SlashMenu.js +208 -0
  21. package/dist/cli/chat/StatusBar.d.ts +16 -0
  22. package/dist/cli/chat/StatusBar.js +22 -0
  23. package/dist/cli/chat/ThemeSelector.d.ts +14 -0
  24. package/dist/cli/chat/ThemeSelector.js +29 -0
  25. package/dist/cli/chat/ToolIndicator.d.ts +8 -0
  26. package/dist/cli/chat/ToolIndicator.js +33 -9
  27. package/dist/cli/chat/hooks/useAgentLoop.d.ts +2 -1
  28. package/dist/cli/chat/hooks/useAgentLoop.js +22 -17
  29. package/dist/cli/chat/hooks/useSlashCommands.d.ts +19 -0
  30. package/dist/cli/chat/hooks/useSlashCommands.js +254 -15
  31. package/dist/cli/commands/config-cmd.js +4 -25
  32. package/dist/cli/commands/db.d.ts +13 -0
  33. package/dist/cli/commands/db.js +243 -0
  34. package/dist/cli/commands/doctor.js +6 -9
  35. package/dist/cli/commands/mcp.js +1 -20
  36. package/dist/cli/services/agent-events.d.ts +22 -1
  37. package/dist/cli/services/agent-events.js +9 -0
  38. package/dist/cli/services/agent-loop.js +66 -2
  39. package/dist/cli/services/agent-worker-base.js +21 -6
  40. package/dist/cli/services/api-retry.d.ts +25 -0
  41. package/dist/cli/services/api-retry.js +91 -0
  42. package/dist/cli/services/auth-service.d.ts +1 -1
  43. package/dist/cli/services/auth-service.js +40 -19
  44. package/dist/cli/services/background-processes.js +26 -2
  45. package/dist/cli/services/config-store.d.ts +13 -1
  46. package/dist/cli/services/config-store.js +116 -13
  47. package/dist/cli/services/format-server-response.js +12 -6
  48. package/dist/cli/services/ink-resize-fix.d.ts +18 -0
  49. package/dist/cli/services/ink-resize-fix.js +66 -0
  50. package/dist/cli/services/interactive-tools.d.ts +14 -0
  51. package/dist/cli/services/interactive-tools.js +47 -2
  52. package/dist/cli/services/keybinding-manager.js +1 -1
  53. package/dist/cli/services/local-tools.js +35 -2
  54. package/dist/cli/services/server-tools.js +175 -3
  55. package/dist/cli/services/subagent.js +15 -3
  56. package/dist/cli/services/system-prompt.js +5 -3
  57. package/dist/cli/services/task-decomposer.d.ts +35 -0
  58. package/dist/cli/services/task-decomposer.js +199 -0
  59. package/dist/cli/services/team-lead.d.ts +18 -0
  60. package/dist/cli/services/team-lead.js +80 -0
  61. package/dist/cli/services/teammate.js +5 -5
  62. package/dist/cli/services/telemetry.d.ts +8 -2
  63. package/dist/cli/services/telemetry.js +116 -92
  64. package/dist/cli/services/tools/agent-tools.d.ts +1 -0
  65. package/dist/cli/services/tools/agent-tools.js +50 -4
  66. package/dist/cli/services/tools/file-ops.d.ts +2 -0
  67. package/dist/cli/services/tools/file-ops.js +71 -19
  68. package/dist/cli/services/tools/shell-exec.js +22 -12
  69. package/dist/cli/shared/Theme.d.ts +1 -2
  70. package/dist/cli/shared/Theme.js +1 -1
  71. package/dist/cli/shared/WhaleBanner.d.ts +4 -1
  72. package/dist/cli/shared/WhaleBanner.js +12 -8
  73. package/dist/cli/shared/markdown.d.ts +5 -4
  74. package/dist/cli/shared/markdown.js +376 -334
  75. package/dist/cli/shared/theme-manager.d.ts +27 -0
  76. package/dist/cli/shared/theme-manager.js +178 -0
  77. package/dist/cli/shared/theme-presets.d.ts +16 -0
  78. package/dist/cli/shared/theme-presets.js +265 -0
  79. package/dist/index.js +0 -51
  80. package/dist/node/adapters/imessage.d.ts +10 -0
  81. package/dist/node/adapters/imessage.js +45 -6
  82. package/dist/node/cli.js +459 -8
  83. package/dist/node/config.d.ts +17 -0
  84. package/dist/node/gateway-client.d.ts +55 -0
  85. package/dist/node/gateway-client.js +201 -0
  86. package/dist/node/portal/clipboard.d.ts +28 -0
  87. package/dist/node/portal/clipboard.js +183 -0
  88. package/dist/node/portal/discovery.d.ts +29 -0
  89. package/dist/node/portal/discovery.js +61 -0
  90. package/dist/node/portal/forward.d.ts +30 -0
  91. package/dist/node/portal/forward.js +90 -0
  92. package/dist/node/portal/index.d.ts +47 -0
  93. package/dist/node/portal/index.js +250 -0
  94. package/dist/node/portal/multiplexer.d.ts +48 -0
  95. package/dist/node/portal/multiplexer.js +207 -0
  96. package/dist/node/portal/permissions.d.ts +36 -0
  97. package/dist/node/portal/permissions.js +131 -0
  98. package/dist/node/portal/protocol.d.ts +140 -0
  99. package/dist/node/portal/protocol.js +193 -0
  100. package/dist/node/portal/screen.d.ts +18 -0
  101. package/dist/node/portal/screen.js +93 -0
  102. package/dist/node/portal/session.d.ts +68 -0
  103. package/dist/node/portal/session.js +127 -0
  104. package/dist/node/portal/shell.d.ts +26 -0
  105. package/dist/node/portal/shell.js +142 -0
  106. package/dist/node/portal/stream.d.ts +43 -0
  107. package/dist/node/portal/stream.js +90 -0
  108. package/dist/node/portal/transfer.d.ts +33 -0
  109. package/dist/node/portal/transfer.js +231 -0
  110. package/dist/node/portal/ui.d.ts +16 -0
  111. package/dist/node/portal/ui.js +148 -0
  112. package/dist/node/remote-desktop/compile-helper.d.ts +13 -0
  113. package/dist/node/remote-desktop/compile-helper.js +73 -0
  114. package/dist/node/remote-desktop/index.d.ts +67 -0
  115. package/dist/node/remote-desktop/index.js +220 -0
  116. package/dist/node/remote-desktop/protocol.d.ts +96 -0
  117. package/dist/node/remote-desktop/protocol.js +67 -0
  118. package/dist/node/runtime.d.ts +8 -1
  119. package/dist/node/runtime.js +117 -9
  120. package/dist/server/handlers/__test-utils__/test-db.d.ts +25 -0
  121. package/dist/server/handlers/__test-utils__/test-db.js +128 -0
  122. package/dist/server/handlers/api-keys.js +26 -2
  123. package/dist/server/handlers/browser.d.ts +0 -4
  124. package/dist/server/handlers/browser.js +0 -46
  125. package/dist/server/handlers/catalog.js +37 -14
  126. package/dist/server/handlers/clickhouse.d.ts +10 -0
  127. package/dist/server/handlers/clickhouse.js +215 -0
  128. package/dist/server/handlers/comms.d.ts +308 -4
  129. package/dist/server/handlers/comms.js +444 -11
  130. package/dist/server/handlers/creations.js +1 -1
  131. package/dist/server/handlers/crm.d.ts +54 -8
  132. package/dist/server/handlers/crm.js +353 -68
  133. package/dist/server/handlers/embeddings.js +3 -3
  134. package/dist/server/handlers/enrichment.js +39 -55
  135. package/dist/server/handlers/inventory.js +1 -1
  136. package/dist/server/handlers/kali.d.ts +9 -1
  137. package/dist/server/handlers/kali.js +50 -1
  138. package/dist/server/handlers/media.d.ts +8 -0
  139. package/dist/server/handlers/media.js +902 -0
  140. package/dist/server/handlers/meta-ads.js +6 -3
  141. package/dist/server/handlers/nodes.d.ts +2 -0
  142. package/dist/server/handlers/nodes.js +331 -40
  143. package/dist/server/handlers/operations.d.ts +4 -6
  144. package/dist/server/handlers/operations.js +99 -38
  145. package/dist/server/handlers/platform.js +224 -107
  146. package/dist/server/handlers/remove-bg.d.ts +6 -0
  147. package/dist/server/handlers/remove-bg.js +96 -0
  148. package/dist/server/handlers/storefront.d.ts +6 -0
  149. package/dist/server/handlers/storefront.js +477 -0
  150. package/dist/server/handlers/supply-chain.js +21 -3
  151. package/dist/server/handlers/workflow-steps.js +87 -31
  152. package/dist/server/handlers/workflows.js +4 -1
  153. package/dist/server/index.js +334 -88
  154. package/dist/server/lib/clickhouse-buffer.d.ts +48 -0
  155. package/dist/server/lib/clickhouse-buffer.js +175 -0
  156. package/dist/server/lib/clickhouse-client.d.ts +112 -0
  157. package/dist/server/lib/clickhouse-client.js +141 -0
  158. package/dist/server/lib/coa-renderer.d.ts +91 -0
  159. package/dist/server/lib/coa-renderer.js +411 -0
  160. package/dist/server/lib/compaction-service.js +45 -1
  161. package/dist/server/lib/pdf-renderer.d.ts +143 -0
  162. package/dist/server/lib/pdf-renderer.js +867 -0
  163. package/dist/server/lib/react-pdf-layout.d.ts +40 -0
  164. package/dist/server/lib/react-pdf-layout.js +437 -0
  165. package/dist/server/lib/server-agent-loop.d.ts +2 -0
  166. package/dist/server/lib/server-agent-loop.js +61 -15
  167. package/dist/server/lib/server-subagent.d.ts +3 -0
  168. package/dist/server/lib/server-subagent.js +7 -4
  169. package/dist/server/lib/supabase-client.js +51 -3
  170. package/dist/server/lib/template-resolver.js +14 -4
  171. package/dist/server/lib/utils.js +15 -0
  172. package/dist/server/local-agent-gateway.d.ts +44 -0
  173. package/dist/server/local-agent-gateway.js +389 -49
  174. package/dist/server/providers/anthropic.js +12 -2
  175. package/dist/server/providers/gemini.js +17 -2
  176. package/dist/server/proxy-handlers.js +151 -0
  177. package/dist/server/tool-router.d.ts +2 -2
  178. package/dist/server/tool-router.js +25 -35
  179. package/dist/shared/agent-core.d.ts +5 -2
  180. package/dist/shared/agent-core.js +30 -4
  181. package/dist/shared/api-client.js +54 -3
  182. package/dist/shared/sse-parser.d.ts +1 -1
  183. package/dist/shared/sse-parser.js +5 -2
  184. package/dist/shared/tool-dispatch.js +1 -1
  185. package/package.json +16 -10
  186. package/dist/server/handlers/__test-utils__/mock-supabase.d.ts +0 -11
  187. package/dist/server/handlers/__test-utils__/mock-supabase.js +0 -393
@@ -0,0 +1,148 @@
1
+ /**
2
+ * Portal UI — terminal progress bars, status display, and interactive menu.
3
+ */
4
+ // ============================================================================
5
+ // NODE LIST
6
+ // ============================================================================
7
+ export function printNodeList(nodes, showOffline = false) {
8
+ if (nodes.length === 0) {
9
+ console.log("No nodes online in your store.");
10
+ console.log(" Run `whale node start` on a machine to bring it online.");
11
+ return;
12
+ }
13
+ const online = nodes.filter(n => n.status === "online");
14
+ const offline = nodes.filter(n => n.status === "offline");
15
+ console.log(`\nOnline Nodes (${online.length}):\n`);
16
+ console.log(" NAME HOSTNAME PLATFORM STATUS");
17
+ console.log(" " + "─".repeat(70));
18
+ for (const n of online) {
19
+ const name = n.name.padEnd(20).slice(0, 20);
20
+ const host = n.hostname.padEnd(20).slice(0, 20);
21
+ const plat = n.platform.padEnd(10).slice(0, 10);
22
+ console.log(` ${name} ${host} ${plat} online`);
23
+ }
24
+ if (showOffline && offline.length > 0) {
25
+ console.log(`\nOffline Nodes (${offline.length}):\n`);
26
+ for (const n of offline) {
27
+ const name = n.name.padEnd(20).slice(0, 20);
28
+ const host = n.hostname.padEnd(20).slice(0, 20);
29
+ const plat = n.platform.padEnd(10).slice(0, 10);
30
+ console.log(` ${name} ${host} ${plat} offline`);
31
+ }
32
+ }
33
+ console.log();
34
+ }
35
+ // ============================================================================
36
+ // TRANSFER PROGRESS
37
+ // ============================================================================
38
+ let lastProgressLine = "";
39
+ export function printProgress(progress) {
40
+ const bar = makeBar(progress.percent, 30);
41
+ const size = formatBytes(progress.bytesTransferred);
42
+ const total = progress.totalBytes > 0 ? formatBytes(progress.totalBytes) : "?";
43
+ const line = ` ${progress.name}: ${bar} ${progress.percent}% (${size}/${total})`;
44
+ if (line !== lastProgressLine) {
45
+ process.stderr.write(`\r${line}`);
46
+ lastProgressLine = line;
47
+ }
48
+ if (progress.percent >= 100) {
49
+ process.stderr.write("\n");
50
+ lastProgressLine = "";
51
+ }
52
+ }
53
+ function makeBar(percent, width) {
54
+ const filled = Math.round((percent / 100) * width);
55
+ return "[" + "█".repeat(filled) + "░".repeat(width - filled) + "]";
56
+ }
57
+ // ============================================================================
58
+ // SESSION STATUS
59
+ // ============================================================================
60
+ export function printSessionStatus(sessions) {
61
+ if (sessions.length === 0) {
62
+ console.log("No active portal sessions.");
63
+ return;
64
+ }
65
+ console.log(`\nActive Sessions (${sessions.length}):\n`);
66
+ console.log(" SESSION TARGET STREAMS STATUS");
67
+ console.log(" " + "─".repeat(75));
68
+ for (const s of sessions) {
69
+ const sid = s.session_id.slice(0, 8);
70
+ const target = (s.target_hostname || s.target_node_id || "unknown").padEnd(20).slice(0, 20);
71
+ const streams = String(s.active_streams).padEnd(8);
72
+ const status = s.accepted ? "connected" : "pending";
73
+ console.log(` ${sid} ${target} ${streams} ${status}`);
74
+ }
75
+ console.log();
76
+ }
77
+ // ============================================================================
78
+ // FORWARD STATUS
79
+ // ============================================================================
80
+ export function printForwardStatus(forwards) {
81
+ if (forwards.length === 0) {
82
+ console.log("No active port forwards.");
83
+ return;
84
+ }
85
+ console.log(`\nActive Forwards (${forwards.length}):\n`);
86
+ for (const f of forwards) {
87
+ console.log(` localhost:${f.localPort} → remote:${f.remotePort} (${f.connections} connections)`);
88
+ }
89
+ console.log();
90
+ }
91
+ // ============================================================================
92
+ // INTERACTIVE MENU
93
+ // ============================================================================
94
+ export async function showPortalMenu(nodeName) {
95
+ const { createInterface } = await import("node:readline");
96
+ console.log(`\nPortal to ${nodeName}\n`);
97
+ console.log(" 1) Shell Open remote terminal");
98
+ console.log(" 2) Push file Send file to remote node");
99
+ console.log(" 3) Pull file Get file from remote node");
100
+ console.log(" 4) Port forward Forward local port to remote");
101
+ console.log(" 5) Screen share View remote desktop");
102
+ console.log(" 6) Clipboard Sync clipboards");
103
+ console.log(" q) Quit\n");
104
+ return new Promise((resolve) => {
105
+ const rl = createInterface({
106
+ input: process.stdin,
107
+ output: process.stdout,
108
+ });
109
+ rl.question(" Choose: ", (answer) => {
110
+ rl.close();
111
+ const choice = answer.trim().toLowerCase();
112
+ const map = {
113
+ "1": "shell", "2": "push", "3": "pull",
114
+ "4": "forward", "5": "screen", "6": "clipboard",
115
+ "q": "quit", "shell": "shell", "push": "push",
116
+ "pull": "pull", "forward": "forward", "screen": "screen",
117
+ "clipboard": "clipboard", "quit": "quit",
118
+ };
119
+ resolve(map[choice] || "unknown");
120
+ });
121
+ });
122
+ }
123
+ // ============================================================================
124
+ // CONNECTION STATUS
125
+ // ============================================================================
126
+ export function printConnecting(nodeName) {
127
+ process.stderr.write(`Connecting to ${nodeName}...`);
128
+ }
129
+ export function printConnected(hostname) {
130
+ process.stderr.write(`\rConnected to ${hostname} \n`);
131
+ }
132
+ export function printDisconnected(reason) {
133
+ console.log(`\nDisconnected${reason ? `: ${reason}` : ""}`);
134
+ }
135
+ export function printError(msg) {
136
+ console.error(`Error: ${msg}`);
137
+ }
138
+ // ============================================================================
139
+ // HELPERS
140
+ // ============================================================================
141
+ function formatBytes(bytes) {
142
+ if (bytes === 0)
143
+ return "0 B";
144
+ const units = ["B", "KB", "MB", "GB", "TB"];
145
+ const i = Math.floor(Math.log(bytes) / Math.log(1024));
146
+ const val = bytes / Math.pow(1024, i);
147
+ return `${val.toFixed(i > 0 ? 1 : 0)} ${units[i]}`;
148
+ }
@@ -0,0 +1,13 @@
1
+ /**
2
+ * Compiles the Swift screen-helper on first use, caches at ~/.whaletools/bin/screen-helper.
3
+ * Recompiles only when source file changes (mtime comparison).
4
+ * No-op on non-macOS platforms.
5
+ */
6
+ export declare function getHelperBinaryPath(): string;
7
+ export declare function isHelperAvailable(): boolean;
8
+ /**
9
+ * Compile the Swift screen-helper if needed.
10
+ * Returns the path to the compiled binary.
11
+ * Throws if compilation fails or not on macOS.
12
+ */
13
+ export declare function compileHelper(): string;
@@ -0,0 +1,73 @@
1
+ /**
2
+ * Compiles the Swift screen-helper on first use, caches at ~/.whaletools/bin/screen-helper.
3
+ * Recompiles only when source file changes (mtime comparison).
4
+ * No-op on non-macOS platforms.
5
+ */
6
+ import { existsSync, mkdirSync, statSync } from "node:fs";
7
+ import { execSync } from "node:child_process";
8
+ import { homedir } from "node:os";
9
+ import { join, dirname } from "node:path";
10
+ import { fileURLToPath } from "node:url";
11
+ const __dirname = dirname(fileURLToPath(import.meta.url));
12
+ const SOURCE_FILE = join(__dirname, "screen-helper.swift");
13
+ const BIN_DIR = join(homedir(), ".whaletools", "bin");
14
+ const BINARY_PATH = join(BIN_DIR, "screen-helper");
15
+ export function getHelperBinaryPath() {
16
+ return BINARY_PATH;
17
+ }
18
+ export function isHelperAvailable() {
19
+ return process.platform === "darwin" && existsSync(BINARY_PATH);
20
+ }
21
+ /**
22
+ * Compile the Swift screen-helper if needed.
23
+ * Returns the path to the compiled binary.
24
+ * Throws if compilation fails or not on macOS.
25
+ */
26
+ export function compileHelper() {
27
+ if (process.platform !== "darwin") {
28
+ throw new Error("Screen helper is only available on macOS");
29
+ }
30
+ // Ensure bin directory exists
31
+ if (!existsSync(BIN_DIR)) {
32
+ mkdirSync(BIN_DIR, { recursive: true });
33
+ }
34
+ // Check if recompilation is needed
35
+ if (existsSync(BINARY_PATH)) {
36
+ try {
37
+ const srcStat = statSync(SOURCE_FILE);
38
+ const binStat = statSync(BINARY_PATH);
39
+ if (binStat.mtimeMs >= srcStat.mtimeMs) {
40
+ return BINARY_PATH; // Binary is up to date
41
+ }
42
+ }
43
+ catch {
44
+ // If stat fails, recompile
45
+ }
46
+ }
47
+ // Find source file — check dist location first, then src
48
+ let sourcePath = SOURCE_FILE;
49
+ if (!existsSync(sourcePath)) {
50
+ // When running from dist/, the swift file might be in src/
51
+ const srcPath = sourcePath.replace("/dist/", "/src/");
52
+ if (existsSync(srcPath)) {
53
+ sourcePath = srcPath;
54
+ }
55
+ else {
56
+ throw new Error(`Swift source not found at ${sourcePath} or ${srcPath}`);
57
+ }
58
+ }
59
+ console.log("[remote-desktop] Compiling screen-helper...");
60
+ try {
61
+ execSync(`swiftc -O "${sourcePath}" -framework ScreenCaptureKit -framework VideoToolbox -framework CoreMedia -framework CoreVideo -framework Network -framework AppKit -o "${BINARY_PATH}"`, {
62
+ stdio: "pipe",
63
+ timeout: 60_000,
64
+ encoding: "utf-8",
65
+ });
66
+ console.log("[remote-desktop] screen-helper compiled successfully");
67
+ return BINARY_PATH;
68
+ }
69
+ catch (err) {
70
+ const stderr = err.stderr || err.message;
71
+ throw new Error(`Failed to compile screen-helper: ${stderr}`);
72
+ }
73
+ }
@@ -0,0 +1,67 @@
1
+ /**
2
+ * ScreenCaptureService — EventEmitter-based screen capture for Mac whale-node.
3
+ *
4
+ * Spawns the Swift screen-helper for H.264 hardware-encoded screen capture.
5
+ * Emits frames as events for any number of consumers (Portal streams, etc.).
6
+ *
7
+ * The Swift helper handles:
8
+ * - Direct LAN WebSocket + Bonjour (iPad connects directly)
9
+ * - H.264 hardware encoding via VTCompressionSession
10
+ * - Input injection via CGEvent
11
+ *
12
+ * This service handles:
13
+ * - Helper lifecycle (compile, spawn, restart)
14
+ * - Reading stdout frames for Node.js consumers (Portal relay, gateway)
15
+ * - Forwarding input events to helper stdin
16
+ * - Caching latest frame + config for new viewers
17
+ */
18
+ import { EventEmitter } from "node:events";
19
+ import { type InputEvent } from "./protocol.js";
20
+ export interface ScreenCaptureConfig {
21
+ enabled: boolean;
22
+ max_fps: number;
23
+ }
24
+ export interface CaptureStatus {
25
+ capturing: boolean;
26
+ logicalWidth: number;
27
+ logicalHeight: number;
28
+ captureWidth: number;
29
+ captureHeight: number;
30
+ codec: string;
31
+ listeners: number;
32
+ }
33
+ export declare class ScreenCaptureService extends EventEmitter {
34
+ private config;
35
+ private helper;
36
+ latestFrame: Buffer | null;
37
+ latestConfig: Buffer | null;
38
+ private _logicalWidth;
39
+ private _logicalHeight;
40
+ private _captureWidth;
41
+ private _captureHeight;
42
+ private _capturing;
43
+ private stdoutBuffer;
44
+ constructor(config?: Partial<ScreenCaptureConfig>);
45
+ get logicalWidth(): number;
46
+ get logicalHeight(): number;
47
+ get captureWidth(): number;
48
+ get captureHeight(): number;
49
+ get capturing(): boolean;
50
+ start(): Promise<void>;
51
+ stop(): Promise<void>;
52
+ /**
53
+ * Forward input event to the Swift helper via stdin.
54
+ */
55
+ injectInput(event: InputEvent): void;
56
+ /**
57
+ * Forward a control message (request_keyframe, quality) to the Swift helper.
58
+ */
59
+ sendControlMessage(msg: {
60
+ type: string;
61
+ [key: string]: unknown;
62
+ }): void;
63
+ getStatus(): CaptureStatus;
64
+ private spawnHelper;
65
+ private processFrameBuffer;
66
+ private handleHelperStatus;
67
+ }
@@ -0,0 +1,220 @@
1
+ /**
2
+ * ScreenCaptureService — EventEmitter-based screen capture for Mac whale-node.
3
+ *
4
+ * Spawns the Swift screen-helper for H.264 hardware-encoded screen capture.
5
+ * Emits frames as events for any number of consumers (Portal streams, etc.).
6
+ *
7
+ * The Swift helper handles:
8
+ * - Direct LAN WebSocket + Bonjour (iPad connects directly)
9
+ * - H.264 hardware encoding via VTCompressionSession
10
+ * - Input injection via CGEvent
11
+ *
12
+ * This service handles:
13
+ * - Helper lifecycle (compile, spawn, restart)
14
+ * - Reading stdout frames for Node.js consumers (Portal relay, gateway)
15
+ * - Forwarding input events to helper stdin
16
+ * - Caching latest frame + config for new viewers
17
+ */
18
+ import { spawn } from "node:child_process";
19
+ import { EventEmitter } from "node:events";
20
+ import { createInterface } from "node:readline";
21
+ import { compileHelper } from "./compile-helper.js";
22
+ import { FrameType, parseFrameHeaderV4, isV4Frame, } from "./protocol.js";
23
+ const DEFAULT_CONFIG = {
24
+ enabled: true,
25
+ max_fps: 60,
26
+ };
27
+ export class ScreenCaptureService extends EventEmitter {
28
+ config;
29
+ helper = null;
30
+ // Frame caching — latest of each type for new viewers
31
+ latestFrame = null;
32
+ latestConfig = null; // SPS/PPS for decoder init
33
+ // Display dimensions (from parsed frames)
34
+ _logicalWidth = 0;
35
+ _logicalHeight = 0;
36
+ _captureWidth = 0;
37
+ _captureHeight = 0;
38
+ _capturing = false;
39
+ // Stdout frame assembly
40
+ stdoutBuffer = Buffer.alloc(0);
41
+ constructor(config = {}) {
42
+ super();
43
+ this.config = { ...DEFAULT_CONFIG, ...config };
44
+ this.setMaxListeners(50); // Multiple portal streams can listen
45
+ }
46
+ get logicalWidth() { return this._logicalWidth; }
47
+ get logicalHeight() { return this._logicalHeight; }
48
+ get captureWidth() { return this._captureWidth; }
49
+ get captureHeight() { return this._captureHeight; }
50
+ get capturing() { return this._capturing; }
51
+ async start() {
52
+ if (process.platform !== "darwin") {
53
+ console.log("[screen-capture] Not macOS, skipping");
54
+ return;
55
+ }
56
+ let helperPath;
57
+ try {
58
+ helperPath = compileHelper();
59
+ }
60
+ catch (err) {
61
+ console.error("[screen-capture] Failed to compile helper:", err.message);
62
+ return;
63
+ }
64
+ this.spawnHelper(helperPath);
65
+ }
66
+ async stop() {
67
+ if (this.helper) {
68
+ this.helper.kill("SIGTERM");
69
+ this.helper = null;
70
+ }
71
+ this._capturing = false;
72
+ this.latestFrame = null;
73
+ this.latestConfig = null;
74
+ this.emit("stopped");
75
+ console.log("[screen-capture] Stopped");
76
+ }
77
+ /**
78
+ * Forward input event to the Swift helper via stdin.
79
+ */
80
+ injectInput(event) {
81
+ if (this.helper?.stdin?.writable) {
82
+ this.helper.stdin.write(JSON.stringify(event) + "\n");
83
+ }
84
+ }
85
+ /**
86
+ * Forward a control message (request_keyframe, quality) to the Swift helper.
87
+ */
88
+ sendControlMessage(msg) {
89
+ if (this.helper?.stdin?.writable) {
90
+ this.helper.stdin.write(JSON.stringify(msg) + "\n");
91
+ }
92
+ }
93
+ getStatus() {
94
+ return {
95
+ capturing: this._capturing,
96
+ logicalWidth: this._logicalWidth,
97
+ logicalHeight: this._logicalHeight,
98
+ captureWidth: this._captureWidth,
99
+ captureHeight: this._captureHeight,
100
+ codec: "h264",
101
+ listeners: this.listenerCount("frame"),
102
+ };
103
+ }
104
+ // ============================================================================
105
+ // INTERNAL
106
+ // ============================================================================
107
+ spawnHelper(helperPath) {
108
+ this.helper = spawn(helperPath, [], {
109
+ stdio: ["pipe", "pipe", "pipe"],
110
+ });
111
+ // Handle stdout — length-prefixed binary frames (v4 H.264)
112
+ this.helper.stdout.on("data", (chunk) => {
113
+ this.stdoutBuffer = Buffer.concat([this.stdoutBuffer, chunk]);
114
+ this.processFrameBuffer();
115
+ });
116
+ // Handle stderr — log lines from Swift helper
117
+ const stderrRL = createInterface({ input: this.helper.stderr });
118
+ stderrRL.on("line", (line) => {
119
+ // Try to parse as JSON status, otherwise just log
120
+ try {
121
+ const status = JSON.parse(line);
122
+ this.handleHelperStatus(status);
123
+ }
124
+ catch {
125
+ console.log(`[screen-helper] ${line}`);
126
+ // Parse display info from log lines
127
+ const displayMatch = line.match(/Display: (\d+)x(\d+) logical/);
128
+ if (displayMatch) {
129
+ this._logicalWidth = parseInt(displayMatch[1]);
130
+ this._logicalHeight = parseInt(displayMatch[2]);
131
+ }
132
+ if (line.includes("SCStream started")) {
133
+ this._capturing = true;
134
+ this.emit("started");
135
+ }
136
+ }
137
+ });
138
+ this.helper.on("exit", (code) => {
139
+ console.log(`[screen-helper] Exited with code ${code}`);
140
+ this._capturing = false;
141
+ this.helper = null;
142
+ this.emit("stopped");
143
+ });
144
+ // Set initial FPS
145
+ this.injectInput({ type: "set_fps", fps: this.config.max_fps });
146
+ }
147
+ processFrameBuffer() {
148
+ while (true) {
149
+ // Need at least 4 bytes for the length prefix
150
+ if (this.stdoutBuffer.length < 4)
151
+ break;
152
+ const payloadLen = this.stdoutBuffer.readUInt32BE(0);
153
+ const totalLen = 4 + payloadLen;
154
+ // Wait for full frame
155
+ if (this.stdoutBuffer.length < totalLen)
156
+ break;
157
+ const frameData = Buffer.from(this.stdoutBuffer.subarray(0, totalLen));
158
+ this.stdoutBuffer = this.stdoutBuffer.subarray(totalLen);
159
+ // Parse v4 header to determine frame type and cache appropriately
160
+ if (frameData.length >= 27 && isV4Frame(frameData)) {
161
+ const header = parseFrameHeaderV4(frameData);
162
+ // Update dimensions from frame header
163
+ if (header.logicalWidth > 0) {
164
+ this._logicalWidth = header.logicalWidth;
165
+ this._logicalHeight = header.logicalHeight;
166
+ }
167
+ if (header.captureWidth > 0) {
168
+ this._captureWidth = header.captureWidth;
169
+ this._captureHeight = header.captureHeight;
170
+ }
171
+ switch (header.frameType) {
172
+ case FrameType.CONFIG:
173
+ this.latestConfig = frameData;
174
+ this.emit("config", frameData);
175
+ break;
176
+ case FrameType.KEYFRAME:
177
+ this.latestFrame = frameData;
178
+ this.emit("frame", frameData);
179
+ break;
180
+ case FrameType.DELTA:
181
+ this.latestFrame = frameData;
182
+ this.emit("frame", frameData);
183
+ break;
184
+ }
185
+ }
186
+ else {
187
+ // Legacy v3 frame — emit as-is for backwards compat
188
+ this.latestFrame = frameData;
189
+ this.emit("frame", frameData);
190
+ }
191
+ }
192
+ }
193
+ handleHelperStatus(status) {
194
+ switch (status.type) {
195
+ case "starting":
196
+ console.log(`[screen-helper] ${status.message}`);
197
+ break;
198
+ case "display_info":
199
+ this._logicalWidth = status.width || 0;
200
+ this._logicalHeight = status.height || 0;
201
+ console.log(`[screen-helper] Display: ${this._logicalWidth}x${this._logicalHeight}`);
202
+ break;
203
+ case "capturing":
204
+ this._capturing = true;
205
+ this.emit("started");
206
+ console.log(`[screen-helper] Capturing at ${status.fps} FPS`);
207
+ break;
208
+ case "permission_needed":
209
+ console.warn(`[screen-helper] ${status.message}`);
210
+ break;
211
+ case "error":
212
+ console.error(`[screen-helper] Error: ${status.error}`);
213
+ break;
214
+ case "stopping":
215
+ this._capturing = false;
216
+ break;
217
+ }
218
+ this.emit("status", status);
219
+ }
220
+ }
@@ -0,0 +1,96 @@
1
+ /**
2
+ * Remote Desktop Protocol — shared types between Node.js server and clients.
3
+ *
4
+ * v3 (legacy): 28-byte header, JPEG payload
5
+ * v4 (current): 27-byte header, H.264 NAL payload
6
+ */
7
+ export declare const FRAME_HEADER_SIZE = 28;
8
+ export interface FrameHeader {
9
+ seq: number;
10
+ timestamp: number;
11
+ width: number;
12
+ height: number;
13
+ quality: number;
14
+ }
15
+ export declare function parseFrameHeader(buf: Buffer): FrameHeader;
16
+ export declare const FRAME_HEADER_SIZE_V4 = 27;
17
+ export declare const NAL_DATA_OFFSET_V4 = 27;
18
+ export declare enum FrameType {
19
+ KEYFRAME = 1,
20
+ DELTA = 2,
21
+ CONFIG = 3
22
+ }
23
+ export interface FrameHeaderV4 {
24
+ payloadLength: number;
25
+ frameType: FrameType;
26
+ seq: number;
27
+ timestamp: number;
28
+ logicalWidth: number;
29
+ logicalHeight: number;
30
+ captureWidth: number;
31
+ captureHeight: number;
32
+ }
33
+ export declare function parseFrameHeaderV4(buf: Buffer): FrameHeaderV4;
34
+ export declare function isV4Frame(buf: Buffer): boolean;
35
+ export interface InputEvent {
36
+ type: "mouse_move" | "mouse_down" | "mouse_up" | "mouse_drag" | "key_down" | "key_up" | "scroll" | "set_quality" | "set_fps";
37
+ x?: number;
38
+ y?: number;
39
+ button?: number;
40
+ keyCode?: number;
41
+ flags?: number;
42
+ scrollX?: number;
43
+ scrollY?: number;
44
+ quality?: number;
45
+ fps?: number;
46
+ clickCount?: number;
47
+ }
48
+ export interface SessionRequest {
49
+ type: "session_request";
50
+ token: string;
51
+ store_id: string;
52
+ }
53
+ export interface SessionResponse {
54
+ type: "session_response";
55
+ success: boolean;
56
+ session_id?: string;
57
+ width?: number;
58
+ height?: number;
59
+ fps?: number;
60
+ error?: string;
61
+ }
62
+ export interface SessionEnd {
63
+ type: "session_end";
64
+ reason: string;
65
+ }
66
+ export interface QualityChange {
67
+ type: "quality_change";
68
+ quality?: number;
69
+ fps?: number;
70
+ }
71
+ export interface HelperStatus {
72
+ type: "starting" | "display_info" | "capturing" | "error" | "stopping" | "permission_needed" | "quality_changed" | "fps_changed" | "input_error";
73
+ message?: string;
74
+ width?: number;
75
+ height?: number;
76
+ fps?: number;
77
+ quality?: number;
78
+ error?: string;
79
+ }
80
+ export interface RemoteDesktopRequest {
81
+ type: "remote_desktop_request";
82
+ target_node_id: string;
83
+ store_id: string;
84
+ }
85
+ export interface RemoteDesktopResponse {
86
+ type: "remote_desktop_response";
87
+ success: boolean;
88
+ session_id?: string;
89
+ error?: string;
90
+ }
91
+ export interface RemoteDesktopEnd {
92
+ type: "remote_desktop_end";
93
+ session_id: string;
94
+ reason: string;
95
+ }
96
+ export type RemoteDesktopMessage = SessionRequest | SessionResponse | SessionEnd | QualityChange | InputEvent | RemoteDesktopRequest | RemoteDesktopResponse | RemoteDesktopEnd;
@@ -0,0 +1,67 @@
1
+ /**
2
+ * Remote Desktop Protocol — shared types between Node.js server and clients.
3
+ *
4
+ * v3 (legacy): 28-byte header, JPEG payload
5
+ * v4 (current): 27-byte header, H.264 NAL payload
6
+ */
7
+ // ============================================================================
8
+ // v3 Frame Header (legacy — JPEG)
9
+ // ============================================================================
10
+ // [0-3] total payload length (big-endian uint32, excludes these 4 bytes)
11
+ // [4-7] sequence number (big-endian uint32)
12
+ // [8-15] timestamp ms (big-endian uint64)
13
+ // [16-17] width (big-endian uint16)
14
+ // [18-19] height (big-endian uint16)
15
+ // [20-21] quality * 100 (big-endian uint16)
16
+ // [22-27] reserved
17
+ export const FRAME_HEADER_SIZE = 28;
18
+ export function parseFrameHeader(buf) {
19
+ return {
20
+ seq: buf.readUInt32BE(4),
21
+ timestamp: Number(buf.readBigUInt64BE(8)),
22
+ width: buf.readUInt16BE(16),
23
+ height: buf.readUInt16BE(18),
24
+ quality: buf.readUInt16BE(20) / 100,
25
+ };
26
+ }
27
+ // ============================================================================
28
+ // v4 Frame Header — H.264 hardware-encoded
29
+ // ============================================================================
30
+ // [0-3] Payload length (big-endian uint32, includes header bytes 4-26 + NAL data)
31
+ // [4] Frame type: 0x01=keyframe, 0x02=delta, 0x03=config (SPS/PPS)
32
+ // [5-8] Sequence number (big-endian uint32)
33
+ // [9-16] Timestamp ms (big-endian uint64)
34
+ // [17-18] Logical width (uint16 BE) — CGEvent coordinate space
35
+ // [19-20] Logical height (uint16 BE)
36
+ // [21-22] Capture width (uint16 BE) — actual encoded pixel dimensions
37
+ // [23-24] Capture height (uint16 BE)
38
+ // [25-26] Reserved
39
+ // [27+] H.264 NAL unit data
40
+ export const FRAME_HEADER_SIZE_V4 = 27;
41
+ export const NAL_DATA_OFFSET_V4 = 27;
42
+ export var FrameType;
43
+ (function (FrameType) {
44
+ FrameType[FrameType["KEYFRAME"] = 1] = "KEYFRAME";
45
+ FrameType[FrameType["DELTA"] = 2] = "DELTA";
46
+ FrameType[FrameType["CONFIG"] = 3] = "CONFIG";
47
+ })(FrameType || (FrameType = {}));
48
+ export function parseFrameHeaderV4(buf) {
49
+ return {
50
+ payloadLength: buf.readUInt32BE(0),
51
+ frameType: buf[4],
52
+ seq: buf.readUInt32BE(5),
53
+ timestamp: Number(buf.readBigUInt64BE(9)),
54
+ logicalWidth: buf.readUInt16BE(17),
55
+ logicalHeight: buf.readUInt16BE(19),
56
+ captureWidth: buf.readUInt16BE(21),
57
+ captureHeight: buf.readUInt16BE(23),
58
+ };
59
+ }
60
+ export function isV4Frame(buf) {
61
+ if (buf.length < 5)
62
+ return false;
63
+ const frameType = buf[4];
64
+ return frameType === FrameType.KEYFRAME
65
+ || frameType === FrameType.DELTA
66
+ || frameType === FrameType.CONFIG;
67
+ }