screen-relay 0.0.0 → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. package/LICENSE +201 -0
  2. package/Makefile +165 -0
  3. package/README.md +112 -1
  4. package/THIRD-PARTY-NOTICES.md +75 -0
  5. package/dist/cli/client.d.ts +28 -0
  6. package/dist/cli/commands.d.ts +52 -0
  7. package/dist/cli/config.d.ts +32 -0
  8. package/dist/cli/format.d.ts +15 -0
  9. package/dist/cli/help.d.ts +5 -0
  10. package/dist/cli/host/common.d.ts +23 -0
  11. package/dist/cli/host/doctor.d.ts +2 -0
  12. package/dist/cli/host/packages.d.ts +13 -0
  13. package/dist/cli/host/session_commands.d.ts +8 -0
  14. package/dist/cli/host/setup.d.ts +22 -0
  15. package/dist/cli/host/state.d.ts +23 -0
  16. package/dist/cli/host/supervisor.d.ts +94 -0
  17. package/dist/cli/host/watcher.d.ts +4 -0
  18. package/dist/cli/host/windows.d.ts +82 -0
  19. package/dist/cli/main.d.ts +3 -0
  20. package/dist/cli/mode.d.ts +14 -0
  21. package/dist/cli/output.d.ts +24 -0
  22. package/dist/cli/parser.d.ts +6 -0
  23. package/dist/cli/relay.mjs +12631 -0
  24. package/dist/cli/skills.d.ts +21 -0
  25. package/dist/cli/theme.d.ts +14 -0
  26. package/dist/protocol/errors.d.ts +155 -0
  27. package/dist/protocol/index.d.ts +119 -0
  28. package/dist/protocol/index.js +418 -0
  29. package/dist/sdk/index.d.ts +78 -0
  30. package/dist/sdk/index.js +834 -0
  31. package/dist/sdk/transport.d.ts +20 -0
  32. package/dist/viewer/index.html +96 -0
  33. package/dist/viewer/viewer.js +1079 -0
  34. package/package.json +47 -7
  35. package/packages.tsv +38 -0
  36. package/skills/core/SKILL.md +171 -0
  37. package/skills/core/references/cli.md +119 -0
  38. package/skills/core/references/config.md +48 -0
  39. package/skills/core/references/macos.md +50 -0
  40. package/skills/core/references/sdk.md +69 -0
  41. package/skills/core/references/troubleshooting.md +33 -0
  42. package/skills/core/references/wayland.md +47 -0
  43. package/skills/screen-relay/SKILL.md +24 -0
  44. package/skills/vercel-sandbox/SKILL.md +68 -0
  45. package/src/engine/backends/linux/keycodes.hpp +35 -0
  46. package/src/engine/backends/linux/wayland/capture.cpp +298 -0
  47. package/src/engine/backends/linux/wayland/input.cpp +559 -0
  48. package/src/engine/backends/linux/wayland/keymap.cpp +118 -0
  49. package/src/engine/backends/linux/wayland/keymap.hpp +54 -0
  50. package/src/engine/backends/linux/wayland/protocols/virtual-keyboard-unstable-v1.xml +112 -0
  51. package/src/engine/backends/linux/wayland/protocols/wlr-screencopy-unstable-v1.xml +234 -0
  52. package/src/engine/backends/linux/wayland/protocols/wlr-virtual-pointer-unstable-v1.xml +152 -0
  53. package/src/engine/backends/linux/wayland/wayland.cpp +166 -0
  54. package/src/engine/backends/linux/wayland/wayland.hpp +60 -0
  55. package/src/engine/backends/linux/x11/capture.cpp +206 -0
  56. package/src/engine/backends/linux/x11/input.cpp +266 -0
  57. package/src/engine/backends/macos/capture.mm +367 -0
  58. package/src/engine/backends/macos/display.hpp +20 -0
  59. package/src/engine/backends/macos/display.mm +84 -0
  60. package/src/engine/backends/macos/input.mm +287 -0
  61. package/src/engine/backends/macos/keys.hpp +34 -0
  62. package/src/engine/backends/windows/capture.cpp +393 -0
  63. package/src/engine/backends/windows/input.cpp +274 -0
  64. package/src/engine/backends/windows/output.cpp +65 -0
  65. package/src/engine/backends/windows/output.hpp +26 -0
  66. package/src/engine/core/capture-bucket.hpp +40 -0
  67. package/src/engine/core/capture-source.hpp +59 -0
  68. package/src/engine/core/capture.cpp +565 -0
  69. package/src/engine/core/capture.hpp +58 -0
  70. package/src/engine/core/color.cpp +176 -0
  71. package/src/engine/core/color.hpp +16 -0
  72. package/src/engine/core/cpu-budget.hpp +196 -0
  73. package/src/engine/core/flow.hpp +317 -0
  74. package/src/engine/core/input.cpp +98 -0
  75. package/src/engine/core/input.hpp +115 -0
  76. package/src/engine/core/wait.cpp +125 -0
  77. package/src/engine/core/wait.hpp +41 -0
  78. package/src/engine/server/backend.cpp +65 -0
  79. package/src/engine/server/backend.hpp +18 -0
  80. package/src/engine/server/config.cpp +257 -0
  81. package/src/engine/server/config.hpp +24 -0
  82. package/src/engine/server/main.cpp +82 -0
  83. package/src/engine/server/metrics.cpp +47 -0
  84. package/src/engine/server/metrics.hpp +10 -0
  85. package/src/engine/server/screenshot.cpp +108 -0
  86. package/src/engine/server/screenshot.hpp +31 -0
  87. package/src/engine/server/server.cpp +535 -0
  88. package/src/engine/server/server.hpp +89 -0
  89. package/src/engine/server/wire.cpp +70 -0
  90. package/src/engine/server/wire.hpp +29 -0
  91. package/index.js +0 -2
@@ -0,0 +1,287 @@
1
+ #include "core/input.hpp"
2
+ #include "backends/linux/keycodes.hpp"
3
+ #include "backends/macos/display.hpp"
4
+ #include "backends/macos/keys.hpp"
5
+ #include <AppKit/AppKit.h>
6
+ #include <Carbon/Carbon.h>
7
+ #include <algorithm>
8
+ #include <chrono>
9
+ #include <cmath>
10
+ #include <condition_variable>
11
+ #include <cstdint>
12
+ #include <map>
13
+ #include <memory>
14
+ #include <mutex>
15
+ #include <optional>
16
+ #include <set>
17
+ #include <thread>
18
+ #include <utility>
19
+ #include <vector>
20
+
21
+ namespace relay {
22
+ namespace {
23
+ /// Tap that receives every posted event.
24
+ constexpr CGEventTapLocation eventTap = kCGHIDEventTap;
25
+ /// Sign that turns a positive relay notch into a line step down or right; flip it if the VM scrolls backwards.
26
+ constexpr int scrollSign = -1;
27
+ /// Points of cursor travel still counted as the same click position.
28
+ constexpr double clickSlop = 5;
29
+ /// Modifier flag a held key contributes to every posted event.
30
+ constexpr CGEventFlags modifierFlag(NativeKey key) {
31
+ switch (key) {
32
+ case kVK_Shift: case kVK_RightShift: return kCGEventFlagMaskShift;
33
+ case kVK_Control: case kVK_RightControl: return kCGEventFlagMaskControl;
34
+ case kVK_Option: case kVK_RightOption: return kCGEventFlagMaskAlternate;
35
+ case kVK_Command: case kVK_RightCommand: return kCGEventFlagMaskCommand;
36
+ default: return 0;
37
+ }
38
+ }
39
+ /// Applies queued input as CGEvents on one thread that owns a private event source.
40
+ class MacInput final : public InputBackend {
41
+ CGRect bounds{};
42
+ uint16_t width = 0, height = 0;
43
+ CGEventSourceRef source = nullptr;
44
+ CGPoint position{}, clickPoint{};
45
+ std::set<NativeKey> held;
46
+ std::map<unsigned, unsigned> heldButtons;
47
+ unsigned clicks = 1, clickButton = 0;
48
+ std::chrono::steady_clock::time_point clickTime{};
49
+ std::chrono::steady_clock::duration clickInterval{};
50
+ bool capsLock = false;
51
+ mutable std::mutex mutex;
52
+ std::condition_variable wake;
53
+ InputQueue queue;
54
+ bool stopping = false;
55
+ std::thread worker;
56
+
57
+ public:
58
+ MacInput() {
59
+ const auto display = macos::findDisplay();
60
+ bounds = display.bounds;
61
+ width = uint16_t(display.width);
62
+ height = uint16_t(display.height);
63
+ position = CGPointMake(CGRectGetMidX(bounds), CGRectGetMidY(bounds));
64
+ capsLock = (CGEventSourceFlagsState(kCGEventSourceStateHIDSystemState) & kCGEventFlagMaskAlphaShift) != 0;
65
+ clickInterval = std::chrono::duration_cast<std::chrono::steady_clock::duration>(
66
+ std::chrono::duration<double>([NSEvent doubleClickInterval]));
67
+ worker = std::thread([this] { run(); });
68
+ }
69
+ ~MacInput() override {
70
+ { std::lock_guard<std::mutex> lock(mutex); stopping = true; }
71
+ wake.notify_all();
72
+ if (worker.joinable()) worker.join();
73
+ }
74
+ std::pair<uint16_t, uint16_t> dimensions() const override { return {width, height}; }
75
+ const char* name() const override { return "macos"; }
76
+ std::optional<NativeKey> nativeKey(uint16_t usage) const override {
77
+ const int key = macos::macKey(hidKeycode(usage));
78
+ if (key < 0) return std::nullopt;
79
+ return NativeKey(key);
80
+ }
81
+ bool typeText(uint32_t codepoint) override {
82
+ if (codepoint == 0 || codepoint > 0x10ffff || (codepoint >= 0xd800 && codepoint <= 0xdfff)) return false;
83
+ InputJob job{InputJob::Kind::text};
84
+ job.codepoint = codepoint;
85
+ post(job);
86
+ return true;
87
+ }
88
+ void pressKey(NativeKey key) override { postKey(key, true); }
89
+ void releaseKey(NativeKey key) override { postKey(key, false); }
90
+ void pressButton(unsigned button) override { postButton(button, true); }
91
+ void releaseButton(unsigned button) override { postButton(button, false); }
92
+ void move(int x, int y) override {
93
+ InputJob job{InputJob::Kind::motion};
94
+ job.x = x; job.y = y;
95
+ post(job);
96
+ }
97
+ void scroll(unsigned axis, int notches) override {
98
+ InputJob job{InputJob::Kind::scroll};
99
+ job.number = axis; job.notches = notches;
100
+ post(job);
101
+ }
102
+ void resetLocks() override { post(InputJob{InputJob::Kind::locks}); }
103
+ void fence(std::shared_ptr<InputFence> mark) override {
104
+ InputJob job{InputJob::Kind::fence};
105
+ job.fence = std::move(mark);
106
+ post(job);
107
+ }
108
+ void cancel() override {
109
+ { std::lock_guard<std::mutex> lock(mutex); queue.cancel(); }
110
+ wake.notify_one();
111
+ }
112
+ void flush() override { post(InputJob{InputJob::Kind::flush}); }
113
+ size_t pending() const override {
114
+ std::lock_guard<std::mutex> lock(mutex);
115
+ return queue.pending();
116
+ }
117
+
118
+ private:
119
+ void postKey(NativeKey key, bool press) {
120
+ InputJob job{InputJob::Kind::key};
121
+ job.code = key; job.press = press;
122
+ post(job);
123
+ }
124
+ void postButton(unsigned button, bool press) {
125
+ InputJob job{InputJob::Kind::button};
126
+ job.number = button; job.press = press;
127
+ post(job);
128
+ }
129
+ void post(InputJob job) {
130
+ { std::lock_guard<std::mutex> lock(mutex); queue.post(std::move(job)); }
131
+ wake.notify_one();
132
+ }
133
+ void run() {
134
+ source = CGEventSourceCreate(kCGEventSourceStatePrivate);
135
+ std::unique_lock<std::mutex> lock(mutex);
136
+ for (;;) {
137
+ wake.wait(lock, [this] { return stopping || !queue.empty(); });
138
+ if (stopping) queue.cancel();
139
+ if (queue.empty()) break;
140
+ const InputJob job = queue.take();
141
+ lock.unlock();
142
+ apply(job);
143
+ lock.lock();
144
+ }
145
+ lock.unlock();
146
+ releaseHeld();
147
+ if (source) { CFRelease(source); source = nullptr; }
148
+ }
149
+ CGEventFlags flags() const {
150
+ CGEventFlags value = capsLock ? kCGEventFlagMaskAlphaShift : 0;
151
+ for (const auto key : held) value |= modifierFlag(key);
152
+ return value;
153
+ }
154
+ static CGEventRef withFlags(CGEventRef event, CGEventFlags state) {
155
+ if (event) CGEventSetFlags(event, state);
156
+ return event;
157
+ }
158
+ CGEventRef keyEvent(NativeKey key, bool press, CGEventFlags state) const {
159
+ return withFlags(CGEventCreateKeyboardEvent(source, CGKeyCode(key), press), state);
160
+ }
161
+ CGEventRef mouseEvent(CGEventType type, CGPoint where, CGMouseButton button, unsigned clickCount,
162
+ CGEventFlags state) const {
163
+ CGEventRef event = withFlags(CGEventCreateMouseEvent(source, type, where, button), state);
164
+ if (event && clickCount) CGEventSetIntegerValueField(event, kCGMouseEventClickState, clickCount);
165
+ return event;
166
+ }
167
+ static void send(CGEventRef event) {
168
+ if (!event) return;
169
+ CGEventPost(eventTap, event);
170
+ CFRelease(event);
171
+ }
172
+ void apply(const InputJob& job) {
173
+ const CGEventFlags state = flags();
174
+ switch (job.kind) {
175
+ case InputJob::Kind::key:
176
+ if (job.press) held.insert(job.code); else held.erase(job.code);
177
+ if (job.press && job.code == NativeKey(kVK_CapsLock)) capsLock = !capsLock;
178
+ send(keyEvent(job.code, job.press, flags()));
179
+ break;
180
+ case InputJob::Kind::text: type(job.codepoint); break;
181
+ case InputJob::Kind::button: applyButton(job.number, job.press, state); break;
182
+ case InputJob::Kind::motion: {
183
+ const CGPoint target = globalPoint(job.x, job.y);
184
+ if (CGPointEqualToPoint(target, position) && !heldButtons.empty()) break;
185
+ position = target;
186
+ send(mouseEvent(motionType(), position, dragButton(), 0, state));
187
+ break;
188
+ }
189
+ case InputJob::Kind::scroll: applyScroll(job.number, job.notches, state); break;
190
+ case InputJob::Kind::locks: applyLocks(state); break;
191
+ case InputJob::Kind::flush: break;
192
+ case InputJob::Kind::fence:
193
+ if (!CGPreflightPostEventAccess() && job.fence) job.fence->complete(false, job.dropped);
194
+ else job.complete();
195
+ break;
196
+ }
197
+ }
198
+ /// UTF-16 units for one codepoint, as a surrogate pair above the BMP; returns the unit count.
199
+ static UniCharCount unicodeUnits(uint32_t codepoint, UniChar units[2]) {
200
+ if (codepoint < 0x10000) { units[0] = UniChar(codepoint); return 1; }
201
+ units[0] = UniChar(0xd800 + ((codepoint - 0x10000) >> 10));
202
+ units[1] = UniChar(0xdc00 + ((codepoint - 0x10000) & 0x3ff));
203
+ return 2;
204
+ }
205
+ void type(uint32_t codepoint) {
206
+ if (codepoint == '\n' || codepoint == '\t') {
207
+ const NativeKey key = NativeKey(codepoint == '\n' ? kVK_Return : kVK_Tab);
208
+ send(keyEvent(key, true, 0));
209
+ send(keyEvent(key, false, 0));
210
+ return;
211
+ }
212
+ UniChar units[2];
213
+ const UniCharCount count = unicodeUnits(codepoint, units);
214
+ for (const bool press : {true, false}) {
215
+ CGEventRef event = CGEventCreateKeyboardEvent(source, 0, press);
216
+ if (!event) continue;
217
+ CGEventKeyboardSetUnicodeString(event, count, units);
218
+ send(withFlags(event, 0));
219
+ }
220
+ }
221
+ void applyButton(unsigned number, bool press, CGEventFlags state) {
222
+ unsigned count = 1;
223
+ if (press) {
224
+ count = nextClicks(number, position);
225
+ heldButtons[number] = count;
226
+ } else {
227
+ const auto found = heldButtons.find(number);
228
+ if (found == heldButtons.end()) return;
229
+ count = found->second;
230
+ heldButtons.erase(found);
231
+ }
232
+ const CGEventType type = number == 1 ? (press ? kCGEventLeftMouseDown : kCGEventLeftMouseUp)
233
+ : number == 3 ? (press ? kCGEventRightMouseDown : kCGEventRightMouseUp)
234
+ : (press ? kCGEventOtherMouseDown : kCGEventOtherMouseUp);
235
+ const CGMouseButton button = number == 1 ? kCGMouseButtonLeft
236
+ : number == 3 ? kCGMouseButtonRight : kCGMouseButtonCenter;
237
+ send(mouseEvent(type, position, button, count, state));
238
+ }
239
+ void applyScroll(unsigned axis, int notches, CGEventFlags state) {
240
+ const int32_t amount = scrollSign * notches;
241
+ CGEventRef event = axis == 0
242
+ ? CGEventCreateScrollWheelEvent2(source, kCGScrollEventUnitLine, 1, amount, 0, 0)
243
+ : CGEventCreateScrollWheelEvent2(source, kCGScrollEventUnitLine, 2, 0, amount, 0);
244
+ send(withFlags(event, state));
245
+ }
246
+ void applyLocks(CGEventFlags state) {
247
+ if (CGEventSourceFlagsState(kCGEventSourceStateHIDSystemState) & kCGEventFlagMaskAlphaShift) {
248
+ send(keyEvent(NativeKey(kVK_CapsLock), true, state));
249
+ send(keyEvent(NativeKey(kVK_CapsLock), false, state));
250
+ }
251
+ capsLock = false;
252
+ }
253
+ void releaseHeld() {
254
+ while (!held.empty()) {
255
+ const NativeKey key = *held.begin();
256
+ held.erase(held.begin());
257
+ send(keyEvent(key, false, flags()));
258
+ }
259
+ while (!heldButtons.empty()) applyButton(heldButtons.begin()->first, false, flags());
260
+ }
261
+ CGPoint globalPoint(int x, int y) const {
262
+ return CGPointMake(bounds.origin.x + (x + 0.5) * bounds.size.width / std::max(1, int(width)),
263
+ bounds.origin.y + (y + 0.5) * bounds.size.height / std::max(1, int(height)));
264
+ }
265
+ CGEventType motionType() const {
266
+ if (heldButtons.count(1)) return kCGEventLeftMouseDragged;
267
+ if (heldButtons.count(3)) return kCGEventRightMouseDragged;
268
+ if (heldButtons.count(2)) return kCGEventOtherMouseDragged;
269
+ return kCGEventMouseMoved;
270
+ }
271
+ CGMouseButton dragButton() const {
272
+ if (heldButtons.count(3)) return kCGMouseButtonRight;
273
+ return heldButtons.count(2) ? kCGMouseButtonCenter : kCGMouseButtonLeft;
274
+ }
275
+ unsigned nextClicks(unsigned number, CGPoint where) {
276
+ const auto now = std::chrono::steady_clock::now();
277
+ const bool near = std::abs(where.x - clickPoint.x) <= clickSlop && std::abs(where.y - clickPoint.y) <= clickSlop;
278
+ clicks = number == clickButton && near && now - clickTime <= clickInterval ? std::min(clicks + 1, 3u) : 1u;
279
+ clickButton = number; clickPoint = where; clickTime = now;
280
+ return clicks;
281
+ }
282
+ };
283
+ }
284
+ namespace macos {
285
+ std::unique_ptr<InputBackend> makeInputBackend() { return std::make_unique<MacInput>(); }
286
+ }
287
+ }
@@ -0,0 +1,34 @@
1
+ #pragma once
2
+ #include <cstdint>
3
+
4
+ namespace relay {
5
+ namespace macos {
6
+ constexpr uint8_t noKey = 0xff;
7
+ constexpr uint8_t virtualKeys[195] = {
8
+ 0xff, 0x35, 0x12, 0x13, 0x14, 0x15, 0x17, 0x16, 0x1a, 0x1c, 0x19, 0x1d, 0x1b, 0x18, 0x33,
9
+ 0x30, 0x0c, 0x0d, 0x0e, 0x0f, 0x11, 0x10, 0x20, 0x22, 0x1f, 0x23, 0x21, 0x1e, 0x24, 0x3b,
10
+ 0x00, 0x01, 0x02, 0x03, 0x05, 0x04, 0x26, 0x28, 0x25, 0x29, 0x27, 0x32, 0x38, 0x2a, 0x06,
11
+ 0x07, 0x08, 0x09, 0x0b, 0x2d, 0x2e, 0x2b, 0x2f, 0x2c, 0x3c, 0x43, 0x3a, 0x31, 0x39, 0x7a,
12
+ 0x78, 0x63, 0x76, 0x60, 0x61, 0x62, 0x64, 0x65, 0x6d, 0x47, 0xff, 0x59, 0x5b, 0x5c, 0x4e,
13
+ 0x56, 0x57, 0x58, 0x45, 0x53, 0x54, 0x55, 0x52, 0x41, 0xff, 0xff, 0x0a, 0x67, 0x6f, 0xff,
14
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x4c, 0x3e, 0x4b, 0xff, 0x3d, 0xff, 0x73, 0x7e, 0x74,
15
+ 0x7b, 0x7c, 0x77, 0x7d, 0x79, 0x72, 0x75, 0xff, 0xff, 0xff, 0xff, 0xff, 0x51, 0xff, 0xff,
16
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0x37, 0x36, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
17
+ 0xff, 0xff, 0xff, 0xff, 0x6e, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
18
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
19
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
20
+ 0xff, 0xff, 0xff, 0x69, 0x6b, 0x71, 0x6a, 0x40, 0x4f, 0x50, 0x5a, 0xff, 0xff, 0xff, 0xff,
21
+ };
22
+ /// macOS virtual key code for an evdev keycode, or -1 when the key has no macOS equivalent.
23
+ constexpr int macKey(int keycode) {
24
+ if (keycode <= 0 || keycode >= int(sizeof virtualKeys)) return -1;
25
+ const uint8_t code = virtualKeys[keycode];
26
+ return code == noKey ? -1 : code;
27
+ }
28
+ static_assert(macKey(30) == 0x00 && macKey(44) == 0x06 && macKey(2) == 0x12 && macKey(1) == 0x35 &&
29
+ macKey(14) == 0x33 && macKey(15) == 0x30 && macKey(28) == 0x24 && macKey(57) == 0x31 &&
30
+ macKey(42) == 0x38 && macKey(126) == 0x36 && macKey(103) == 0x7e && macKey(59) == 0x7a &&
31
+ macKey(88) == 0x6f && macKey(96) == 0x4c && macKey(86) == 0x0a && macKey(0) == -1 &&
32
+ macKey(70) == -1 && macKey(99) == -1 && macKey(119) == -1 && macKey(194) == -1);
33
+ }
34
+ }