screen-relay 0.0.0 → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. package/LICENSE +201 -0
  2. package/Makefile +165 -0
  3. package/README.md +112 -1
  4. package/THIRD-PARTY-NOTICES.md +75 -0
  5. package/dist/cli/client.d.ts +28 -0
  6. package/dist/cli/commands.d.ts +52 -0
  7. package/dist/cli/config.d.ts +32 -0
  8. package/dist/cli/format.d.ts +15 -0
  9. package/dist/cli/help.d.ts +5 -0
  10. package/dist/cli/host/common.d.ts +23 -0
  11. package/dist/cli/host/doctor.d.ts +2 -0
  12. package/dist/cli/host/packages.d.ts +13 -0
  13. package/dist/cli/host/session_commands.d.ts +8 -0
  14. package/dist/cli/host/setup.d.ts +22 -0
  15. package/dist/cli/host/state.d.ts +23 -0
  16. package/dist/cli/host/supervisor.d.ts +94 -0
  17. package/dist/cli/host/watcher.d.ts +4 -0
  18. package/dist/cli/host/windows.d.ts +82 -0
  19. package/dist/cli/main.d.ts +3 -0
  20. package/dist/cli/mode.d.ts +14 -0
  21. package/dist/cli/output.d.ts +24 -0
  22. package/dist/cli/parser.d.ts +6 -0
  23. package/dist/cli/relay.mjs +12631 -0
  24. package/dist/cli/skills.d.ts +21 -0
  25. package/dist/cli/theme.d.ts +14 -0
  26. package/dist/protocol/errors.d.ts +155 -0
  27. package/dist/protocol/index.d.ts +119 -0
  28. package/dist/protocol/index.js +418 -0
  29. package/dist/sdk/index.d.ts +78 -0
  30. package/dist/sdk/index.js +834 -0
  31. package/dist/sdk/transport.d.ts +20 -0
  32. package/dist/viewer/index.html +96 -0
  33. package/dist/viewer/viewer.js +1079 -0
  34. package/package.json +47 -7
  35. package/packages.tsv +38 -0
  36. package/skills/core/SKILL.md +171 -0
  37. package/skills/core/references/cli.md +119 -0
  38. package/skills/core/references/config.md +48 -0
  39. package/skills/core/references/macos.md +50 -0
  40. package/skills/core/references/sdk.md +69 -0
  41. package/skills/core/references/troubleshooting.md +33 -0
  42. package/skills/core/references/wayland.md +47 -0
  43. package/skills/screen-relay/SKILL.md +24 -0
  44. package/skills/vercel-sandbox/SKILL.md +68 -0
  45. package/src/engine/backends/linux/keycodes.hpp +35 -0
  46. package/src/engine/backends/linux/wayland/capture.cpp +298 -0
  47. package/src/engine/backends/linux/wayland/input.cpp +559 -0
  48. package/src/engine/backends/linux/wayland/keymap.cpp +118 -0
  49. package/src/engine/backends/linux/wayland/keymap.hpp +54 -0
  50. package/src/engine/backends/linux/wayland/protocols/virtual-keyboard-unstable-v1.xml +112 -0
  51. package/src/engine/backends/linux/wayland/protocols/wlr-screencopy-unstable-v1.xml +234 -0
  52. package/src/engine/backends/linux/wayland/protocols/wlr-virtual-pointer-unstable-v1.xml +152 -0
  53. package/src/engine/backends/linux/wayland/wayland.cpp +166 -0
  54. package/src/engine/backends/linux/wayland/wayland.hpp +60 -0
  55. package/src/engine/backends/linux/x11/capture.cpp +206 -0
  56. package/src/engine/backends/linux/x11/input.cpp +266 -0
  57. package/src/engine/backends/macos/capture.mm +367 -0
  58. package/src/engine/backends/macos/display.hpp +20 -0
  59. package/src/engine/backends/macos/display.mm +84 -0
  60. package/src/engine/backends/macos/input.mm +287 -0
  61. package/src/engine/backends/macos/keys.hpp +34 -0
  62. package/src/engine/backends/windows/capture.cpp +393 -0
  63. package/src/engine/backends/windows/input.cpp +274 -0
  64. package/src/engine/backends/windows/output.cpp +65 -0
  65. package/src/engine/backends/windows/output.hpp +26 -0
  66. package/src/engine/core/capture-bucket.hpp +40 -0
  67. package/src/engine/core/capture-source.hpp +59 -0
  68. package/src/engine/core/capture.cpp +565 -0
  69. package/src/engine/core/capture.hpp +58 -0
  70. package/src/engine/core/color.cpp +176 -0
  71. package/src/engine/core/color.hpp +16 -0
  72. package/src/engine/core/cpu-budget.hpp +196 -0
  73. package/src/engine/core/flow.hpp +317 -0
  74. package/src/engine/core/input.cpp +98 -0
  75. package/src/engine/core/input.hpp +115 -0
  76. package/src/engine/core/wait.cpp +125 -0
  77. package/src/engine/core/wait.hpp +41 -0
  78. package/src/engine/server/backend.cpp +65 -0
  79. package/src/engine/server/backend.hpp +18 -0
  80. package/src/engine/server/config.cpp +257 -0
  81. package/src/engine/server/config.hpp +24 -0
  82. package/src/engine/server/main.cpp +82 -0
  83. package/src/engine/server/metrics.cpp +47 -0
  84. package/src/engine/server/metrics.hpp +10 -0
  85. package/src/engine/server/screenshot.cpp +108 -0
  86. package/src/engine/server/screenshot.hpp +31 -0
  87. package/src/engine/server/server.cpp +535 -0
  88. package/src/engine/server/server.hpp +89 -0
  89. package/src/engine/server/wire.cpp +70 -0
  90. package/src/engine/server/wire.hpp +29 -0
  91. package/index.js +0 -2
@@ -0,0 +1,565 @@
1
+ #include "capture.hpp"
2
+ #include "capture-bucket.hpp"
3
+ #include "color.hpp"
4
+ #include "flow.hpp"
5
+ #include "capture-source.hpp"
6
+ #include "cpu-budget.hpp"
7
+ #include "wait.hpp"
8
+
9
+ #include <cstdint>
10
+ #include <x264.h>
11
+ #include <algorithm>
12
+ #include <atomic>
13
+ #include <cassert>
14
+ #include <chrono>
15
+ #include <condition_variable>
16
+ #include <cstdio>
17
+ #include <cstring>
18
+ #include <limits>
19
+ #include <mutex>
20
+ #include <stdexcept>
21
+ #include <thread>
22
+
23
+ namespace relay {
24
+ namespace {
25
+ /// Motion frames get 4 frame budgets; forced keyframes and refinement get 8.
26
+ constexpr int vbvBufferKbit(uint64_t bitrate, unsigned fps, bool wide) {
27
+ const uint64_t motion = 4 * bitrate / (1000ull * fps);
28
+ const uint64_t legacy_wide = 8 * bitrate / (1000ull * fps);
29
+ return int(std::max<uint64_t>(1, wide ? std::max(motion, legacy_wide) : motion));
30
+ }
31
+ static_assert(vbvBufferKbit(1000000, 10, false) == 400);
32
+ static_assert(vbvBufferKbit(1000000, 30, false) == 133);
33
+ static_assert(vbvBufferKbit(1000000, 60, false) == 66);
34
+ static_assert(vbvBufferKbit(250000, 60, false) == 16);
35
+ static_assert(vbvBufferKbit(1000000, 10, true) == 800);
36
+ static_assert(vbvBufferKbit(1000000, 30, true) == 266);
37
+ static_assert(vbvBufferKbit(250000, 60, true) == 33);
38
+ static_assert(vbvBufferKbit(1000000, 120, false) == 33);
39
+ static_assert(vbvBufferKbit(1000000, 120, true) == 66);
40
+ static_assert(vbvBufferKbit(62500, 120, false) == 2);
41
+ static_assert(vbvBufferKbit(62500, 120, true) == 4);
42
+ constexpr float encoder_crf = 10;
43
+ /// Changed macroblocks above which damage conversion is split across worker threads.
44
+ constexpr int parallel_convert_macroblocks = 512;
45
+ using Clock = std::chrono::steady_clock;
46
+
47
+ uint64_t ns(Clock::duration d) {
48
+ return std::chrono::duration_cast<std::chrono::nanoseconds>(d).count();
49
+ }
50
+ bool hasNal(const std::vector<uint8_t>& bytes, uint8_t type) {
51
+ for (size_t i = 0; i + 3 < bytes.size(); ++i) {
52
+ if (bytes[i] == 0 && bytes[i + 1] == 0 && bytes[i + 2] == 1 &&
53
+ (bytes[i + 3] & 31) == type) return true;
54
+ }
55
+ return false;
56
+ }
57
+ void require(bool ok, const char* message) {
58
+ if (!ok) throw std::runtime_error(message);
59
+ }
60
+ /// Runs one job on every thread at once, the caller included, and returns when all of them finish.
61
+ class Workers {
62
+ public:
63
+ explicit Workers(int count) {
64
+ for (int i = 1; i < count; ++i) threads.emplace_back([this] { loop(); });
65
+ }
66
+ ~Workers() {
67
+ { std::lock_guard<std::mutex> lock(mutex); stopping = true; }
68
+ wake.notify_all();
69
+ for (auto& thread : threads) thread.join();
70
+ }
71
+ int size() const { return int(threads.size()) + 1; }
72
+ void run(const std::function<void(int)>& job) {
73
+ { std::lock_guard<std::mutex> lock(mutex); task = &job; next = 1; pending = int(threads.size()); ++round; }
74
+ wake.notify_all();
75
+ job(0);
76
+ std::unique_lock<std::mutex> lock(mutex);
77
+ finished.wait(lock, [this] { return pending == 0; });
78
+ task = nullptr;
79
+ }
80
+ private:
81
+ void loop() {
82
+ uint64_t seen = 0;
83
+ std::unique_lock<std::mutex> lock(mutex);
84
+ while (true) {
85
+ wake.wait(lock, [&] { return stopping || round != seen; });
86
+ if (stopping) return;
87
+ seen = round;
88
+ const int part = next++;
89
+ const auto* job = task;
90
+ lock.unlock();
91
+ (*job)(part);
92
+ lock.lock();
93
+ if (--pending == 0) finished.notify_one();
94
+ }
95
+ }
96
+ std::vector<std::thread> threads;
97
+ std::mutex mutex;
98
+ std::condition_variable wake, finished;
99
+ const std::function<void(int)>* task = nullptr;
100
+ uint64_t round = 0;
101
+ int next = 0, pending = 0;
102
+ bool stopping = false;
103
+ };
104
+ struct Resources {
105
+ PixelFormat format;
106
+ int width, height;
107
+ CaptureSettings settings;
108
+ x264_t* codec = nullptr;
109
+ x264_picture_t frame{};
110
+ std::vector<uint8_t> mb_info;
111
+ int threads;
112
+ std::vector<std::vector<uint8_t>> scratch;
113
+ Workers workers;
114
+ bool frame_allocated = false;
115
+ uint64_t bitrate;
116
+ bool wide_vbv = false;
117
+ Resources(PixelFormat f, int w, int h, CaptureSettings s)
118
+ : format(f), width(w), height(h), settings(s),
119
+ mb_info(size_t((w + 15) / 16) * size_t((h + 15) / 16)),
120
+ threads(encoderThreads(h)),
121
+ scratch(size_t(threads), std::vector<uint8_t>(size_t(w) * 24)), workers(threads),
122
+ bitrate(s.start_bitrate) {}
123
+ ~Resources() {
124
+ if (codec) x264_encoder_close(codec);
125
+ if (frame_allocated) x264_picture_clean(&frame);
126
+ }
127
+ void openEncoder() {
128
+ if (codec) x264_encoder_close(codec);
129
+ codec = nullptr;
130
+ x264_param_t params;
131
+ require(x264_param_default_preset(&params, "ultrafast", "zerolatency") >= 0,
132
+ "setting x264 options failed");
133
+ params.i_width = width; params.i_height = height; params.i_csp = X264_CSP_NV12;
134
+ params.i_timebase_num = 1; params.i_timebase_den = 1000000;
135
+ params.i_fps_num = int(settings.fps); params.i_fps_den = 1;
136
+ params.i_threads = threads; params.b_sliced_threads = 1; params.i_keyint_max = X264_KEYINT_MAX_INFINITE; params.i_bframe = 0;
137
+ params.i_scenecut_threshold = 0; params.i_sync_lookahead = 0;
138
+ params.rc.i_rc_method = X264_RC_CRF; params.rc.f_rf_constant = encoder_crf;
139
+ params.rc.i_lookahead = 0; params.rc.i_vbv_max_bitrate = int(settings.start_bitrate / 1000); params.rc.i_vbv_buffer_size = vbvBufferKbit(settings.start_bitrate, settings.fps, true);
140
+ params.analyse.b_mb_info = 1; params.analyse.b_mb_info_update = 0;
141
+ params.b_annexb = 1; params.b_repeat_headers = 1;
142
+ params.vui.b_fullrange = 0; params.vui.i_colmatrix = 1;
143
+ params.vui.i_colorprim = 1; params.vui.i_transfer = 1;
144
+ params.vui.i_chroma_loc = 1;
145
+ codec = x264_encoder_open(&params);
146
+ require(codec, "opening libx264 failed");
147
+ bitrate = settings.start_bitrate;
148
+ wide_vbv = true;
149
+ }
150
+ void init() {
151
+ openEncoder();
152
+ require(x264_picture_alloc(&frame, X264_CSP_NV12, width, height) == 0,
153
+ "allocating YUV buffer failed");
154
+ frame_allocated = true;
155
+ }
156
+ bool convertDamage(const FrameView& view, const std::vector<DamageRect>& rects) {
157
+ const int columns = (width + 15) / 16;
158
+ std::fill(mb_info.begin(), mb_info.end(), uint8_t(X264_MBINFO_CONSTANT));
159
+ int damaged = 0;
160
+ for (const auto& rect : rects) {
161
+ const int64_t x0 = std::max<int64_t>(view.x, rect.x);
162
+ const int64_t y0 = std::max<int64_t>(view.y, rect.y);
163
+ const int64_t x1 = std::min<int64_t>(view.x + view.width, int64_t(rect.x) + rect.width);
164
+ const int64_t y1 = std::min<int64_t>(view.y + view.height, int64_t(rect.y) + rect.height);
165
+ if (x1 <= x0 || y1 <= y0) continue;
166
+ for (int row = int(y0) / 16; row <= (y1 - 1) / 16; ++row)
167
+ for (int column = int(x0) / 16; column <= (x1 - 1) / 16; ++column)
168
+ if (mb_info[row * columns + column]) { mb_info[row * columns + column] = 0; ++damaged; }
169
+ }
170
+ const int first = view.y / 16, rows = (view.y + view.height - 1) / 16 - first + 1;
171
+ std::atomic<bool> changed{false}, failed{false};
172
+ const int parts = damaged >= parallel_convert_macroblocks ? workers.size() : 1;
173
+ const std::function<void(int)> job = [&](int part) {
174
+ bool different = false;
175
+ if (!convertRows(view, first + rows * part / parts, first + rows * (part + 1) / parts, scratch[part], different))
176
+ failed = true;
177
+ if (different) changed = true;
178
+ };
179
+ if (parts > 1) workers.run(job); else job(0);
180
+ require(!failed, "RGB conversion failed");
181
+ return changed;
182
+ }
183
+ bool convertRows(const FrameView& view, int begin, int end, std::vector<uint8_t>& buffer, bool& changed) {
184
+ const int columns = (width + 15) / 16;
185
+ const auto convert = view.format == PixelFormat::BGRA ? bgraToNv12 : argbToNv12;
186
+ for (int row = begin; row < end; ++row) {
187
+ const int y0 = std::max(row * 16, view.y);
188
+ const int y1 = std::min(row * 16 + 16, view.y + view.height);
189
+ for (int column = view.x / 16; column <= (view.x + view.width - 1) / 16;) {
190
+ if (mb_info[row * columns + column]) { ++column; continue; }
191
+ const int first = column;
192
+ while (column <= (view.x + view.width - 1) / 16 && !mb_info[row * columns + column]) ++column;
193
+ const int x0 = std::max(first * 16, view.x);
194
+ const int x1 = std::min(column * 16, view.x + view.width);
195
+ if (!convert(view.data + (y0 - view.y) * view.stride + (x0 - view.x) * 4, view.stride,
196
+ buffer.data(), width, buffer.data() + width * 16, width, x1 - x0, y1 - y0)) return false;
197
+ for (int block = first; block < column; ++block) {
198
+ const int left = std::max(block * 16, x0);
199
+ const int right = std::min(block * 16 + 16, x1);
200
+ bool different = false;
201
+ for (int plane = 0; plane < 2; ++plane) {
202
+ const int lines = plane ? (y1 - y0) / 2 : y1 - y0;
203
+ uint8_t* target = frame.img.plane[plane] + (plane ? y0 / 2 : y0) * frame.img.i_stride[plane] + left;
204
+ const uint8_t* source = buffer.data() + (plane ? width * 16 : 0) + left - x0;
205
+ for (int line = 0; line < lines; ++line) {
206
+ if (std::memcmp(target, source, right - left) != 0) {
207
+ std::memcpy(target, source, right - left);
208
+ different = true;
209
+ }
210
+ target += frame.img.i_stride[plane];
211
+ source += width;
212
+ }
213
+ }
214
+ mb_info[row * columns + block] = different ? 0 : uint8_t(X264_MBINFO_CONSTANT);
215
+ changed |= different;
216
+ }
217
+ }
218
+ }
219
+ return true;
220
+ }
221
+ };
222
+ } // namespace
223
+
224
+ struct CaptureEncoder::Impl {
225
+ explicit Impl(Publish callback, CaptureSettings options, CaptureSourceFactory factory, PublishCursor cursor)
226
+ : source_factory(std::move(factory)), publish(std::move(callback)), publish_cursor(std::move(cursor)),
227
+ settings(options) {}
228
+ CaptureSourceFactory source_factory;
229
+ Publish publish;
230
+ PublishCursor publish_cursor;
231
+ CaptureSettings settings;
232
+ mutable std::mutex mutex;
233
+ std::condition_variable ready;
234
+ std::thread worker;
235
+ Wake wake_event;
236
+ bool stopping = false, initialized = false, running = false;
237
+ std::string startup_error;
238
+ std::string fatal_error;
239
+ uint32_t generation = 0, sequence = 0, acknowledged = 0;
240
+ unsigned window = 1;
241
+ uint64_t bitrate = settings.start_bitrate, admit_until_us = 0;
242
+ bool paused = false;
243
+ uint64_t revision = 0;
244
+ bool active = false;
245
+ CaptureStats counters;
246
+ void wake() { wake_event.signal(); }
247
+ void run() {
248
+ try {
249
+ auto source = source_factory();
250
+ require(bool(source), "capture source factory returned null");
251
+ require(validWaitHandle(source->fd()), "invalid capture source fd");
252
+ const auto format = source->pixelFormat();
253
+ require(format == PixelFormat::BGRA || format == PixelFormat::ARGB, "unsupported capture pixel format");
254
+ const auto dimensions = source->dimensions();
255
+ const int width = dimensions.first, height = dimensions.second;
256
+ requireSurfaceSize(width, height);
257
+ Resources r(format, width, height, settings);
258
+ r.init();
259
+ { std::lock_guard<std::mutex> lock(mutex);
260
+ counters.cursor_supported = source->cursorSupported(); initialized = true; running = true;
261
+ }
262
+ ready.notify_all();
263
+ bool dirty = true, force_idr = true;
264
+ uint64_t seen_revision = 0;
265
+ int64_t last_pts = -1;
266
+ std::vector<DamageRect> rects;
267
+ unsigned consecutive_failures = 0;
268
+ const auto frame_interval = std::chrono::nanoseconds(1000000000LL / settings.fps);
269
+ const auto refinement_delay = std::chrono::milliseconds(150);
270
+ constexpr unsigned refinement_max_frames = 8;
271
+ CaptureBucket bucket;
272
+ const auto started = Clock::now();
273
+ bucket.reset(ns(started.time_since_epoch()), ns(frame_interval));
274
+ auto last_publish = started;
275
+ auto retry_after = Clock::time_point::min();
276
+ bool last_was_refinement = true, refining = false;
277
+ unsigned cascade_frames = 0;
278
+ uint64_t first_refinement_size = 0;
279
+ auto reset_refinement = [&] {
280
+ refining = false;
281
+ cascade_frames = 0;
282
+ first_refinement_size = 0;
283
+ };
284
+ while (true) {
285
+ if (source->drainDamage()) dirty = true;
286
+ uint32_t gen, seq;
287
+ uint64_t rev, rate;
288
+ bool admitted, cursor_active, queue_empty, refresh_cursor = false;
289
+ {
290
+ std::lock_guard<std::mutex> lock(mutex);
291
+ if (stopping) break;
292
+ if (revision != seen_revision) {
293
+ seen_revision = revision; dirty = true; force_idr = true;
294
+ reset_refinement();
295
+ refresh_cursor = true;
296
+ }
297
+ gen = generation; rev = revision; rate = bitrate;
298
+ cursor_active = gen != 0 && active && bool(publish_cursor);
299
+ queue_empty = sequence == acknowledged;
300
+ admitted = gen != 0 && active && !paused &&
301
+ (!admit_until_us || uint64_t(std::chrono::duration_cast<std::chrono::microseconds>(
302
+ Clock::now().time_since_epoch()).count()) < admit_until_us) &&
303
+ sequence - acknowledged < window &&
304
+ sequence < std::numeric_limits<uint32_t>::max();
305
+ seq = sequence + (admitted ? 1 : 0);
306
+ }
307
+ bool cursor_published = false;
308
+ if (cursor_active) {
309
+ if (auto cursor = source->cursor(refresh_cursor)) {
310
+ publish_cursor(gen, std::move(*cursor)); cursor_published = true;
311
+ }
312
+ }
313
+ const auto now = Clock::now();
314
+ const int64_t now_ns = ns(now.time_since_epoch());
315
+ const bool refinement_pending = !dirty && refining;
316
+ const auto refinement_pace = last_was_refinement ? frame_interval : refinement_delay;
317
+ const bool refinement_ready = !refinement_pending || now >= last_publish + refinement_pace;
318
+ if (admitted && (dirty || (refinement_pending && queue_empty)) && now >= retry_after &&
319
+ refinement_ready && bucket.ready(now_ns)) {
320
+ bool refinement = !dirty;
321
+ const bool captured_frame = !refinement;
322
+ const auto capture_start = Clock::now();
323
+ try {
324
+ assert(!refinement || (!force_idr && last_pts >= 0));
325
+ dirty = false;
326
+ auto captured = capture_start, converted = capture_start;
327
+ if (!refinement) {
328
+ const FrameView view = source->captureChanged(force_idr);
329
+ captured = Clock::now();
330
+ require(view.data && view.width > 0 && view.height > 0 &&
331
+ view.x >= 0 && view.x <= width && view.y >= 0 && view.y <= height &&
332
+ view.width <= width - view.x && view.height <= height - view.y &&
333
+ view.stride >= view.width * 4 && view.stride <= width * 8 &&
334
+ !(view.x & 1) && !(view.y & 1) && !(view.width & 1) && !(view.height & 1) &&
335
+ !(force_idr && (view.width != width || view.height != height)) &&
336
+ view.format == r.format, "invalid capture frame");
337
+ bool changed = true;
338
+ if (force_idr) {
339
+ const auto convert = view.format == PixelFormat::BGRA ? bgraToNv12 : argbToNv12;
340
+ require(convert(view.data, view.stride, r.frame.img.plane[0], r.frame.img.i_stride[0],
341
+ r.frame.img.plane[1], r.frame.img.i_stride[1], view.width, view.height),
342
+ "RGB conversion failed");
343
+ } else {
344
+ if (!source->changedRects(rects)) rects = {{view.x, view.y, view.width, view.height}};
345
+ changed = r.convertDamage(view, rects);
346
+ }
347
+ converted = Clock::now();
348
+ if (!changed) {
349
+ if (refining && queue_empty && converted >= last_publish + refinement_pace) {
350
+ refinement = true;
351
+ } else {
352
+ { std::lock_guard<std::mutex> lock(mutex);
353
+ ++counters.captures;
354
+ counters.capture_ns += ns(captured - capture_start);
355
+ counters.convert_ns += ns(converted - captured);
356
+ }
357
+ consecutive_failures = 0;
358
+ bucket.spend(ns(converted.time_since_epoch()));
359
+ continue;
360
+ }
361
+ }
362
+ }
363
+ const int64_t timestamp = std::chrono::duration_cast<std::chrono::microseconds>(capture_start.time_since_epoch()).count();
364
+ r.frame.i_pts = last_pts = std::max(last_pts + 1, timestamp);
365
+ const bool wide_vbv = refinement || force_idr;
366
+ if (rate != r.bitrate || wide_vbv != r.wide_vbv) {
367
+ x264_param_t params;
368
+ x264_encoder_parameters(r.codec, &params);
369
+ params.rc.i_vbv_max_bitrate = int(rate / 1000);
370
+ params.rc.i_vbv_buffer_size = vbvBufferKbit(rate, settings.fps, wide_vbv);
371
+ require(x264_encoder_reconfig(r.codec, &params) == 0, "encoder reconfiguration failed");
372
+ r.bitrate = rate;
373
+ r.wide_vbv = wide_vbv;
374
+ }
375
+ r.frame.i_type = force_idr ? X264_TYPE_IDR : X264_TYPE_AUTO;
376
+ r.frame.prop.mb_info = nullptr; r.frame.prop.mb_info_free = nullptr;
377
+ if (!refinement && !force_idr) r.frame.prop.mb_info = r.mb_info.data();
378
+ x264_nal_t* nals = nullptr;
379
+ int nal_count = 0;
380
+ x264_picture_t picture;
381
+ const int size = x264_encoder_encode(r.codec, &nals, &nal_count, &r.frame, &picture);
382
+ require(size > 0 && nal_count > 0 && nals && picture.i_pts == last_pts &&
383
+ static_cast<size_t>(size) <= FlowControl::max_bytes,
384
+ "invalid or delayed encoded access unit");
385
+ const bool converged = refinement && picture.prop.f_crf_avg <= encoder_crf + 1 &&
386
+ uint64_t(size) <= rate / (4ull * settings.fps);
387
+ EncodedFrame output;
388
+ output.generation = gen; output.sequence = seq; output.pts_us = last_pts;
389
+ output.width = uint16_t(width); output.height = uint16_t(height);
390
+ output.keyframe = picture.b_keyframe;
391
+ output.annex_b.assign(nals[0].p_payload, nals[0].p_payload + size);
392
+ require(!force_idr || (output.keyframe &&
393
+ hasNal(output.annex_b, 5) && hasNal(output.annex_b, 7) && hasNal(output.annex_b, 8)),
394
+ "encoder buffered frame or omitted IDR/SPS/PPS");
395
+ const auto encoded = Clock::now();
396
+ bucket.spend(ns(encoded.time_since_epoch()));
397
+ bool current;
398
+ {
399
+ std::lock_guard<std::mutex> lock(mutex);
400
+ if (captured_frame) ++counters.captures;
401
+ ++counters.encoded;
402
+ counters.capture_ns += ns(captured - capture_start);
403
+ counters.convert_ns += ns(converted - captured);
404
+ counters.encode_ns += ns(encoded - converted);
405
+ current = revision == rev && generation == gen && active;
406
+ if (current) sequence = seq;
407
+ }
408
+ if (current) {
409
+ if (!publish(std::move(output))) {
410
+ // A rejected reference packet stalls this generation until transport recovery.
411
+ std::fprintf(stderr, "capture: publish rejected; awaiting new session\n");
412
+ { std::lock_guard<std::mutex> lock(mutex); if (generation == gen) active = false; }
413
+ dirty = true; force_idr = true;
414
+ reset_refinement();
415
+ } else {
416
+ force_idr = false;
417
+ last_publish = Clock::now();
418
+ last_was_refinement = refinement;
419
+ if (refinement) {
420
+ if (cascade_frames == 0) first_refinement_size = uint64_t(size);
421
+ ++cascade_frames;
422
+ refining = !converged && cascade_frames < refinement_max_frames &&
423
+ uint64_t(size) * 8 > first_refinement_size;
424
+ } else {
425
+ cascade_frames = 0;
426
+ first_refinement_size = 0;
427
+ refining = true;
428
+ }
429
+ }
430
+ } else {
431
+ dirty = true;
432
+ force_idr = true;
433
+ reset_refinement();
434
+ }
435
+ consecutive_failures = 0;
436
+ } catch (const std::exception& error) {
437
+ std::fprintf(stderr, "capture: %s; retrying with IDR\n", error.what());
438
+ dirty = true;
439
+ force_idr = true;
440
+ reset_refinement();
441
+ if (++consecutive_failures >= 3) {
442
+ throw std::runtime_error(std::string("capture failed three consecutive times: ") + error.what());
443
+ }
444
+ r.openEncoder();
445
+ retry_after = Clock::now() + std::chrono::seconds(1);
446
+ }
447
+ continue;
448
+ }
449
+ // A synchronous cursor read may have buffered display events in Xlib.
450
+ if (cursor_published) continue;
451
+ int64_t timeout_ns = -1;
452
+ if (admitted && (dirty || (refinement_pending && queue_empty))) {
453
+ const auto wait_now = Clock::now();
454
+ const int64_t wait_now_ns = ns(wait_now.time_since_epoch());
455
+ auto wake_at = wait_now + std::chrono::nanoseconds(bucket.wait_ns(wait_now_ns));
456
+ if (retry_after > wake_at) wake_at = retry_after;
457
+ if (refinement_pending && queue_empty && last_publish + refinement_pace > wake_at)
458
+ wake_at = last_publish + refinement_pace;
459
+ const auto remaining = std::chrono::duration_cast<std::chrono::nanoseconds>(
460
+ wake_at - wait_now).count();
461
+ timeout_ns = std::max<int64_t>(0, remaining);
462
+ }
463
+ if (wait(source->fd(), wake_event, timeout_ns).wake) wake_event.consume();
464
+ }
465
+ } catch (const std::exception& error) {
466
+ std::fprintf(stderr, "capture worker stopped: %s\n", error.what());
467
+ {
468
+ std::lock_guard<std::mutex> lock(mutex);
469
+ fatal_error = error.what();
470
+ if (!initialized) startup_error = fatal_error;
471
+ initialized = true;
472
+ running = false;
473
+ active = false;
474
+ }
475
+ ready.notify_all();
476
+ }
477
+ std::lock_guard<std::mutex> lock(mutex); running = false;
478
+ }
479
+ };
480
+
481
+ CaptureEncoder::CaptureEncoder(Publish publish, CaptureSettings settings, CaptureSourceFactory factory, PublishCursor cursor)
482
+ : impl_(std::make_unique<Impl>(std::move(publish), settings, std::move(factory), std::move(cursor))) {}
483
+ CaptureEncoder::~CaptureEncoder() { stop(); }
484
+ bool CaptureEncoder::start(std::string& error) {
485
+ auto& i = *impl_;
486
+ std::unique_lock<std::mutex> lock(i.mutex);
487
+ if (i.worker.joinable()) { error = "capture already started"; return false; }
488
+ if (!i.publish) { error = "capture publish callback missing"; return false; }
489
+ if (!i.wake_event.valid()) { error = "creating capture wake event failed"; return false; }
490
+ i.wake_event.consume();
491
+ i.stopping = false;
492
+ i.initialized = false;
493
+ i.startup_error.clear();
494
+ i.fatal_error.clear();
495
+ try { i.worker = std::thread([&i] { i.run(); }); }
496
+ catch (const std::exception& e) { error = e.what(); return false; }
497
+ i.ready.wait(lock, [&i] { return i.initialized; });
498
+ error = i.startup_error.empty() ? i.fatal_error : i.startup_error;
499
+ const bool success = i.running;
500
+ lock.unlock();
501
+ if (!success) stop();
502
+ return success;
503
+ }
504
+ void CaptureEncoder::beginSession(uint32_t generation) {
505
+ auto& i = *impl_;
506
+ {
507
+ std::lock_guard<std::mutex> lock(i.mutex);
508
+ if (!generation || !i.running) return;
509
+ i.generation = generation;
510
+ i.sequence = 0;
511
+ i.acknowledged = 0;
512
+ i.window = 1; i.bitrate = i.settings.start_bitrate; i.paused = false; i.admit_until_us = 0;
513
+ i.active = true;
514
+ ++i.revision;
515
+ i.wake();
516
+ }
517
+ }
518
+ void CaptureEncoder::endSession(uint32_t generation) {
519
+ auto& i = *impl_;
520
+ std::lock_guard<std::mutex> lock(i.mutex);
521
+ if (generation != i.generation) return;
522
+ i.generation = 0;
523
+ i.active = false;
524
+ i.acknowledged = 0;
525
+ ++i.revision;
526
+ i.wake();
527
+ }
528
+ void CaptureEncoder::acknowledge(uint32_t generation, uint32_t sequence) {
529
+ auto& i = *impl_;
530
+ std::lock_guard<std::mutex> lock(i.mutex);
531
+ if (!generation || generation != i.generation || sequence != i.acknowledged + 1 || sequence > i.sequence) return;
532
+ i.acknowledged = sequence;
533
+ i.wake();
534
+ }
535
+ void CaptureEncoder::setFlowControl(uint32_t generation, unsigned window, uint64_t bitrate, bool paused, uint64_t admit_until_us) {
536
+ auto& i = *impl_;
537
+ std::lock_guard<std::mutex> lock(i.mutex);
538
+ if (!generation || generation != i.generation) return;
539
+ i.window = std::clamp(window, 1u, unsigned(FlowControl::max_frames));
540
+ i.bitrate = std::clamp<uint64_t>(bitrate, std::max<uint64_t>(1, i.settings.min_bitrate / 4), i.settings.max_bitrate);
541
+ i.paused = paused;
542
+ i.admit_until_us = admit_until_us;
543
+ i.wake();
544
+ }
545
+ CaptureStats CaptureEncoder::stats() const {
546
+ std::lock_guard<std::mutex> lock(impl_->mutex);
547
+ return impl_->counters;
548
+ }
549
+ std::string CaptureEncoder::error() const {
550
+ std::lock_guard<std::mutex> lock(impl_->mutex);
551
+ return impl_->fatal_error;
552
+ }
553
+ void CaptureEncoder::stop() {
554
+ auto& i = *impl_;
555
+ {
556
+ std::lock_guard<std::mutex> lock(i.mutex);
557
+ i.stopping = true;
558
+ i.wake();
559
+ }
560
+ if (i.worker.joinable()) i.worker.join();
561
+ std::lock_guard<std::mutex> lock(i.mutex);
562
+ i.generation = 0;
563
+ i.active = false;
564
+ }
565
+ } // namespace relay
@@ -0,0 +1,58 @@
1
+ #pragma once
2
+
3
+ #include "capture-source.hpp"
4
+ #include <cstdint>
5
+ #include <functional>
6
+ #include <memory>
7
+ #include <string>
8
+ #include <vector>
9
+
10
+ namespace relay {
11
+ struct EncodedFrame {
12
+ uint32_t generation = 0;
13
+ uint32_t sequence = 0;
14
+ uint64_t pts_us = 0;
15
+ uint16_t width = 0;
16
+ uint16_t height = 0;
17
+ bool keyframe = false;
18
+ std::vector<uint8_t> annex_b;
19
+ };
20
+
21
+ struct CaptureStats {
22
+ bool cursor_supported = false;
23
+ uint64_t captures = 0;
24
+ uint64_t encoded = 0;
25
+ uint64_t capture_ns = 0;
26
+ uint64_t convert_ns = 0;
27
+ uint64_t encode_ns = 0;
28
+ };
29
+
30
+ struct CaptureSettings {
31
+ unsigned fps;
32
+ uint64_t min_bitrate, start_bitrate, max_bitrate;
33
+ };
34
+
35
+ class CaptureEncoder {
36
+ public:
37
+ using Publish = std::function<bool(EncodedFrame&&)>;
38
+ using PublishCursor = std::function<void(uint32_t, CursorImage&&)>;
39
+ explicit CaptureEncoder(Publish publish, CaptureSettings settings,
40
+ CaptureSourceFactory factory,
41
+ PublishCursor cursor = {});
42
+ ~CaptureEncoder();
43
+ CaptureEncoder(const CaptureEncoder&) = delete;
44
+ CaptureEncoder& operator=(const CaptureEncoder&) = delete;
45
+ bool start(std::string& error);
46
+ void beginSession(uint32_t generation);
47
+ void endSession(uint32_t generation);
48
+ void acknowledge(uint32_t generation, uint32_t sequence);
49
+ void setFlowControl(uint32_t generation, unsigned window, uint64_t bitrate, bool paused, uint64_t admit_until_us);
50
+ CaptureStats stats() const;
51
+ std::string error() const;
52
+ void stop();
53
+
54
+ private:
55
+ struct Impl;
56
+ std::unique_ptr<Impl> impl_;
57
+ };
58
+ } // namespace relay