xbintsc 0.5.21 → 0.5.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +1 -1
- package/README.md +15 -5
- package/README.zh-CN.md +11 -4
- package/dist/src/cli/hints.js +11 -0
- package/dist/src/cli/hints.js.map +1 -1
- package/dist/src/cli/main.js +1 -1
- package/dist/src/driver/compiler.js +1 -0
- package/dist/src/driver/compiler.js.map +1 -1
- package/dist/src/driver/version.d.ts +1 -1
- package/dist/src/driver/version.js +1 -1
- package/dist/src/extensions/catalog.js +2 -1
- package/dist/src/extensions/catalog.js.map +1 -1
- package/dist/src/extensions/media/index.d.ts +27 -0
- package/dist/src/extensions/media/index.js +60 -0
- package/dist/src/extensions/media/index.js.map +1 -0
- package/dist/src/extensions/registry.d.ts +12 -0
- package/dist/src/extensions/registry.js +16 -0
- package/dist/src/extensions/registry.js.map +1 -1
- package/dist/src/index.d.ts +1 -0
- package/dist/src/index.js +1 -0
- package/dist/src/index.js.map +1 -1
- package/dist/tests/cli/hints.test.js +15 -0
- package/dist/tests/cli/hints.test.js.map +1 -1
- package/dist/tests/cli/process.test.js +12 -0
- package/dist/tests/cli/process.test.js.map +1 -1
- package/dist/tests/e2e/gui-canvas.test.js +159 -0
- package/dist/tests/e2e/gui-canvas.test.js.map +1 -1
- package/dist/tests/e2e/gui-example-canvas.test.d.ts +11 -0
- package/dist/tests/e2e/gui-example-canvas.test.js +72 -0
- package/dist/tests/e2e/gui-example-canvas.test.js.map +1 -0
- package/dist/tests/e2e/gui-helpers.d.ts +3 -0
- package/dist/tests/e2e/gui-helpers.js +2 -0
- package/dist/tests/e2e/gui-helpers.js.map +1 -1
- package/dist/tests/e2e/gui-media-disabled.test.d.ts +9 -0
- package/dist/tests/e2e/gui-media-disabled.test.js +42 -0
- package/dist/tests/e2e/gui-media-disabled.test.js.map +1 -0
- package/dist/tests/e2e/gui-media.test.d.ts +6 -4
- package/dist/tests/e2e/gui-media.test.js +15 -12
- package/dist/tests/e2e/gui-media.test.js.map +1 -1
- package/dist/tests/e2e/unicode-output.test.d.ts +16 -0
- package/dist/tests/e2e/unicode-output.test.js +65 -0
- package/dist/tests/e2e/unicode-output.test.js.map +1 -0
- package/dist/tests/extensions/media.test.d.ts +1 -0
- package/dist/tests/extensions/media.test.js +58 -0
- package/dist/tests/extensions/media.test.js.map +1 -0
- package/dist/tests/extensions/registry.test.js +8 -0
- package/dist/tests/extensions/registry.test.js.map +1 -1
- package/dist/tests/lexer/identifiers.test.js +12 -0
- package/dist/tests/lexer/identifiers.test.js.map +1 -1
- package/dist/tests/lexer/scanner.test.js +8 -0
- package/dist/tests/lexer/scanner.test.js.map +1 -1
- package/dist/tests/lexer/strings.test.js +30 -0
- package/dist/tests/lexer/strings.test.js.map +1 -1
- package/doc/ai/cli.md +1 -1
- package/doc/ai/extensions.md +35 -22
- package/doc/ai/troubleshooting.md +9 -0
- package/doc/ai/zh-CN/cli.md +1 -1
- package/doc/ai/zh-CN/extensions.md +22 -9
- package/doc/ai/zh-CN/troubleshooting.md +8 -0
- package/doc/gui/build.md +37 -21
- package/doc/gui/canvas.md +47 -21
- package/doc/gui/known-issues.md +7 -6
- package/doc/gui/media.md +26 -11
- package/doc/gui.md +50 -34
- package/doc/zh-CN/gui/build.md +28 -16
- package/doc/zh-CN/gui/canvas.md +36 -17
- package/doc/zh-CN/gui/known-issues.md +6 -5
- package/doc/zh-CN/gui/media.md +21 -10
- package/doc/zh-CN/gui.md +43 -34
- package/package.json +2 -1
- package/runtime/ext_gui/canvas.cpp +114 -199
- package/runtime/ext_gui/canvas.h +101 -2
- package/runtime/ext_gui/canvas_gradient.cpp +112 -0
- package/runtime/ext_gui/canvas_internal.h +6 -6
- package/runtime/ext_gui/canvas_raster.cpp +27 -5
- package/runtime/ext_gui/canvas_stroke.cpp +252 -0
- package/runtime/ext_gui/canvas_style.cpp +240 -0
- package/runtime/ext_gui/dom_api.cpp +1 -0
- package/runtime/ext_gui/dom_api_canvas.cpp +38 -4
- package/runtime/ext_gui/dom_api_canvas_extra.cpp +232 -0
- package/runtime/ext_gui/dom_api_internal.h +31 -0
- package/runtime/ext_gui/dom_api_proto.cpp +24 -0
- package/runtime/ext_gui/gui_media.cpp +69 -2
- package/runtime/ext_gui/media.cpp +34 -488
- package/runtime/ext_gui/media.h +54 -19
- package/runtime/ext_gui/media_api.h +65 -0
- package/runtime/ext_media/media.cpp +507 -0
- package/runtime/ext_media/media_bridge.c +35 -0
- package/scripts/build-gui.ts +6 -72
- package/scripts/build-media.ts +245 -0
- package/src/cli/hints.ts +11 -0
- package/src/cli/main.ts +1 -1
- package/src/driver/compiler.ts +1 -0
- package/src/driver/version.ts +1 -1
- package/src/extensions/catalog.ts +2 -1
- package/src/extensions/media/index.ts +67 -0
- package/src/extensions/registry.ts +24 -0
- package/src/index.ts +1 -0
|
@@ -3,9 +3,13 @@
|
|
|
3
3
|
*
|
|
4
4
|
* This unit bridges three pieces:
|
|
5
5
|
* - the DOM (`Node`) and the per-window media table on `XtGuiWindow`;
|
|
6
|
-
* - the decoders
|
|
6
|
+
* - the optional decoders the `media` extension installs (the engine's
|
|
7
|
+
* `media.cpp` dispatch; FFmpeg itself lives in `runtime/ext_media/`);
|
|
7
8
|
* - the renderer (`renderer_image.cpp`), which owns the GPU textures.
|
|
8
9
|
*
|
|
10
|
+
* It also provides the SDL audio sink the decoder writes PCM to, which keeps
|
|
11
|
+
* SDL out of `media.a`.
|
|
12
|
+
*
|
|
9
13
|
* The frame loop calls `xt_gui_media_sync` (open/close decoders so the next
|
|
10
14
|
* layout sees the current sources and intrinsic sizes) and `xt_gui_media_tick`
|
|
11
15
|
* (decode for the elapsed wall clock and fire the resulting DOM events) once
|
|
@@ -15,12 +19,75 @@
|
|
|
15
19
|
|
|
16
20
|
#include "gui_internal.h"
|
|
17
21
|
|
|
22
|
+
#include <cstring>
|
|
18
23
|
#include <string>
|
|
19
24
|
#include <unordered_set>
|
|
20
25
|
#include <vector>
|
|
21
26
|
|
|
22
27
|
namespace {
|
|
23
28
|
|
|
29
|
+
/** Audio format the decoder resamples everything to. */
|
|
30
|
+
constexpr int kAudioRate = 48000;
|
|
31
|
+
constexpr int kAudioChannels = 2;
|
|
32
|
+
|
|
33
|
+
/** Initialise SDL's audio subsystem once, best effort: a headless CI box has
|
|
34
|
+
* no device, and video still plays without sound. */
|
|
35
|
+
bool audioSubsystemReady() {
|
|
36
|
+
static const bool ready = SDL_InitSubSystem(SDL_INIT_AUDIO);
|
|
37
|
+
return ready;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/** SDL push-mode audio output for one decoder.
|
|
41
|
+
*
|
|
42
|
+
* There is no audio callback: the decoder writes PCM on the main thread and SDL
|
|
43
|
+
* pulls from the queued buffer on its own thread. `queued_seconds()` lets the
|
|
44
|
+
* decoder keep that buffer shallow. */
|
|
45
|
+
class SdlAudioSink final : public xtgui::AudioSink {
|
|
46
|
+
public:
|
|
47
|
+
~SdlAudioSink() override {
|
|
48
|
+
if (stream_ != nullptr) SDL_DestroyAudioStream(stream_);
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
bool open() override {
|
|
52
|
+
if (tried_) return stream_ != nullptr;
|
|
53
|
+
tried_ = true;
|
|
54
|
+
if (!audioSubsystemReady()) return false;
|
|
55
|
+
SDL_AudioSpec spec;
|
|
56
|
+
spec.format = SDL_AUDIO_F32;
|
|
57
|
+
spec.channels = kAudioChannels;
|
|
58
|
+
spec.freq = kAudioRate;
|
|
59
|
+
stream_ = SDL_OpenAudioDeviceStream(SDL_AUDIO_DEVICE_DEFAULT_PLAYBACK, &spec, nullptr, nullptr);
|
|
60
|
+
if (stream_ == nullptr) return false;
|
|
61
|
+
SDL_PauseAudioStreamDevice(stream_);
|
|
62
|
+
return true;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
void write(const void *data, size_t bytes) override {
|
|
66
|
+
if (stream_ != nullptr) SDL_PutAudioStreamData(stream_, data, static_cast<int>(bytes));
|
|
67
|
+
}
|
|
68
|
+
void clear() override {
|
|
69
|
+
if (stream_ != nullptr) SDL_ClearAudioStream(stream_);
|
|
70
|
+
}
|
|
71
|
+
void pause() override {
|
|
72
|
+
if (stream_ != nullptr) SDL_PauseAudioStreamDevice(stream_);
|
|
73
|
+
}
|
|
74
|
+
void resume() override {
|
|
75
|
+
if (stream_ != nullptr) SDL_ResumeAudioStreamDevice(stream_);
|
|
76
|
+
}
|
|
77
|
+
void set_gain(float gain) override {
|
|
78
|
+
if (stream_ != nullptr) SDL_SetAudioStreamGain(stream_, gain);
|
|
79
|
+
}
|
|
80
|
+
double queued_seconds() const override {
|
|
81
|
+
if (stream_ == nullptr) return 0.0;
|
|
82
|
+
const int bytes = SDL_GetAudioStreamQueued(stream_);
|
|
83
|
+
return bytes / static_cast<double>(kAudioRate * kAudioChannels * sizeof(float));
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
private:
|
|
87
|
+
SDL_AudioStream *stream_ = nullptr;
|
|
88
|
+
bool tried_ = false;
|
|
89
|
+
};
|
|
90
|
+
|
|
24
91
|
bool isMediaElement(const xtgui::Node *node) {
|
|
25
92
|
return node != nullptr && node->isElement() && (node->isTag("video") || node->isTag("audio"));
|
|
26
93
|
}
|
|
@@ -65,7 +132,7 @@ xtgui::MediaPlayer *openEntry(XtGuiWindow *win, const xtgui::Node *node, const s
|
|
|
65
132
|
XtGuiMediaEntry entry;
|
|
66
133
|
entry.src = src;
|
|
67
134
|
std::string error;
|
|
68
|
-
entry.player = xtgui::MediaPlayer::open(src, &error);
|
|
135
|
+
entry.player = xtgui::MediaPlayer::open(src, &error, std::make_unique<SdlAudioSink>());
|
|
69
136
|
if (!entry.player) {
|
|
70
137
|
entry.failed = true;
|
|
71
138
|
win->media_players[node] = std::move(entry);
|
|
@@ -1,514 +1,60 @@
|
|
|
1
1
|
/*
|
|
2
|
-
* xbintsc GUI engine —
|
|
2
|
+
* xbintsc GUI engine — media decoder dispatch.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
* `
|
|
4
|
+
* The engine models `<video>`/`<audio>` elements, but the decoder is an
|
|
5
|
+
* optional backend supplied by the separate `media` extension. That backend
|
|
6
|
+
* installs its table through `xt_media_install` from a bridge object that is
|
|
7
|
+
* linked directly when the extension is enabled (see
|
|
8
|
+
* `runtime/ext_media/media_bridge.c`), so this translation unit stays tiny and
|
|
9
|
+
* FFmpeg-free.
|
|
9
10
|
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
* the queued buffer on its own thread. `queued_seconds()` keeps the buffer
|
|
15
|
-
* shallow so there is no unbounded latency.
|
|
11
|
+
* A program built with `--ext gui` alone leaves `g_api` null. `MediaPlayer::open`
|
|
12
|
+
* then fails with an actionable message and the element fires `error`, exactly
|
|
13
|
+
* as it does for an unreadable file. Nothing here touches GUI state, so it can
|
|
14
|
+
* be compiled independently of SDL and the renderer.
|
|
16
15
|
*/
|
|
17
16
|
|
|
18
17
|
#include "media.h"
|
|
19
18
|
|
|
20
|
-
#include <
|
|
21
|
-
#include <cmath>
|
|
19
|
+
#include <cstdio>
|
|
22
20
|
#include <cstring>
|
|
23
|
-
#include <unordered_map>
|
|
24
|
-
|
|
25
|
-
#include <SDL3/SDL.h>
|
|
26
|
-
|
|
27
|
-
#ifdef XT_GUI_WITH_FFMPEG
|
|
28
|
-
extern "C" {
|
|
29
|
-
#include <libavcodec/avcodec.h>
|
|
30
|
-
#include <libavformat/avformat.h>
|
|
31
|
-
#include <libavutil/channel_layout.h>
|
|
32
|
-
#include <libavutil/opt.h>
|
|
33
|
-
#include <libswresample/swresample.h>
|
|
34
|
-
#include <libswscale/swscale.h>
|
|
35
|
-
}
|
|
36
|
-
#endif
|
|
37
|
-
|
|
38
|
-
namespace xtgui {
|
|
39
21
|
|
|
40
22
|
namespace {
|
|
41
23
|
|
|
42
|
-
|
|
43
|
-
static std::unordered_map<uint64_t, MediaPlayer*> table;
|
|
44
|
-
return table;
|
|
45
|
-
}
|
|
46
|
-
|
|
47
|
-
#ifdef XT_GUI_WITH_FFMPEG
|
|
48
|
-
|
|
49
|
-
/** Audio format everything is resampled to before it reaches SDL. */
|
|
50
|
-
constexpr int kAudioRate = 48000;
|
|
51
|
-
constexpr int kAudioChannels = 2;
|
|
52
|
-
/** How much audio to keep buffered ahead of the play head. */
|
|
53
|
-
constexpr double kAudioAhead = 0.25;
|
|
54
|
-
/** Minimum gap between `timeupdate` events. */
|
|
55
|
-
constexpr double kTimeUpdateIntervalMs = 250.0;
|
|
56
|
-
|
|
57
|
-
double NowMs() { return static_cast<double>(SDL_GetTicks()); }
|
|
58
|
-
|
|
59
|
-
uint64_t NextId() {
|
|
60
|
-
static uint64_t next = 1;
|
|
61
|
-
return next++;
|
|
62
|
-
}
|
|
63
|
-
|
|
64
|
-
bool EnsureAudioSubsystem() {
|
|
65
|
-
static bool tried = false;
|
|
66
|
-
static bool ready = false;
|
|
67
|
-
if (!tried) {
|
|
68
|
-
tried = true;
|
|
69
|
-
/* Audio is best effort: a headless CI box has no device. Video still
|
|
70
|
-
* plays, and audio is silently dropped. */
|
|
71
|
-
ready = SDL_InitSubSystem(SDL_INIT_AUDIO);
|
|
72
|
-
}
|
|
73
|
-
return ready;
|
|
74
|
-
}
|
|
75
|
-
|
|
76
|
-
struct QueuedFrame {
|
|
77
|
-
Image image;
|
|
78
|
-
double pts = 0;
|
|
79
|
-
};
|
|
80
|
-
|
|
81
|
-
class FfmpegPlayer final : public MediaPlayer {
|
|
82
|
-
public:
|
|
83
|
-
~FfmpegPlayer() override {
|
|
84
|
-
PlayerTable().erase(id_);
|
|
85
|
-
if (audio_stream_) SDL_DestroyAudioStream(audio_stream_);
|
|
86
|
-
if (sws_) sws_freeContext(sws_);
|
|
87
|
-
if (swr_) swr_free(&swr_);
|
|
88
|
-
if (vframe_) av_frame_free(&vframe_);
|
|
89
|
-
if (aframe_) av_frame_free(&aframe_);
|
|
90
|
-
if (pkt_) av_packet_free(&pkt_);
|
|
91
|
-
if (vctx_) avcodec_free_context(&vctx_);
|
|
92
|
-
if (actx_) avcodec_free_context(&actx_);
|
|
93
|
-
if (fmt_) avformat_close_input(&fmt_);
|
|
94
|
-
}
|
|
95
|
-
|
|
96
|
-
bool Open(const std::string& path, std::string* error) {
|
|
97
|
-
source_ = path;
|
|
98
|
-
if (avformat_open_input(&fmt_, path.c_str(), nullptr, nullptr) < 0) {
|
|
99
|
-
if (error) *error = "cannot open media: " + path;
|
|
100
|
-
return false;
|
|
101
|
-
}
|
|
102
|
-
if (avformat_find_stream_info(fmt_, nullptr) < 0) {
|
|
103
|
-
if (error) *error = "cannot read media metadata: " + path;
|
|
104
|
-
return false;
|
|
105
|
-
}
|
|
106
|
-
duration_ = fmt_->duration > 0 ? static_cast<double>(fmt_->duration) / AV_TIME_BASE : 0.0;
|
|
24
|
+
const XtMediaApi* g_api = nullptr;
|
|
107
25
|
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
if (vstream_ < 0 && astream_ < 0) {
|
|
111
|
-
if (error) *error = "no audio or video stream: " + path;
|
|
112
|
-
return false;
|
|
113
|
-
}
|
|
114
|
-
if (vstream_ >= 0 && !OpenVideo(fmt_->streams[vstream_])) {
|
|
115
|
-
vstream_ = -1;
|
|
116
|
-
}
|
|
117
|
-
if (astream_ >= 0 && !OpenAudio(fmt_->streams[astream_])) {
|
|
118
|
-
astream_ = -1;
|
|
119
|
-
}
|
|
120
|
-
if (vstream_ < 0 && astream_ < 0) {
|
|
121
|
-
if (error) *error = "no supported stream: " + path;
|
|
122
|
-
return false;
|
|
123
|
-
}
|
|
124
|
-
pkt_ = av_packet_alloc();
|
|
125
|
-
vframe_ = av_frame_alloc();
|
|
126
|
-
aframe_ = av_frame_alloc();
|
|
26
|
+
/** Message length used for a backend's open failure. */
|
|
27
|
+
constexpr size_t kErrorSize = 256;
|
|
127
28
|
|
|
128
|
-
|
|
129
|
-
* timestamp of the video stream, which is enough for the `duration`
|
|
130
|
-
* property and for end-of-stream detection. */
|
|
131
|
-
if (duration_ <= 0 && vstream_ >= 0) {
|
|
132
|
-
AVStream* vs = fmt_->streams[vstream_];
|
|
133
|
-
if (vs->duration > 0) duration_ = vs->duration * av_q2d(vs->time_base);
|
|
134
|
-
}
|
|
135
|
-
id_ = NextId();
|
|
136
|
-
PlayerTable()[id_] = this;
|
|
137
|
-
pending_events_ |= kMediaEventLoadedMetadata;
|
|
138
|
-
return true;
|
|
139
|
-
}
|
|
140
|
-
|
|
141
|
-
uint64_t id() const override { return id_; }
|
|
142
|
-
const std::string& source() const override { return source_; }
|
|
143
|
-
bool has_video() const override { return vstream_ >= 0; }
|
|
144
|
-
bool has_audio() const override { return astream_ >= 0; }
|
|
145
|
-
double duration() const override { return duration_; }
|
|
146
|
-
|
|
147
|
-
bool paused() const override { return paused_; }
|
|
148
|
-
bool ended() const override { return ended_; }
|
|
149
|
-
double current_time() const override {
|
|
150
|
-
if (paused_) return clock_base_pts_;
|
|
151
|
-
return clock_base_pts_ + (NowMs() - clock_base_ms_) / 1000.0;
|
|
152
|
-
}
|
|
153
|
-
double volume() const override { return volume_; }
|
|
154
|
-
bool muted() const override { return muted_; }
|
|
155
|
-
bool loop() const override { return loop_; }
|
|
156
|
-
bool autoplay() const override { return autoplay_; }
|
|
157
|
-
|
|
158
|
-
void play() override {
|
|
159
|
-
if (ended_) Seek(0);
|
|
160
|
-
if (!paused_) return;
|
|
161
|
-
paused_ = false;
|
|
162
|
-
clock_base_ms_ = NowMs();
|
|
163
|
-
if (audio_stream_) SDL_ResumeAudioStreamDevice(audio_stream_);
|
|
164
|
-
pending_events_ |= kMediaEventPlay;
|
|
165
|
-
}
|
|
166
|
-
|
|
167
|
-
void pause() override {
|
|
168
|
-
if (paused_) return;
|
|
169
|
-
clock_base_pts_ = current_time();
|
|
170
|
-
paused_ = true;
|
|
171
|
-
if (audio_stream_) SDL_PauseAudioStreamDevice(audio_stream_);
|
|
172
|
-
pending_events_ |= kMediaEventPause;
|
|
173
|
-
}
|
|
174
|
-
|
|
175
|
-
void seek(double seconds) override { Seek(seconds); }
|
|
176
|
-
|
|
177
|
-
void set_volume(double value) override {
|
|
178
|
-
volume_ = std::min(1.0, std::max(0.0, value));
|
|
179
|
-
ApplyGain();
|
|
180
|
-
}
|
|
181
|
-
void set_muted(bool value) override {
|
|
182
|
-
muted_ = value;
|
|
183
|
-
ApplyGain();
|
|
184
|
-
}
|
|
185
|
-
void set_loop(bool value) override { loop_ = value; }
|
|
186
|
-
void set_autoplay(bool value) override { autoplay_ = value; }
|
|
187
|
-
|
|
188
|
-
int frame_width() const override { return vctx_ ? vctx_->width : 0; }
|
|
189
|
-
int frame_height() const override { return vctx_ ? vctx_->height : 0; }
|
|
190
|
-
const Image* frame() const override { return current_valid_ ? ¤t_ : nullptr; }
|
|
191
|
-
uint64_t frame_generation() const override { return generation_; }
|
|
192
|
-
|
|
193
|
-
unsigned tick(double now_ms) override {
|
|
194
|
-
unsigned events = pending_events_;
|
|
195
|
-
pending_events_ = 0;
|
|
196
|
-
if (vstream_ < 0 && astream_ < 0) return events;
|
|
197
|
-
if (!started_ && autoplay_) {
|
|
198
|
-
started_ = true;
|
|
199
|
-
play();
|
|
200
|
-
events |= pending_events_;
|
|
201
|
-
pending_events_ = 0;
|
|
202
|
-
}
|
|
203
|
-
if (paused_ || ended_) return events;
|
|
204
|
-
if (now_ms - last_timeupdate_ms_ >= kTimeUpdateIntervalMs) {
|
|
205
|
-
events |= kMediaEventTimeUpdate;
|
|
206
|
-
last_timeupdate_ms_ = now_ms;
|
|
207
|
-
}
|
|
208
|
-
const double t = current_time();
|
|
209
|
-
PumpVideo(t);
|
|
210
|
-
PumpAudio();
|
|
211
|
-
events |= CheckEnded(now_ms);
|
|
212
|
-
return events;
|
|
213
|
-
}
|
|
214
|
-
|
|
215
|
-
void invalidate_texture() override {}
|
|
216
|
-
|
|
217
|
-
private:
|
|
218
|
-
bool OpenVideo(AVStream* stream) {
|
|
219
|
-
const AVCodec* codec = avcodec_find_decoder(stream->codecpar->codec_id);
|
|
220
|
-
if (!codec) return false;
|
|
221
|
-
vctx_ = avcodec_alloc_context3(codec);
|
|
222
|
-
if (!vctx_) return false;
|
|
223
|
-
if (avcodec_parameters_to_context(vctx_, stream->codecpar) < 0) return false;
|
|
224
|
-
vctx_->pkt_timebase = stream->time_base;
|
|
225
|
-
if (avcodec_open2(vctx_, codec, nullptr) < 0) return false;
|
|
226
|
-
return true;
|
|
227
|
-
}
|
|
228
|
-
|
|
229
|
-
bool OpenAudio(AVStream* stream) {
|
|
230
|
-
const AVCodec* codec = avcodec_find_decoder(stream->codecpar->codec_id);
|
|
231
|
-
if (!codec) return false;
|
|
232
|
-
actx_ = avcodec_alloc_context3(codec);
|
|
233
|
-
if (!actx_) return false;
|
|
234
|
-
if (avcodec_parameters_to_context(actx_, stream->codecpar) < 0) return false;
|
|
235
|
-
actx_->pkt_timebase = stream->time_base;
|
|
236
|
-
if (avcodec_open2(actx_, codec, nullptr) < 0) return false;
|
|
237
|
-
|
|
238
|
-
AVChannelLayout out_layout;
|
|
239
|
-
av_channel_layout_default(&out_layout, kAudioChannels);
|
|
240
|
-
if (swr_alloc_set_opts2(&swr_, &out_layout, AV_SAMPLE_FMT_FLT, kAudioRate,
|
|
241
|
-
&actx_->ch_layout, actx_->sample_fmt, actx_->sample_rate,
|
|
242
|
-
0, nullptr) < 0 ||
|
|
243
|
-
!swr_ || swr_init(swr_) < 0) {
|
|
244
|
-
return false;
|
|
245
|
-
}
|
|
246
|
-
av_channel_layout_uninit(&out_layout);
|
|
247
|
-
|
|
248
|
-
if (EnsureAudioSubsystem()) {
|
|
249
|
-
SDL_AudioSpec spec;
|
|
250
|
-
spec.format = SDL_AUDIO_F32;
|
|
251
|
-
spec.channels = kAudioChannels;
|
|
252
|
-
spec.freq = kAudioRate;
|
|
253
|
-
audio_stream_ = SDL_OpenAudioDeviceStream(SDL_AUDIO_DEVICE_DEFAULT_PLAYBACK, &spec, nullptr, nullptr);
|
|
254
|
-
if (audio_stream_) {
|
|
255
|
-
SDL_PauseAudioStreamDevice(audio_stream_);
|
|
256
|
-
ApplyGain();
|
|
257
|
-
}
|
|
258
|
-
}
|
|
259
|
-
return true;
|
|
260
|
-
}
|
|
261
|
-
|
|
262
|
-
void ApplyGain() {
|
|
263
|
-
if (audio_stream_) SDL_SetAudioStreamGain(audio_stream_, muted_ ? 0.0f : static_cast<float>(volume_));
|
|
264
|
-
}
|
|
265
|
-
|
|
266
|
-
double queued_seconds() const {
|
|
267
|
-
if (!audio_stream_) return 0.0;
|
|
268
|
-
const int bytes = SDL_GetAudioStreamQueued(audio_stream_);
|
|
269
|
-
return bytes / static_cast<double>(kAudioRate * kAudioChannels * sizeof(float));
|
|
270
|
-
}
|
|
271
|
-
|
|
272
|
-
double stream_pts(const AVStream* stream, int64_t ts) const {
|
|
273
|
-
return static_cast<double>(ts) * av_q2d(stream->time_base);
|
|
274
|
-
}
|
|
275
|
-
|
|
276
|
-
void Seek(double seconds) {
|
|
277
|
-
double target = seconds;
|
|
278
|
-
if (duration_ > 0) target = std::min(duration_, std::max(0.0, target));
|
|
279
|
-
if (target < 0) target = 0;
|
|
280
|
-
clock_base_pts_ = target;
|
|
281
|
-
clock_base_ms_ = NowMs();
|
|
282
|
-
ended_ = false;
|
|
283
|
-
started_ = true;
|
|
284
|
-
|
|
285
|
-
if (fmt_) {
|
|
286
|
-
const int seek_stream = vstream_ >= 0 ? vstream_ : astream_;
|
|
287
|
-
const int64_t ts = static_cast<int64_t>(target / av_q2d(fmt_->streams[seek_stream]->time_base));
|
|
288
|
-
if (av_seek_frame(fmt_, seek_stream, ts, AVSEEK_FLAG_BACKWARD) < 0) {
|
|
289
|
-
avformat_flush(fmt_);
|
|
290
|
-
}
|
|
291
|
-
if (vctx_) avcodec_flush_buffers(vctx_);
|
|
292
|
-
if (actx_) avcodec_flush_buffers(actx_);
|
|
293
|
-
}
|
|
294
|
-
if (audio_stream_) SDL_ClearAudioStream(audio_stream_);
|
|
295
|
-
video_queue_.clear();
|
|
296
|
-
current_valid_ = false;
|
|
297
|
-
drop_before_ = vstream_ >= 0 ? target : -1;
|
|
298
|
-
reader_eof_ = false;
|
|
299
|
-
video_flushed_ = false;
|
|
300
|
-
audio_flushed_ = false;
|
|
301
|
-
video_drained_ = false;
|
|
302
|
-
audio_drained_ = false;
|
|
303
|
-
}
|
|
304
|
-
|
|
305
|
-
bool ReadPacket() {
|
|
306
|
-
const int ret = av_read_frame(fmt_, pkt_);
|
|
307
|
-
if (ret < 0) {
|
|
308
|
-
reader_eof_ = true;
|
|
309
|
-
return false;
|
|
310
|
-
}
|
|
311
|
-
if (pkt_->stream_index == vstream_) {
|
|
312
|
-
avcodec_send_packet(vctx_, pkt_);
|
|
313
|
-
DrainVideo();
|
|
314
|
-
} else if (pkt_->stream_index == astream_) {
|
|
315
|
-
avcodec_send_packet(actx_, pkt_);
|
|
316
|
-
DrainAudio();
|
|
317
|
-
}
|
|
318
|
-
av_packet_unref(pkt_);
|
|
319
|
-
return true;
|
|
320
|
-
}
|
|
321
|
-
|
|
322
|
-
void DrainVideo() {
|
|
323
|
-
while (avcodec_receive_frame(vctx_, vframe_) == 0) {
|
|
324
|
-
const double pts = stream_pts(fmt_->streams[vstream_], vframe_->best_effort_timestamp);
|
|
325
|
-
Image image;
|
|
326
|
-
ConvertVideo(vframe_, &image);
|
|
327
|
-
video_queue_.push_back({std::move(image), pts});
|
|
328
|
-
}
|
|
329
|
-
}
|
|
330
|
-
|
|
331
|
-
void ConvertVideo(const AVFrame* frame, Image* out) {
|
|
332
|
-
const int w = frame->width;
|
|
333
|
-
const int h = frame->height;
|
|
334
|
-
if (!sws_ || sws_width_ != w || sws_height_ != h || sws_pix_fmt_ != frame->format) {
|
|
335
|
-
if (sws_) sws_freeContext(sws_);
|
|
336
|
-
sws_ = sws_getContext(w, h, static_cast<AVPixelFormat>(frame->format), w, h, AV_PIX_FMT_RGBA,
|
|
337
|
-
SWS_BILINEAR, nullptr, nullptr, nullptr);
|
|
338
|
-
sws_width_ = w;
|
|
339
|
-
sws_height_ = h;
|
|
340
|
-
sws_pix_fmt_ = frame->format;
|
|
341
|
-
}
|
|
342
|
-
if (!sws_) return;
|
|
343
|
-
out->width = w;
|
|
344
|
-
out->height = h;
|
|
345
|
-
out->pixels.resize(static_cast<size_t>(w) * h * 4);
|
|
346
|
-
uint8_t* dst[4] = {out->pixels.data(), nullptr, nullptr, nullptr};
|
|
347
|
-
int dst_linesize[4] = {w * 4, 0, 0, 0};
|
|
348
|
-
sws_scale(sws_, frame->data, frame->linesize, 0, h, dst, dst_linesize);
|
|
349
|
-
}
|
|
350
|
-
|
|
351
|
-
void DrainAudio() {
|
|
352
|
-
while (avcodec_receive_frame(actx_, aframe_) == 0) {
|
|
353
|
-
if (!audio_stream_) continue;
|
|
354
|
-
const int capacity = swr_get_out_samples(swr_, aframe_->nb_samples);
|
|
355
|
-
if (capacity <= 0) continue;
|
|
356
|
-
audio_buf_.resize(static_cast<size_t>(capacity) * kAudioChannels * sizeof(float));
|
|
357
|
-
uint8_t* out = audio_buf_.data();
|
|
358
|
-
const int got = swr_convert(swr_, &out, capacity,
|
|
359
|
-
reinterpret_cast<const uint8_t* const*>(aframe_->extended_data),
|
|
360
|
-
aframe_->nb_samples);
|
|
361
|
-
if (got > 0) {
|
|
362
|
-
const int bytes = got * kAudioChannels * static_cast<int>(sizeof(float));
|
|
363
|
-
SDL_PutAudioStreamData(audio_stream_, audio_buf_.data(), bytes);
|
|
364
|
-
}
|
|
365
|
-
}
|
|
366
|
-
}
|
|
29
|
+
} // namespace
|
|
367
30
|
|
|
368
|
-
|
|
369
|
-
if (vstream_ < 0 || video_drained_) return;
|
|
370
|
-
while (true) {
|
|
371
|
-
if (video_queue_.empty()) {
|
|
372
|
-
if (reader_eof_) {
|
|
373
|
-
if (!video_flushed_) {
|
|
374
|
-
avcodec_send_packet(vctx_, nullptr);
|
|
375
|
-
video_flushed_ = true;
|
|
376
|
-
DrainVideo();
|
|
377
|
-
continue;
|
|
378
|
-
}
|
|
379
|
-
video_drained_ = true;
|
|
380
|
-
return;
|
|
381
|
-
}
|
|
382
|
-
if (!ReadPacket() && video_queue_.empty() && reader_eof_) continue;
|
|
383
|
-
if (video_queue_.empty()) continue;
|
|
384
|
-
}
|
|
385
|
-
QueuedFrame& front = video_queue_.front();
|
|
386
|
-
if (drop_before_ >= 0 && front.pts < drop_before_ - 0.001) {
|
|
387
|
-
video_queue_.erase(video_queue_.begin());
|
|
388
|
-
continue;
|
|
389
|
-
}
|
|
390
|
-
if (front.pts <= t + 0.01) {
|
|
391
|
-
current_ = std::move(front.image);
|
|
392
|
-
current_valid_ = true;
|
|
393
|
-
++generation_;
|
|
394
|
-
video_queue_.erase(video_queue_.begin());
|
|
395
|
-
drop_before_ = -1;
|
|
396
|
-
} else {
|
|
397
|
-
return;
|
|
398
|
-
}
|
|
399
|
-
}
|
|
400
|
-
}
|
|
31
|
+
namespace xtgui {
|
|
401
32
|
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
avcodec_send_packet(actx_, nullptr);
|
|
408
|
-
audio_flushed_ = true;
|
|
409
|
-
DrainAudio();
|
|
410
|
-
continue;
|
|
411
|
-
}
|
|
412
|
-
audio_drained_ = true;
|
|
413
|
-
return;
|
|
414
|
-
}
|
|
415
|
-
if (!ReadPacket()) return;
|
|
416
|
-
}
|
|
33
|
+
std::unique_ptr<MediaPlayer> MediaPlayer::open(const std::string& path, std::string* error,
|
|
34
|
+
std::unique_ptr<AudioSink> audio) {
|
|
35
|
+
if (g_api == nullptr || g_api->open == nullptr) {
|
|
36
|
+
if (error) *error = "media support requires the 'media' extension (pass --ext media)";
|
|
37
|
+
return nullptr;
|
|
417
38
|
}
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
if (astream_ >= 0 && !audio_drained_) return 0;
|
|
424
|
-
if (astream_ >= 0 && queued_seconds() > 0.05) return 0;
|
|
425
|
-
if (vstream_ >= 0 && duration_ > 0 && current_time() < duration_ - 0.05) return 0;
|
|
426
|
-
if (loop_) {
|
|
427
|
-
Seek(0);
|
|
428
|
-
play();
|
|
429
|
-
return kMediaEventPlay;
|
|
430
|
-
}
|
|
431
|
-
ended_ = true;
|
|
432
|
-
paused_ = true;
|
|
433
|
-
clock_base_pts_ = duration_ > 0 ? duration_ : current_time();
|
|
434
|
-
last_timeupdate_ms_ = now_ms;
|
|
435
|
-
return kMediaEventEnded;
|
|
39
|
+
char message[kErrorSize] = {0};
|
|
40
|
+
auto* player = static_cast<MediaPlayer*>(
|
|
41
|
+
g_api->open(path.c_str(), message, sizeof(message), audio.release()));
|
|
42
|
+
if (player == nullptr && error != nullptr) {
|
|
43
|
+
*error = message[0] != '\0' ? message : "cannot open media: " + path;
|
|
436
44
|
}
|
|
437
|
-
|
|
438
|
-
uint64_t id_ = 0;
|
|
439
|
-
std::string source_;
|
|
440
|
-
AVFormatContext* fmt_ = nullptr;
|
|
441
|
-
AVCodecContext* vctx_ = nullptr;
|
|
442
|
-
AVCodecContext* actx_ = nullptr;
|
|
443
|
-
int vstream_ = -1;
|
|
444
|
-
int astream_ = -1;
|
|
445
|
-
AVPacket* pkt_ = nullptr;
|
|
446
|
-
AVFrame* vframe_ = nullptr;
|
|
447
|
-
AVFrame* aframe_ = nullptr;
|
|
448
|
-
SwsContext* sws_ = nullptr;
|
|
449
|
-
SwrContext* swr_ = nullptr;
|
|
450
|
-
int sws_width_ = 0;
|
|
451
|
-
int sws_height_ = 0;
|
|
452
|
-
int sws_pix_fmt_ = -1;
|
|
453
|
-
|
|
454
|
-
std::vector<QueuedFrame> video_queue_;
|
|
455
|
-
Image current_;
|
|
456
|
-
bool current_valid_ = false;
|
|
457
|
-
uint64_t generation_ = 0;
|
|
458
|
-
double drop_before_ = -1;
|
|
459
|
-
|
|
460
|
-
SDL_AudioStream* audio_stream_ = nullptr;
|
|
461
|
-
std::vector<uint8_t> audio_buf_;
|
|
462
|
-
|
|
463
|
-
bool reader_eof_ = false;
|
|
464
|
-
bool video_flushed_ = false;
|
|
465
|
-
bool audio_flushed_ = false;
|
|
466
|
-
bool video_drained_ = false;
|
|
467
|
-
bool audio_drained_ = false;
|
|
468
|
-
|
|
469
|
-
double duration_ = 0;
|
|
470
|
-
double volume_ = 1.0;
|
|
471
|
-
bool muted_ = false;
|
|
472
|
-
bool loop_ = false;
|
|
473
|
-
bool autoplay_ = false;
|
|
474
|
-
bool paused_ = true;
|
|
475
|
-
bool ended_ = false;
|
|
476
|
-
bool started_ = false;
|
|
477
|
-
double clock_base_pts_ = 0;
|
|
478
|
-
double clock_base_ms_ = 0;
|
|
479
|
-
double last_timeupdate_ms_ = -1e9;
|
|
480
|
-
unsigned pending_events_ = 0;
|
|
481
|
-
};
|
|
482
|
-
|
|
483
|
-
#endif // XT_GUI_WITH_FFMPEG
|
|
484
|
-
|
|
485
|
-
} // namespace
|
|
486
|
-
|
|
487
|
-
std::unique_ptr<MediaPlayer> MediaPlayer::open(const std::string& path, std::string* error) {
|
|
488
|
-
#ifdef XT_GUI_WITH_FFMPEG
|
|
489
|
-
std::unique_ptr<FfmpegPlayer> player(new FfmpegPlayer());
|
|
490
|
-
if (!player->Open(path, error)) return nullptr;
|
|
491
|
-
return player;
|
|
492
|
-
#else
|
|
493
|
-
(void)path;
|
|
494
|
-
if (error) *error = "media support was not built into this gui.a";
|
|
495
|
-
return nullptr;
|
|
496
|
-
#endif
|
|
45
|
+
return std::unique_ptr<MediaPlayer>(player);
|
|
497
46
|
}
|
|
498
47
|
|
|
499
48
|
MediaPlayer* xt_media_by_id(uint64_t id) {
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
49
|
+
return g_api != nullptr && g_api->by_id != nullptr
|
|
50
|
+
? static_cast<MediaPlayer*>(g_api->by_id(id))
|
|
51
|
+
: nullptr;
|
|
503
52
|
}
|
|
504
53
|
|
|
505
54
|
void xt_media_shutdown() {
|
|
506
|
-
|
|
507
|
-
* entry, and a destructor mutating the container it is being cleared from
|
|
508
|
-
* would invalidate the clearing loop. With an empty table the erases are
|
|
509
|
-
* no-ops. */
|
|
510
|
-
auto pending = std::move(PlayerTable());
|
|
511
|
-
PlayerTable().clear();
|
|
55
|
+
if (g_api != nullptr && g_api->shutdown != nullptr) g_api->shutdown();
|
|
512
56
|
}
|
|
513
57
|
|
|
514
58
|
} // namespace xtgui
|
|
59
|
+
|
|
60
|
+
extern "C" void xt_media_install(const XtMediaApi* api) { g_api = api; }
|