xbintsc 0.3.69 → 0.3.79

package/runtime/ext_gui/media.cpp

@@ -0,0 +1,514 @@
+/*
+ * xbintsc GUI engine — `<video>`/`<audio>` decoding.
+ *
+ * This translation unit is the only place that knows about FFmpeg. It is built
+ * with `-DXT_GUI_WITH_FFMPEG=1` when `scripts/build-gui.ts` managed to build a
+ * minimal static FFmpeg; without it `MediaPlayer::open` always fails and the
+ * rest of the engine still compiles and runs (the element simply reports an
+ * `error` event).
+ *
+ * Threading: the engine is single-threaded and its values live on the main
+ * thread, so nothing here may run on an SDL audio callback. Instead the SDL
+ * audio device is opened with a *null* callback (push mode): `tick()` decodes
+ * PCM on the main thread and `SDL_PutAudioStreamData`s it, and SDL pulls from
+ * the queued buffer on its own thread. `queued_seconds()` keeps the buffer
+ * shallow so there is no unbounded latency.
+ */
+
+#include "media.h"
+
+#include <algorithm>
+#include <cmath>
+#include <cstring>
+#include <unordered_map>
+
+#include <SDL3/SDL.h>
+
+#ifdef XT_GUI_WITH_FFMPEG
+extern "C" {
+#include <libavcodec/avcodec.h>
+#include <libavformat/avformat.h>
+#include <libavutil/channel_layout.h>
+#include <libavutil/opt.h>
+#include <libswresample/swresample.h>
+#include <libswscale/swscale.h>
+}
+#endif
+
+namespace xtgui {
+
+namespace {
+
+std::unordered_map<uint64_t, MediaPlayer*>& PlayerTable() {
+  static std::unordered_map<uint64_t, MediaPlayer*> table;
+  return table;
+}
+
+#ifdef XT_GUI_WITH_FFMPEG
+
+/** Audio format everything is resampled to before it reaches SDL. */
+constexpr int kAudioRate = 48000;
+constexpr int kAudioChannels = 2;
+/** How much audio to keep buffered ahead of the play head. */
+constexpr double kAudioAhead = 0.25;
+/** Minimum gap between `timeupdate` events. */
+constexpr double kTimeUpdateIntervalMs = 250.0;
+
+double NowMs() { return static_cast<double>(SDL_GetTicks()); }
+
+uint64_t NextId() {
+  static uint64_t next = 1;
+  return next++;
+}
+
+bool EnsureAudioSubsystem() {
+  static bool tried = false;
+  static bool ready = false;
+  if (!tried) {
+    tried = true;
+    /* Audio is best effort: a headless CI box has no device. Video still
+     * plays, and audio is silently dropped. */
+    ready = SDL_InitSubSystem(SDL_INIT_AUDIO);
+  }
+  return ready;
+}
+
+struct QueuedFrame {
+  Image image;
+  double pts = 0;
+};
+
+class FfmpegPlayer final : public MediaPlayer {
+ public:
+  ~FfmpegPlayer() override {
+    PlayerTable().erase(id_);
+    if (audio_stream_) SDL_DestroyAudioStream(audio_stream_);
+    if (sws_) sws_freeContext(sws_);
+    if (swr_) swr_free(&swr_);
+    if (vframe_) av_frame_free(&vframe_);
+    if (aframe_) av_frame_free(&aframe_);
+    if (pkt_) av_packet_free(&pkt_);
+    if (vctx_) avcodec_free_context(&vctx_);
+    if (actx_) avcodec_free_context(&actx_);
+    if (fmt_) avformat_close_input(&fmt_);
+  }
+
+  bool Open(const std::string& path, std::string* error) {
+    source_ = path;
+    if (avformat_open_input(&fmt_, path.c_str(), nullptr, nullptr) < 0) {
+      if (error) *error = "cannot open media: " + path;
+      return false;
+    }
+    if (avformat_find_stream_info(fmt_, nullptr) < 0) {
+      if (error) *error = "cannot read media metadata: " + path;
+      return false;
+    }
+    duration_ = fmt_->duration > 0 ? static_cast<double>(fmt_->duration) / AV_TIME_BASE : 0.0;
+
+    vstream_ = av_find_best_stream(fmt_, AVMEDIA_TYPE_VIDEO, -1, -1, nullptr, 0);
+    astream_ = av_find_best_stream(fmt_, AVMEDIA_TYPE_AUDIO, -1, -1, nullptr, 0);
+    if (vstream_ < 0 && astream_ < 0) {
+      if (error) *error = "no audio or video stream: " + path;
+      return false;
+    }
+    if (vstream_ >= 0 && !OpenVideo(fmt_->streams[vstream_])) {
+      vstream_ = -1;
+    }
+    if (astream_ >= 0 && !OpenAudio(fmt_->streams[astream_])) {
+      astream_ = -1;
+    }
+    if (vstream_ < 0 && astream_ < 0) {
+      if (error) *error = "no supported stream: " + path;
+      return false;
+    }
+    pkt_ = av_packet_alloc();
+    vframe_ = av_frame_alloc();
+    aframe_ = av_frame_alloc();
+
+    /* A file whose default duration is unknown falls back to the last
+     * timestamp of the video stream, which is enough for the `duration`
+     * property and for end-of-stream detection. */
+    if (duration_ <= 0 && vstream_ >= 0) {
+      AVStream* vs = fmt_->streams[vstream_];
+      if (vs->duration > 0) duration_ = vs->duration * av_q2d(vs->time_base);
+    }
+    id_ = NextId();
+    PlayerTable()[id_] = this;
+    pending_events_ |= kMediaEventLoadedMetadata;
+    return true;
+  }
+
+  uint64_t id() const override { return id_; }
+  const std::string& source() const override { return source_; }
+  bool has_video() const override { return vstream_ >= 0; }
+  bool has_audio() const override { return astream_ >= 0; }
+  double duration() const override { return duration_; }
+
+  bool paused() const override { return paused_; }
+  bool ended() const override { return ended_; }
+  double current_time() const override {
+    if (paused_) return clock_base_pts_;
+    return clock_base_pts_ + (NowMs() - clock_base_ms_) / 1000.0;
+  }
+  double volume() const override { return volume_; }
+  bool muted() const override { return muted_; }
+  bool loop() const override { return loop_; }
+  bool autoplay() const override { return autoplay_; }
+
+  void play() override {
+    if (ended_) Seek(0);
+    if (!paused_) return;
+    paused_ = false;
+    clock_base_ms_ = NowMs();
+    if (audio_stream_) SDL_ResumeAudioStreamDevice(audio_stream_);
+    pending_events_ |= kMediaEventPlay;
+  }
+
+  void pause() override {
+    if (paused_) return;
+    clock_base_pts_ = current_time();
+    paused_ = true;
+    if (audio_stream_) SDL_PauseAudioStreamDevice(audio_stream_);
+    pending_events_ |= kMediaEventPause;
+  }
+
+  void seek(double seconds) override { Seek(seconds); }
+
+  void set_volume(double value) override {
+    volume_ = std::min(1.0, std::max(0.0, value));
+    ApplyGain();
+  }
+  void set_muted(bool value) override {
+    muted_ = value;
+    ApplyGain();
+  }
+  void set_loop(bool value) override { loop_ = value; }
+  void set_autoplay(bool value) override { autoplay_ = value; }
+
+  int frame_width() const override { return vctx_ ? vctx_->width : 0; }
+  int frame_height() const override { return vctx_ ? vctx_->height : 0; }
+  const Image* frame() const override { return current_valid_ ? &current_ : nullptr; }
+  uint64_t frame_generation() const override { return generation_; }
+
+  unsigned tick(double now_ms) override {
+    unsigned events = pending_events_;
+    pending_events_ = 0;
+    if (vstream_ < 0 && astream_ < 0) return events;
+    if (!started_ && autoplay_) {
+      started_ = true;
+      play();
+      events |= pending_events_;
+      pending_events_ = 0;
+    }
+    if (paused_ || ended_) return events;
+    if (now_ms - last_timeupdate_ms_ >= kTimeUpdateIntervalMs) {
+      events |= kMediaEventTimeUpdate;
+      last_timeupdate_ms_ = now_ms;
+    }
+    const double t = current_time();
+    PumpVideo(t);
+    PumpAudio();
+    events |= CheckEnded(now_ms);
+    return events;
+  }
+
+  void invalidate_texture() override {}
+
+ private:
+  bool OpenVideo(AVStream* stream) {
+    const AVCodec* codec = avcodec_find_decoder(stream->codecpar->codec_id);
+    if (!codec) return false;
+    vctx_ = avcodec_alloc_context3(codec);
+    if (!vctx_) return false;
+    if (avcodec_parameters_to_context(vctx_, stream->codecpar) < 0) return false;
+    vctx_->pkt_timebase = stream->time_base;
+    if (avcodec_open2(vctx_, codec, nullptr) < 0) return false;
+    return true;
+  }
+
+  bool OpenAudio(AVStream* stream) {
+    const AVCodec* codec = avcodec_find_decoder(stream->codecpar->codec_id);
+    if (!codec) return false;
+    actx_ = avcodec_alloc_context3(codec);
+    if (!actx_) return false;
+    if (avcodec_parameters_to_context(actx_, stream->codecpar) < 0) return false;
+    actx_->pkt_timebase = stream->time_base;
+    if (avcodec_open2(actx_, codec, nullptr) < 0) return false;
+
+    AVChannelLayout out_layout;
+    av_channel_layout_default(&out_layout, kAudioChannels);
+    if (swr_alloc_set_opts2(&swr_, &out_layout, AV_SAMPLE_FMT_FLT, kAudioRate,
+                            &actx_->ch_layout, actx_->sample_fmt, actx_->sample_rate,
+                            0, nullptr) < 0 ||
+        !swr_ || swr_init(swr_) < 0) {
+      return false;
+    }
+    av_channel_layout_uninit(&out_layout);
+
+    if (EnsureAudioSubsystem()) {
+      SDL_AudioSpec spec;
+      spec.format = SDL_AUDIO_F32;
+      spec.channels = kAudioChannels;
+      spec.freq = kAudioRate;
+      audio_stream_ = SDL_OpenAudioDeviceStream(SDL_AUDIO_DEVICE_DEFAULT_PLAYBACK, &spec, nullptr, nullptr);
+      if (audio_stream_) {
+        SDL_PauseAudioStreamDevice(audio_stream_);
+        ApplyGain();
+      }
+    }
+    return true;
+  }
+
+  void ApplyGain() {
+    if (audio_stream_) SDL_SetAudioStreamGain(audio_stream_, muted_ ? 0.0f : static_cast<float>(volume_));
+  }
+
+  double queued_seconds() const {
+    if (!audio_stream_) return 0.0;
+    const int bytes = SDL_GetAudioStreamQueued(audio_stream_);
+    return bytes / static_cast<double>(kAudioRate * kAudioChannels * sizeof(float));
+  }
+
+  double stream_pts(const AVStream* stream, int64_t ts) const {
+    return static_cast<double>(ts) * av_q2d(stream->time_base);
+  }
+
+  void Seek(double seconds) {
+    double target = seconds;
+    if (duration_ > 0) target = std::min(duration_, std::max(0.0, target));
+    if (target < 0) target = 0;
+    clock_base_pts_ = target;
+    clock_base_ms_ = NowMs();
+    ended_ = false;
+    started_ = true;
+
+    if (fmt_) {
+      const int seek_stream = vstream_ >= 0 ? vstream_ : astream_;
+      const int64_t ts = static_cast<int64_t>(target / av_q2d(fmt_->streams[seek_stream]->time_base));
+      if (av_seek_frame(fmt_, seek_stream, ts, AVSEEK_FLAG_BACKWARD) < 0) {
+        avformat_flush(fmt_);
+      }
+      if (vctx_) avcodec_flush_buffers(vctx_);
+      if (actx_) avcodec_flush_buffers(actx_);
+    }
+    if (audio_stream_) SDL_ClearAudioStream(audio_stream_);
+    video_queue_.clear();
+    current_valid_ = false;
+    drop_before_ = vstream_ >= 0 ? target : -1;
+    reader_eof_ = false;
+    video_flushed_ = false;
+    audio_flushed_ = false;
+    video_drained_ = false;
+    audio_drained_ = false;
+  }
+
+  bool ReadPacket() {
+    const int ret = av_read_frame(fmt_, pkt_);
+    if (ret < 0) {
+      reader_eof_ = true;
+      return false;
+    }
+    if (pkt_->stream_index == vstream_) {
+      avcodec_send_packet(vctx_, pkt_);
+      DrainVideo();
+    } else if (pkt_->stream_index == astream_) {
+      avcodec_send_packet(actx_, pkt_);
+      DrainAudio();
+    }
+    av_packet_unref(pkt_);
+    return true;
+  }
+
+  void DrainVideo() {
+    while (avcodec_receive_frame(vctx_, vframe_) == 0) {
+      const double pts = stream_pts(fmt_->streams[vstream_], vframe_->best_effort_timestamp);
+      Image image;
+      ConvertVideo(vframe_, &image);
+      video_queue_.push_back({std::move(image), pts});
+    }
+  }
+
+  void ConvertVideo(const AVFrame* frame, Image* out) {
+    const int w = frame->width;
+    const int h = frame->height;
+    if (!sws_ || sws_width_ != w || sws_height_ != h || sws_pix_fmt_ != frame->format) {
+      if (sws_) sws_freeContext(sws_);
+      sws_ = sws_getContext(w, h, static_cast<AVPixelFormat>(frame->format), w, h, AV_PIX_FMT_RGBA,
+                            SWS_BILINEAR, nullptr, nullptr, nullptr);
+      sws_width_ = w;
+      sws_height_ = h;
+      sws_pix_fmt_ = frame->format;
+    }
+    if (!sws_) return;
+    out->width = w;
+    out->height = h;
+    out->pixels.resize(static_cast<size_t>(w) * h * 4);
+    uint8_t* dst[4] = {out->pixels.data(), nullptr, nullptr, nullptr};
+    int dst_linesize[4] = {w * 4, 0, 0, 0};
+    sws_scale(sws_, frame->data, frame->linesize, 0, h, dst, dst_linesize);
+  }
+
+  void DrainAudio() {
+    while (avcodec_receive_frame(actx_, aframe_) == 0) {
+      if (!audio_stream_) continue;
+      const int capacity = swr_get_out_samples(swr_, aframe_->nb_samples);
+      if (capacity <= 0) continue;
+      audio_buf_.resize(static_cast<size_t>(capacity) * kAudioChannels * sizeof(float));
+      uint8_t* out = audio_buf_.data();
+      const int got = swr_convert(swr_, &out, capacity,
+                                  reinterpret_cast<const uint8_t* const*>(aframe_->extended_data),
+                                  aframe_->nb_samples);
+      if (got > 0) {
+        const int bytes = got * kAudioChannels * static_cast<int>(sizeof(float));
+        SDL_PutAudioStreamData(audio_stream_, audio_buf_.data(), bytes);
+      }
+    }
+  }
+
+  void PumpVideo(double t) {
+    if (vstream_ < 0 || video_drained_) return;
+    while (true) {
+      if (video_queue_.empty()) {
+        if (reader_eof_) {
+          if (!video_flushed_) {
+            avcodec_send_packet(vctx_, nullptr);
+            video_flushed_ = true;
+            DrainVideo();
+            continue;
+          }
+          video_drained_ = true;
+          return;
+        }
+        if (!ReadPacket() && video_queue_.empty() && reader_eof_) continue;
+        if (video_queue_.empty()) continue;
+      }
+      QueuedFrame& front = video_queue_.front();
+      if (drop_before_ >= 0 && front.pts < drop_before_ - 0.001) {
+        video_queue_.erase(video_queue_.begin());
+        continue;
+      }
+      if (front.pts <= t + 0.01) {
+        current_ = std::move(front.image);
+        current_valid_ = true;
+        ++generation_;
+        video_queue_.erase(video_queue_.begin());
+        drop_before_ = -1;
+      } else {
+        return;
+      }
+    }
+  }
+
+  void PumpAudio() {
+    if (astream_ < 0 || audio_drained_) return;
+    while (queued_seconds() < kAudioAhead) {
+      if (reader_eof_) {
+        if (!audio_flushed_) {
+          avcodec_send_packet(actx_, nullptr);
+          audio_flushed_ = true;
+          DrainAudio();
+          continue;
+        }
+        audio_drained_ = true;
+        return;
+      }
+      if (!ReadPacket()) return;
+    }
+  }
+
+  unsigned CheckEnded(double now_ms) {
+    if (ended_) return 0;
+    if (!reader_eof_) return 0;
+    if (vstream_ >= 0 && !video_drained_) return 0;
+    if (astream_ >= 0 && !audio_drained_) return 0;
+    if (astream_ >= 0 && queued_seconds() > 0.05) return 0;
+    if (vstream_ >= 0 && duration_ > 0 && current_time() < duration_ - 0.05) return 0;
+    if (loop_) {
+      Seek(0);
+      play();
+      return kMediaEventPlay;
+    }
+    ended_ = true;
+    paused_ = true;
+    clock_base_pts_ = duration_ > 0 ? duration_ : current_time();
+    last_timeupdate_ms_ = now_ms;
+    return kMediaEventEnded;
+  }
+
+  uint64_t id_ = 0;
+  std::string source_;
+  AVFormatContext* fmt_ = nullptr;
+  AVCodecContext* vctx_ = nullptr;
+  AVCodecContext* actx_ = nullptr;
+  int vstream_ = -1;
+  int astream_ = -1;
+  AVPacket* pkt_ = nullptr;
+  AVFrame* vframe_ = nullptr;
+  AVFrame* aframe_ = nullptr;
+  SwsContext* sws_ = nullptr;
+  SwrContext* swr_ = nullptr;
+  int sws_width_ = 0;
+  int sws_height_ = 0;
+  int sws_pix_fmt_ = -1;
+
+  std::vector<QueuedFrame> video_queue_;
+  Image current_;
+  bool current_valid_ = false;
+  uint64_t generation_ = 0;
+  double drop_before_ = -1;
+
+  SDL_AudioStream* audio_stream_ = nullptr;
+  std::vector<uint8_t> audio_buf_;
+
+  bool reader_eof_ = false;
+  bool video_flushed_ = false;
+  bool audio_flushed_ = false;
+  bool video_drained_ = false;
+  bool audio_drained_ = false;
+
+  double duration_ = 0;
+  double volume_ = 1.0;
+  bool muted_ = false;
+  bool loop_ = false;
+  bool autoplay_ = false;
+  bool paused_ = true;
+  bool ended_ = false;
+  bool started_ = false;
+  double clock_base_pts_ = 0;
+  double clock_base_ms_ = 0;
+  double last_timeupdate_ms_ = -1e9;
+  unsigned pending_events_ = 0;
+};
+
+#endif  // XT_GUI_WITH_FFMPEG
+
+}  // namespace
+
+std::unique_ptr<MediaPlayer> MediaPlayer::open(const std::string& path, std::string* error) {
+#ifdef XT_GUI_WITH_FFMPEG
+  std::unique_ptr<FfmpegPlayer> player(new FfmpegPlayer());
+  if (!player->Open(path, error)) return nullptr;
+  return player;
+#else
+  (void)path;
+  if (error) *error = "media support was not built into this gui.a";
+  return nullptr;
+#endif
+}
+
+MediaPlayer* xt_media_by_id(uint64_t id) {
+  auto& table = PlayerTable();
+  const auto it = table.find(id);
+  return it == table.end() ? nullptr : it->second;
+}
+
+void xt_media_shutdown() {
+  /* Move the table out before destroying it: `~FfmpegPlayer` erases its own
+   * entry, and a destructor mutating the container it is being cleared from
+   * would invalidate the clearing loop. With an empty table the erases are
+   * no-ops. */
+  auto pending = std::move(PlayerTable());
+  PlayerTable().clear();
+}
+
+}  // namespace xtgui

package/runtime/ext_gui/media.h

@@ -0,0 +1,104 @@
+#ifndef XT_GUI_MEDIA_H
+#define XT_GUI_MEDIA_H
+
+/*
+ * Media decoding for `<video>` and `<audio>`.
+ *
+ * A `MediaPlayer` owns one FFmpeg demux/decode pipeline for a single source
+ * file. It turns the file into two streams of interest:
+ *
+ *   - the latest decoded video frame (RGBA8, `Image`), which the renderer
+ *     uploads as a GPU texture and stretches over the element's box;
+ *   - 48 kHz stereo float PCM, which is pushed into an SDL audio stream.
+ *
+ * Decoding happens on the main thread (see the threading note in `media.cpp`):
+ * `tick()` advances the wall clock, decodes whatever is due and returns a
+ * bitmask of DOM events the caller should dispatch. The engine already runs a
+ * frame loop, so there is no separate media thread and no interaction with the
+ * garbage collector.
+ *
+ * The FFmpeg headers stay inside `media.cpp`; this header only exposes value
+ * types so the rest of the engine (and `gui_engine.h`) can hold a player
+ * without pulling in libav*.
+ */
+
+#include <cstdint>
+#include <memory>
+#include <string>
+
+#include "image.h"
+
+namespace xtgui {
+
+/** DOM events a media element may fire after `tick()`. */
+enum MediaEvent {
+  kMediaEventLoadedMetadata = 1 << 0,
+  kMediaEventPlay = 1 << 1,
+  kMediaEventPause = 1 << 2,
+  kMediaEventEnded = 1 << 3,
+  kMediaEventTimeUpdate = 1 << 4,
+  kMediaEventError = 1 << 5,
+};
+
+/**
+ * One decoder for one `<video>`/`<audio>` source.
+ *
+ * Instances are created with `open()`, which returns `nullptr` and fills
+ * `*error` when the file cannot be read or when the build has no decoder
+ * (`gui.a` built without FFmpeg). All times are seconds.
+ */
+class MediaPlayer {
+ public:
+  virtual ~MediaPlayer() {}
+
+  /** Open `path`; returns `nullptr` (with `*error` set) on failure. */
+  static std::unique_ptr<MediaPlayer> open(const std::string& path, std::string* error);
+
+  /** Process-wide identity used to key GPU textures and the lookup below. */
+  virtual uint64_t id() const = 0;
+  virtual const std::string& source() const = 0;
+  virtual bool has_video() const = 0;
+  virtual bool has_audio() const = 0;
+  virtual double duration() const = 0;
+
+  /* Playback state ------------------------------------------------------- */
+  virtual bool paused() const = 0;
+  virtual bool ended() const = 0;
+  virtual double current_time() const = 0;
+  virtual double volume() const = 0;
+  virtual bool muted() const = 0;
+  virtual bool loop() const = 0;
+  virtual bool autoplay() const = 0;
+
+  virtual void play() = 0;
+  virtual void pause() = 0;
+  virtual void seek(double seconds) = 0;
+  virtual void set_volume(double value) = 0;
+  virtual void set_muted(bool value) = 0;
+  virtual void set_loop(bool value) = 0;
+  virtual void set_autoplay(bool value) = 0;
+
+  /* Video ---------------------------------------------------------------- */
+  virtual int frame_width() const = 0;
+  virtual int frame_height() const = 0;
+  /** Latest decoded RGBA frame, or `nullptr` before the first one is ready. */
+  virtual const Image* frame() const = 0;
+  /** Bumped every time `frame()` changes, so the renderer can re-upload. */
+  virtual uint64_t frame_generation() const = 0;
+
+  /** Advance the clock, decode/buffer, and return the events that happened. */
+  virtual unsigned tick(double now_ms) = 0;
+
+  /** Drop the cached GPU texture handle (called when it is released). */
+  virtual void invalidate_texture() = 0;
+};
+
+/** The live player for `id`, or `nullptr`. Used by the renderer. */
+MediaPlayer* xt_media_by_id(uint64_t id);
+
+/** Tear down process-wide media state (audio subsystem, id table). */
+void xt_media_shutdown();
+
+}  // namespace xtgui
+
+#endif  // XT_GUI_MEDIA_H