rill-node 0.0.0-stage → 2026.11.14

package/native-src/src/rill/errors.cr

@@ -0,0 +1,155 @@
+require "http/status"
+require "uri"
+
+module Rill
+  class Error < Exception
+  end
+
+  class UnsupportedUrlError < Error
+    getter url : String
+
+    def initialize(@url : String)
+      super("unsupported URL: #{@url}")
+    end
+  end
+
+  class ExtractionError < Error
+  end
+
+  class MediaUnavailableError < ExtractionError
+    getter url : String
+
+    def initialize(@url : String, reason : String)
+      super("media unavailable at #{@url}: #{reason}")
+    end
+  end
+
+  class InvalidExtractionError < ExtractionError
+    getter url : String
+
+    def initialize(@url : String, reason : String)
+      super("invalid extraction result for #{@url}: #{reason}")
+    end
+  end
+
+  class HTTPError < Error
+    getter url : URI
+    getter status : HTTP::Status
+    getter response_body : String
+
+    def initialize(@url : URI, @status : HTTP::Status, @response_body : String)
+      super("GET #{@url} returned #{@status.code} #{@status.description}")
+    end
+  end
+
+  class ResponseTooLargeError < Error
+    getter url : URI
+    getter limit : Int64
+
+    def initialize(@url : URI, @limit : Int64)
+      super("response from #{@url} exceeded #{@limit} bytes")
+    end
+  end
+
+  class TooManyRedirectsError < Error
+    getter url : URI
+    getter limit : Int32
+
+    def initialize(@url : URI, @limit : Int32)
+      super("request to #{@url} exceeded #{@limit} redirects")
+    end
+  end
+
+  class IncompleteResponseError < Error
+    getter url : URI
+    getter expected_bytes : Int64
+    getter actual_bytes : Int64
+
+    def initialize(@url : URI, @expected_bytes : Int64, @actual_bytes : Int64)
+      super("response from #{@url} ended after #{@actual_bytes} of #{@expected_bytes} bytes")
+    end
+  end
+
+  class NetworkError < Error
+    getter url : URI
+
+    def initialize(@url : URI, cause : Exception)
+      super("request to #{@url} failed: #{cause.message}", cause)
+    end
+  end
+
+  class InvalidUrlError < Error
+    getter url : String
+
+    def initialize(@url : String, reason : String)
+      super("invalid URL #{@url}: #{reason}")
+    end
+  end
+
+  class InvalidOptionsError < Error
+    def initialize(reason : String)
+      super("invalid request options: #{reason}")
+    end
+  end
+
+  class MediaBackendUnavailableError < Error
+  end
+
+  class MediaMuxError < Error
+  end
+
+  class ManifestError < Error
+  end
+
+  class UnsupportedManifestError < ManifestError
+  end
+
+  class CancelledError < Error
+    def initialize
+      super("operation cancelled")
+    end
+  end
+
+  class DownloadError < Error
+  end
+
+  class DownloadWriteError < DownloadError
+    getter path : String
+
+    def initialize(@path : String, cause : Exception)
+      super("failed to write #{@path}: #{cause.message}", cause)
+    end
+  end
+
+  class FileExistsError < DownloadError
+    getter path : String
+
+    def initialize(@path : String)
+      super("file already exists: #{@path}")
+    end
+  end
+
+  class IncompleteDownloadError < DownloadError
+    getter path : String
+    getter expected_bytes : Int64
+    getter actual_bytes : Int64
+
+    def initialize(@path : String, @expected_bytes : Int64, @actual_bytes : Int64)
+      super("incomplete download for #{@path}: expected #{@expected_bytes} bytes, got #{@actual_bytes}")
+    end
+  end
+
+  class ResumeError < DownloadError
+    getter path : String
+
+    def initialize(@path : String, reason : String)
+      super("cannot resume #{@path}: #{reason}")
+    end
+  end
+
+  class InvalidOutputPathError < DownloadError
+    def initialize(path : String, directory : String)
+      super("refusing to write #{path} outside #{directory}")
+    end
+  end
+end

package/native-src/src/rill/extractor.cr

@@ -0,0 +1,18 @@
+require "json"
+require "uri"
+
+require "./errors"
+require "./format"
+require "./http_client"
+require "./media"
+require "./playlist"
+
+module Rill
+  abstract class Extractor
+    abstract def name : String
+
+    abstract def matches?(url : URI) : Bool
+
+    abstract def extract(url : URI, client : HTTPClient) : Media | Playlist
+  end
+end

package/native-src/src/rill/extractor_registry.cr

@@ -0,0 +1,98 @@
+require "uri"
+
+require "./errors"
+require "./extractor"
+require "./extractors/bandcamp"
+require "./extractors/dash"
+require "./extractors/direct_media"
+require "./extractors/generic"
+require "./extractors/hls"
+require "./extractors/internet_archive"
+require "./extractors/m3u"
+require "./extractors/media_ccc_de"
+require "./extractors/mixcloud"
+require "./extractors/peer_tube"
+require "./extractors/rumble"
+require "./extractors/soundcloud"
+require "./extractors/spotify"
+require "./extractors/twitch_clip"
+require "./extractors/youtube"
+require "./http_client"
+require "./playlist"
+require "./request_options"
+
+module Rill
+  class ExtractorRegistry
+    def self.default : self
+      registry = new
+      registry.register(Extractors::InternetArchive)
+      registry.register(Extractors::MediaCccDe)
+      registry.register(Extractors::PeerTube)
+      registry.register(Extractors::YouTube)
+      registry.register(Extractors::Spotify)
+      registry.register(Extractors::Bandcamp)
+      registry.register(Extractors::Rumble)
+      registry.register(Extractors::SoundCloud)
+      registry.register(Extractors::Mixcloud)
+      registry.register(Extractors::TwitchClip)
+      registry.register(Extractors::Hls)
+      registry.register(Extractors::Dash)
+      registry.register(Extractors::M3u)
+      registry.register(Extractors::DirectMedia)
+      registry.register(Extractors::Generic)
+      registry
+    end
+
+    def initialize
+      @extractors = [] of Extractor.class
+    end
+
+    def register(extractor : Extractor.class) : Nil
+      @extractors << extractor
+    end
+
+    def extractors : Array(Extractor.class)
+      @extractors.dup
+    end
+
+    def extractor_for(url : String | URI) : Extractor.class | Nil
+      uri = url.is_a?(URI) ? url : URI.parse(url)
+      return nil unless uri.scheme == "http" || uri.scheme == "https"
+
+      @extractors.find { |extractor| extractor.new.matches?(uri) }
+    rescue URI::Error
+      nil
+    end
+
+    def extract(url : String | URI, options : RequestOptions = RequestOptions.new) : Media | Playlist
+      extract(url, HTTPClient.new(options))
+    end
+
+    def extract(url : String | URI, client : HTTPClient) : Media | Playlist
+      uri = parse_url(url)
+      extractor = extractor_for(uri)
+      unless extractor
+        raise UnsupportedUrlError.new(uri.to_s)
+      end
+
+      result = extractor.new.extract(uri, client)
+      case result
+      when Media
+        result.validate!
+      when Playlist
+        result.validate!
+      end
+      result
+    end
+
+    private def parse_url(url : String | URI) : URI
+      uri = url.is_a?(URI) ? url : URI.parse(url)
+      unless uri.scheme == "http" || uri.scheme == "https"
+        raise UnsupportedUrlError.new(uri.to_s)
+      end
+      uri
+    rescue URI::Error
+      raise UnsupportedUrlError.new(url.to_s)
+    end
+  end
+end

package/native-src/src/rill/extractors/bandcamp.cr

@@ -0,0 +1,265 @@
+require "html"
+require "json"
+require "time"
+require "uri"
+
+require "../errors"
+require "../extractor"
+require "../format"
+require "../http_url"
+require "../json_value"
+require "../media"
+require "../playlist"
+
+module Rill::Extractors
+  class Bandcamp < Extractor
+    HOST     = /\A[a-z0-9-]+\.bandcamp\.com\z/i
+    PATH     = %r{\A/(track|album)/([^/]+)/?\z}
+    ISO_DATE = /\A\d{4}-\d{2}-\d{2}/
+    DATE     = "%d %b %Y %H:%M:%S GMT"
+    TRALBUM  = /data-tralbum\s*=\s*(?:"([^"]*)"|'([^']*)')/m
+    OG_IMAGE = /\bproperty\s*=\s*(?:"og:image"|'og:image')/i
+    CONTENT  = /\bcontent\s*=\s*(?:"([^"]*)"|'([^']*)')/i
+
+    def name : String
+      "bandcamp"
+    end
+
+    def matches?(url : URI) : Bool
+      return false unless http_url?(url)
+      return false unless artist_host?(url.host)
+
+      !PATH.match(url.path).nil?
+    end
+
+    def extract(url : URI, client : HTTPClient) : Media | Playlist
+      match = PATH.match(url.path)
+      unless match
+        raise UnsupportedUrlError.new(url.to_s)
+      end
+
+      response = fetch_page(client, url)
+      tralbum = tralbum_data(response.body, url)
+
+      if match[1] == "track"
+        extract_track(url, tralbum, thumbnail_url(response.body, response.url))
+      else
+        extract_album(url, tralbum)
+      end
+    end
+
+    private def http_url?(url : URI) : Bool
+      url.scheme == "http" || url.scheme == "https"
+    end
+
+    private def artist_host?(host : String?) : Bool
+      return false unless host
+
+      name = host.downcase
+      return false if name == "www.bandcamp.com"
+
+      !HOST.match(name).nil?
+    end
+
+    private def fetch_page(client : HTTPClient, url : URI) : HTTPClient::Response
+      client.get(url)
+    rescue ex : HTTPError
+      if ex.status.code == 404
+        raise MediaUnavailableError.new(url.to_s, "page not found")
+      end
+
+      raise ex
+    end
+
+    private def tralbum_data(body : String, url : URI) : Hash(String, JSON::Any)
+      attribute = tralbum_attribute(body)
+      unless attribute
+        raise MediaUnavailableError.new(url.to_s, "page has no data-tralbum attribute")
+      end
+
+      payload = JSON.parse(HTML.unescape(attribute)).as_h?
+      unless payload
+        raise MediaUnavailableError.new(url.to_s, "data-tralbum is not a JSON object")
+      end
+
+      payload
+    rescue JSON::ParseException
+      raise MediaUnavailableError.new(url.to_s, "data-tralbum is not valid JSON")
+    end
+
+    private def tralbum_attribute(body : String) : String?
+      match = TRALBUM.match(body)
+      return nil unless match
+
+      match[1]? || match[2]?
+    end
+
+    private def extract_track(
+      url : URI,
+      tralbum : Hash(String, JSON::Any),
+      thumbnail : URI?,
+    ) : Media
+      if tralbum["is_private_stream"]?.try(&.as_bool?)
+        raise MediaUnavailableError.new(url.to_s, "track stream is private")
+      end
+
+      current = tralbum["current"]?.try(&.as_h?) || {} of String => JSON::Any
+      track = first_track(tralbum, url)
+
+      if track["private"]?.try(&.as_bool?)
+        raise MediaUnavailableError.new(url.to_s, "track is private")
+      end
+
+      title = JsonValue.string(current["title"]?) || JsonValue.string(track["title"]?)
+      unless title
+        raise MediaUnavailableError.new(url.to_s, "track title is missing")
+      end
+
+      duration = JsonValue.seconds(track["duration"]?)
+      format = audio_format(track, url, duration)
+
+      Media.new(
+        id: track_id(track) || JsonValue.string(tralbum["id"]?),
+        title: title,
+        description: JsonValue.string(current["about"]?),
+        webpage_url: url,
+        uploader: JsonValue.string(tralbum["artist"]?) || JsonValue.string(current["artist"]?),
+        duration_seconds: duration,
+        thumbnail_url: thumbnail,
+        upload_date: upload_date(current),
+        extractor: name,
+        formats: [format],
+      )
+    end
+
+    private def first_track(
+      tralbum : Hash(String, JSON::Any),
+      url : URI,
+    ) : Hash(String, JSON::Any)
+      tracks = tralbum["trackinfo"]?.try(&.as_a?)
+      unless tracks
+        raise MediaUnavailableError.new(url.to_s, "page has no trackinfo")
+      end
+
+      track = tracks.first?.try(&.as_h?)
+      unless track
+        raise MediaUnavailableError.new(url.to_s, "track data is missing")
+      end
+
+      track
+    end
+
+    private def audio_format(
+      track : Hash(String, JSON::Any),
+      url : URI,
+      duration : Float64?,
+    ) : Format
+      file = track["file"]?.try(&.as_h?)
+      value = file.try { |hash| JsonValue.string(hash["mp3-128"]?) }
+      unless value
+        raise MediaUnavailableError.new(url.to_s, "stream is unavailable")
+      end
+
+      stream_url = HttpUrl.absolute(protocol_url(value))
+      unless stream_url
+        raise MediaUnavailableError.new(url.to_s, "stream URL is invalid")
+      end
+
+      Format.new(
+        format_id: "mp3-128",
+        url: stream_url,
+        kind: FormatKind::Audio,
+        container: "mp3",
+        bitrate: 128_000_i64,
+        duration_seconds: duration,
+      )
+    end
+
+    private def protocol_url(value : String) : String
+      value.starts_with?("//") ? "https:#{value}" : value
+    end
+
+    private def extract_album(
+      page_url : URI,
+      tralbum : Hash(String, JSON::Any),
+    ) : Playlist
+      current = tralbum["current"]?.try(&.as_h?) || {} of String => JSON::Any
+      title = JsonValue.string(current["title"]?)
+      unless title
+        raise MediaUnavailableError.new(page_url.to_s, "album title is missing")
+      end
+
+      entries = album_entries(tralbum["trackinfo"]?, page_url)
+      if entries.empty?
+        raise MediaUnavailableError.new(page_url.to_s, "album has no tracks")
+      end
+
+      Playlist.new(
+        id: JsonValue.string(current["id"]?),
+        title: title,
+        webpage_url: page_url,
+        extractor: name,
+        entries: entries,
+      )
+    end
+
+    private def album_entries(
+      value : JSON::Any?,
+      page_url : URI,
+    ) : Array(PlaylistEntry)
+      tracks = value.try(&.as_a?)
+      return [] of PlaylistEntry unless tracks
+
+      entries = [] of PlaylistEntry
+      tracks.each do |item|
+        track = item.as_h?
+        next unless track
+
+        title_link = JsonValue.string(track["title_link"]?)
+        next unless title_link
+
+        entry_url = HttpUrl.resolve(page_url, title_link)
+        next unless entry_url
+
+        entries << PlaylistEntry.new(
+          url: entry_url,
+          title: JsonValue.string(track["title"]?),
+          id: track_id(track),
+        )
+      end
+
+      entries
+    end
+
+    private def track_id(track : Hash(String, JSON::Any)) : String?
+      JsonValue.string(track["id"]?) || JsonValue.string(track["track_id"]?)
+    end
+
+    private def upload_date(current : Hash(String, JSON::Any)) : String?
+      value = JsonValue.string(current["release_date"]?) || JsonValue.string(current["publish_date"]?)
+      return nil unless value
+      return value[0, 10] if ISO_DATE.match(value)
+
+      Time.parse(value, DATE, Time::Location::UTC).to_s("%Y-%m-%d")
+    rescue Time::Format::Error
+      nil
+    end
+
+    private def thumbnail_url(body : String, base_url : URI) : URI?
+      body.scan(/<meta\b[^>]*>/i) do |match|
+        meta = match[0]
+        next unless OG_IMAGE.match(meta)
+
+        content = CONTENT.match(meta)
+        next unless content
+
+        value = content[1]? || content[2]?
+        next unless value
+
+        return HttpUrl.resolve(base_url, HTML.unescape(value))
+      end
+
+      nil
+    end
+  end
+end

package/native-src/src/rill/extractors/dash.cr

@@ -0,0 +1,77 @@
+require "uri"
+
+require "../dash/parser"
+require "../errors"
+require "../extractor"
+require "../format"
+require "../media"
+
+module Rill::Extractors
+  class Dash < Extractor
+    def name : String
+      "dash"
+    end
+
+    def matches?(url : URI) : Bool
+      return false unless url.scheme == "http" || url.scheme == "https"
+
+      File.extname(url.path).downcase == ".mpd"
+    end
+
+    def extract(url : URI, client : HTTPClient) : Media
+      response = client.get(url)
+      manifest = Rill::Dash::Parser.parse(response.body, response.url)
+
+      formats = [] of Format
+      unsupported = false
+
+      manifest.representations.each do |representation|
+        unless representation.supported?
+          unsupported = true
+          next
+        end
+
+        mime = representation.mime
+        next unless mime
+
+        audio = mime.audio?
+        formats << Format.new(
+          format_id: representation.reference,
+          url: response.url,
+          kind: audio ? FormatKind::Audio : FormatKind::Video,
+          protocol: FormatProtocol::Dash,
+          container: mime.container,
+          bitrate: representation.bandwidth,
+          video_codec: audio ? nil : (representation.codecs || mime.codecs),
+          audio_codec: audio ? (representation.codecs || mime.audio_codec) : nil,
+          width: representation.width,
+          height: representation.height,
+          fps: representation.fps,
+          duration_seconds: representation.duration_seconds,
+          language: representation.language,
+          note: representation.codecs || "DASH",
+        )
+      end
+
+      if formats.empty?
+        reason = unsupported ? "DASH representations use unsupported segment information" : "DASH manifest has no representations"
+        raise MediaUnavailableError.new(url.to_s, reason)
+      end
+
+      Media.new(
+        title: title_for(url),
+        webpage_url: url,
+        duration_seconds: manifest.duration_seconds,
+        extractor: name,
+        formats: formats,
+      )
+    end
+
+    private def title_for(url : URI) : String
+      basename = File.basename(url.path)
+      stem = File.basename(basename, File.extname(basename))
+      decoded = URI.decode(stem)
+      decoded.empty? ? "manifest" : decoded
+    end
+  end
+end

package/native-src/src/rill/extractors/direct_media.cr

@@ -0,0 +1,73 @@
+require "uri"
+
+require "../extractor"
+require "../format"
+require "../media"
+
+module Rill::Extractors
+  class DirectMedia < Extractor
+    VIDEO_EXTENSIONS = %w(
+      3gp avi flv m2ts mkv mov mp4 m4v mpg mpeg ogv ts webm wmv
+    )
+
+    AUDIO_EXTENSIONS = %w(
+      aac aif aiff ape flac m4a mp3 oga ogg opus shn wav wma
+    )
+
+    def name : String
+      "direct"
+    end
+
+    def matches?(url : URI) : Bool
+      extension = extension_for(url)
+      !extension.nil?
+    end
+
+    def extract(url : URI, client : HTTPClient) : Media
+      extension = extension_for(url)
+      unless extension
+        raise UnsupportedUrlError.new(url.to_s)
+      end
+
+      format = Format.new(
+        format_id: "direct",
+        url: url,
+        kind: kind_for(extension),
+        container: extension,
+      )
+
+      Media.new(
+        title: title_for(url),
+        webpage_url: url,
+        extractor: name,
+        formats: [format],
+      )
+    end
+
+    private def extension_for(url : URI) : String?
+      return nil unless url.scheme == "http" || url.scheme == "https"
+
+      extension = File.extname(url.path).lchop('.').downcase
+      return nil if extension.empty?
+
+      if VIDEO_EXTENSIONS.includes?(extension) || AUDIO_EXTENSIONS.includes?(extension)
+        extension
+      end
+    end
+
+    private def kind_for(extension : String) : FormatKind
+      if AUDIO_EXTENSIONS.includes?(extension)
+        FormatKind::Audio
+      else
+        FormatKind::Combined
+      end
+    end
+
+    private def title_for(url : URI) : String
+      basename = File.basename(url.path)
+      stem = File.basename(basename, File.extname(basename))
+      decoded = URI.decode(stem)
+      decoded.empty? ? "media" : decoded
+    end
+  end
+end