rill-node 0.0.0-stage → 2026.11.15

package/native-src/src/rill/extractors/mixcloud.cr

@@ -0,0 +1,495 @@
+require "base64"
+require "json"
+require "uri"
+
+require "../errors"
+require "../extractor"
+require "../format"
+require "../http_url"
+require "../json_value"
+require "../media"
+require "../playlist"
+
+module Rill::Extractors
+  class Mixcloud < Extractor
+    DEFAULT_API_BASE = "https://app.mixcloud.com"
+    DEFAULT_LIST     = "uploads"
+    ISO_DATE         = /\A\d{4}-\d{2}-\d{2}/
+    DECRYPTION_KEY   = "IFYOUWANTTHEARTISTSTOGETPAIDDONOTDOWNLOADFROMMIXCLOUD"
+    FIELDS           = "name description audioLength publishDate restrictedReason " \
+                       "picture(width: 1024, height: 1024) { url } " \
+                       "owner { displayName username url } " \
+                       "streamInfo { url hlsUrl dashUrl }"
+
+    RESERVED_SLUGS = %w(stream uploads favorites listens playlists)
+
+    CLOUDCAST_PATH = %r{\A/([^/]+)/([^/]+)/?\z}
+    USER_LIST_PATH = %r{\A/([^/]+)/(uploads|favorites|listens|stream)/?\z}
+    PLAYLIST_PATH  = %r{\A/([^/]+)/playlists/([^/]+)/?\z}
+    USER_PATH      = %r{\A/([^/]+)/?\z}
+
+    struct CloudcastRoute
+      getter username : String
+      getter slug : String
+
+      def initialize(@username : String, @slug : String)
+      end
+    end
+
+    struct UserRoute
+      getter username : String
+      getter list : String
+
+      def initialize(@username : String, @list : String)
+      end
+    end
+
+    struct PlaylistRoute
+      getter username : String
+      getter slug : String
+
+      def initialize(@username : String, @slug : String)
+      end
+    end
+
+    alias Route = CloudcastRoute | UserRoute | PlaylistRoute
+
+    def initialize(@api_base : String = DEFAULT_API_BASE)
+      @api_base = @api_base.rstrip('/')
+    end
+
+    def name : String
+      "mixcloud"
+    end
+
+    def matches?(url : URI) : Bool
+      return false unless url.scheme == "http" || url.scheme == "https"
+
+      case url.host.try(&.downcase)
+      when "mixcloud.com", "www.mixcloud.com", "beta.mixcloud.com", "m.mixcloud.com"
+        !lookup(url.path).nil?
+      else
+        false
+      end
+    end
+
+    def extract(url : URI, client : HTTPClient) : Media | Playlist
+      route = lookup(url.path)
+      unless route
+        raise UnsupportedUrlError.new(url.to_s)
+      end
+
+      case route
+      when CloudcastRoute
+        extract_cloudcast(route, url, client)
+      when UserRoute
+        extract_user(route, url, client)
+      when PlaylistRoute
+        extract_playlist(route, url, client)
+      else
+        raise UnsupportedUrlError.new(url.to_s)
+      end
+    end
+
+    private def lookup(path : String) : Route?
+      if match = PLAYLIST_PATH.match(path)
+        return PlaylistRoute.new(match[1], match[2])
+      end
+
+      if match = USER_LIST_PATH.match(path)
+        return UserRoute.new(match[1], match[2])
+      end
+
+      if match = CLOUDCAST_PATH.match(path)
+        username = match[1]
+        slug = match[2]
+        return CloudcastRoute.new(username, slug) unless RESERVED_SLUGS.includes?(slug.downcase)
+      end
+
+      match = USER_PATH.match(path)
+      return nil unless match
+
+      UserRoute.new(match[1], DEFAULT_LIST)
+    end
+
+    private def extract_cloudcast(
+      route : CloudcastRoute,
+      url : URI,
+      client : HTTPClient,
+    ) : Media
+      cloudcast = fetch_cloudcast(client, route.username, route.slug, url)
+      check_restrictions(cloudcast, url)
+      title = title(cloudcast, url)
+      duration = JsonValue.seconds(cloudcast["audioLength"]?)
+      format = audio_format(cloudcast, url, duration)
+
+      Media.new(
+        id: "#{route.username}_#{route.slug}",
+        title: title,
+        description: JsonValue.string(cloudcast["description"]?),
+        webpage_url: url,
+        uploader: uploader(cloudcast),
+        duration_seconds: duration,
+        thumbnail_url: thumbnail_url(cloudcast),
+        upload_date: upload_date(cloudcast),
+        extractor: name,
+        formats: [format],
+      )
+    end
+
+    private def extract_user(
+      route : UserRoute,
+      url : URI,
+      client : HTTPClient,
+    ) : Playlist
+      entries = [] of PlaylistEntry
+      display_name = nil
+      biography = nil
+      cursor = nil
+
+      loop do
+        query = user_query(route.username, route.list, cursor)
+        user = graphql_root(client, query, "userLookup", url, "user")
+        display_name ||= JsonValue.string(user["displayName"]?)
+        biography ||= JsonValue.string(user["biog"]?)
+        connection = user[route.list]?.try(&.as_h?)
+        break unless connection
+
+        append_cloudcast_entries(connection, entries)
+        cursor = next_cursor(connection)
+        break unless cursor
+      end
+
+      if entries.empty?
+        raise MediaUnavailableError.new(url.to_s, "user has no #{route.list} entries")
+      end
+
+      title = display_name || route.username
+
+      Playlist.new(
+        id: "#{route.username}_#{route.list}",
+        title: "#{title} (#{route.list})",
+        description: biography,
+        webpage_url: url,
+        extractor: name,
+        entries: entries,
+      )
+    end
+
+    private def extract_playlist(
+      route : PlaylistRoute,
+      url : URI,
+      client : HTTPClient,
+    ) : Playlist
+      entries = [] of PlaylistEntry
+      playlist_name = nil
+      description = nil
+      cursor = nil
+
+      loop do
+        query = playlist_query(route.username, route.slug, cursor)
+        playlist = graphql_root(client, query, "playlistLookup", url, "playlist")
+        playlist_name ||= JsonValue.string(playlist["name"]?)
+        description ||= JsonValue.string(playlist["description"]?)
+        connection = playlist["items"]?.try(&.as_h?)
+        break unless connection
+
+        append_playlist_entries(connection, entries)
+        cursor = next_cursor(connection)
+        break unless cursor
+      end
+
+      if entries.empty?
+        raise MediaUnavailableError.new(url.to_s, "playlist has no entries")
+      end
+
+      title = playlist_name
+      unless title
+        raise MediaUnavailableError.new(url.to_s, "playlist name is missing")
+      end
+
+      Playlist.new(
+        id: "#{route.username}_#{route.slug}",
+        title: title,
+        description: description,
+        webpage_url: url,
+        extractor: name,
+        entries: entries,
+      )
+    end
+
+    private def fetch_cloudcast(
+      client : HTTPClient,
+      username : String,
+      slug : String,
+      url : URI,
+    ) : Hash(String, JSON::Any)
+      query = cloudcast_query(username, slug)
+      graphql_root(client, query, "cloudcastLookup", url, "cloudcast")
+    end
+
+    private def graphql_root(
+      client : HTTPClient,
+      query : String,
+      lookup : String,
+      url : URI,
+      label : String,
+    ) : Hash(String, JSON::Any)
+      endpoint = URI.parse("#{@api_base}/graphql")
+      endpoint.query = URI::Params.encode({"query" => query})
+      response = client.get(endpoint)
+      payload = JSON.parse(response.body)
+      data = payload["data"]?.try(&.as_h?)
+      root = data.try { |hash| hash[lookup]?.try(&.as_h?) }
+      unless root
+        raise MediaUnavailableError.new(url.to_s, "#{label} was not found")
+      end
+
+      root
+    rescue JSON::ParseException
+      raise MediaUnavailableError.new(url.to_s, "GraphQL response is not valid JSON")
+    rescue ex : HTTPError
+      if ex.status.code == 404
+        raise MediaUnavailableError.new(url.to_s, "#{label} was not found")
+      end
+
+      raise ex
+    end
+
+    private def cloudcast_query(username : String, slug : String) : String
+      <<-GRAPHQL
+        {
+          cloudcastLookup(lookup: {username: #{username.to_json}, slug: #{slug.to_json}}) {
+            #{FIELDS}
+          }
+        }
+        GRAPHQL
+    end
+
+    private def user_query(username : String, list : String, cursor : String?) : String
+      after = cursor_clause(cursor)
+      <<-GRAPHQL
+        {
+          userLookup(lookup: {username: #{username.to_json}}) {
+            displayName
+            biog
+            #{list}(first: 100#{after}) {
+              edges {
+                node {
+                  name
+                  slug
+                  url
+                  owner { username }
+                }
+              }
+              pageInfo {
+                endCursor
+                hasNextPage
+              }
+            }
+          }
+        }
+        GRAPHQL
+    end
+
+    private def playlist_query(username : String, slug : String, cursor : String?) : String
+      after = cursor_clause(cursor)
+      <<-GRAPHQL
+        {
+          playlistLookup(lookup: {username: #{username.to_json}, slug: #{slug.to_json}}) {
+            name
+            description
+            items(first: 100#{after}) {
+              edges {
+                node {
+                  cloudcast {
+                    name
+                    slug
+                    url
+                    owner { username }
+                  }
+                }
+              }
+              pageInfo {
+                endCursor
+                hasNextPage
+              }
+            }
+          }
+        }
+        GRAPHQL
+    end
+
+    private def cursor_clause(cursor : String?) : String
+      return "" unless cursor
+
+      ", after: #{cursor.to_json}"
+    end
+
+    private def append_cloudcast_entries(
+      connection : Hash(String, JSON::Any),
+      entries : Array(PlaylistEntry),
+    ) : Nil
+      edges = connection["edges"]?.try(&.as_a?)
+      return unless edges
+
+      edges.each do |edge|
+        node = edge.as_h?
+        next unless node
+
+        cloudcast = node["node"]?.try(&.as_h?)
+        next unless cloudcast
+
+        entry = cloudcast_entry(cloudcast)
+        entries << entry if entry
+      end
+    end
+
+    private def append_playlist_entries(
+      connection : Hash(String, JSON::Any),
+      entries : Array(PlaylistEntry),
+    ) : Nil
+      edges = connection["edges"]?.try(&.as_a?)
+      return unless edges
+
+      edges.each do |edge|
+        node = edge.as_h?
+        next unless node
+
+        item = node["node"]?.try(&.as_h?)
+        next unless item
+
+        cloudcast = item["cloudcast"]?.try(&.as_h?)
+        next unless cloudcast
+
+        entry = cloudcast_entry(cloudcast)
+        entries << entry if entry
+      end
+    end
+
+    private def cloudcast_entry(cloudcast : Hash(String, JSON::Any)) : PlaylistEntry?
+      cloudcast_url = HttpUrl.absolute(JsonValue.string(cloudcast["url"]?))
+      return nil unless cloudcast_url
+
+      slug = JsonValue.string(cloudcast["slug"]?)
+      owner = cloudcast["owner"]?.try(&.as_h?)
+      owner_username = owner.try { |hash| JsonValue.string(hash["username"]?) }
+      id = if slug && owner_username
+             "#{owner_username}_#{slug}"
+           end
+
+      PlaylistEntry.new(
+        url: cloudcast_url,
+        title: JsonValue.string(cloudcast["name"]?),
+        id: id,
+      )
+    end
+
+    private def next_cursor(connection : Hash(String, JSON::Any)) : String?
+      page_info = connection["pageInfo"]?.try(&.as_h?)
+      return nil unless page_info
+      return nil unless page_info["hasNextPage"]?.try(&.as_bool?)
+
+      JsonValue.string(page_info["endCursor"]?)
+    end
+
+    private def check_restrictions(cloudcast : Hash(String, JSON::Any), url : URI) : Nil
+      reason = JsonValue.string(cloudcast["restrictedReason"]?)
+      return if reason.nil? || reason.empty?
+
+      case reason
+      when "tracklist"
+        raise MediaUnavailableError.new(url.to_s, "track is unavailable due to licensing restrictions")
+      when "repeat_play"
+        raise MediaUnavailableError.new(url.to_s, "play limit was reached")
+      else
+        raise MediaUnavailableError.new(url.to_s, "track is restricted")
+      end
+    end
+
+    private def title(cloudcast : Hash(String, JSON::Any), url : URI) : String
+      value = JsonValue.string(cloudcast["name"]?)
+      unless value
+        raise MediaUnavailableError.new(url.to_s, "cloudcast name is missing")
+      end
+
+      value
+    end
+
+    private def uploader(cloudcast : Hash(String, JSON::Any)) : String?
+      owner = cloudcast["owner"]?.try(&.as_h?)
+      return nil unless owner
+
+      JsonValue.string(owner["displayName"]?) || JsonValue.string(owner["username"]?)
+    end
+
+    private def thumbnail_url(cloudcast : Hash(String, JSON::Any)) : URI?
+      picture = cloudcast["picture"]?.try(&.as_h?)
+      value = picture.try { |hash| JsonValue.string(hash["url"]?) }
+      HttpUrl.absolute(value)
+    end
+
+    private def upload_date(cloudcast : Hash(String, JSON::Any)) : String?
+      value = JsonValue.string(cloudcast["publishDate"]?)
+      return nil unless value
+      return value[0, 10] if ISO_DATE.match(value)
+
+      nil
+    end
+
+    private def audio_format(
+      cloudcast : Hash(String, JSON::Any),
+      url : URI,
+      duration : Float64?,
+    ) : Format
+      info = cloudcast["streamInfo"]?.try(&.as_h?)
+      if info
+        if stream_url = decrypt_stream_url(info["url"]?)
+          return stream_format("progressive", FormatProtocol::Http, stream_url, duration)
+        end
+
+        if stream_url = decrypt_stream_url(info["hlsUrl"]?)
+          return stream_format("hls", FormatProtocol::Hls, stream_url, duration)
+        end
+
+        if stream_url = decrypt_stream_url(info["dashUrl"]?)
+          return stream_format("dash", FormatProtocol::Dash, stream_url, duration)
+        end
+      end
+
+      raise MediaUnavailableError.new(url.to_s, "cloudcast has no playable stream")
+    end
+
+    private def decrypt_stream_url(value : JSON::Any?) : URI?
+      encoded = JsonValue.string(value)
+      return nil unless encoded
+
+      cipher = Base64.decode(encoded)
+      plain = String.build(cipher.size) do |io|
+        cipher.each_with_index do |byte, index|
+          io.write_byte(byte ^ DECRYPTION_KEY.to_slice[index % DECRYPTION_KEY.bytesize])
+        end
+      end
+
+      HttpUrl.absolute(plain)
+    rescue Base64::Error
+      nil
+    end
+
+    private def stream_format(
+      format_id : String,
+      protocol : FormatProtocol,
+      stream_url : URI,
+      duration : Float64?,
+    ) : Format
+      Format.new(
+        format_id: format_id,
+        url: stream_url,
+        kind: FormatKind::Audio,
+        protocol: protocol,
+        container: "m4a",
+        duration_seconds: duration,
+      )
+    end
+  end
+end

package/native-src/src/rill/extractors/peer_tube.cr

@@ -0,0 +1,226 @@
+require "json"
+require "set"
+require "uri"
+
+require "../errors"
+require "../extractor"
+require "../format"
+require "../http_url"
+require "../json_value"
+require "../media"
+
+module Rill::Extractors
+  class PeerTube < Extractor
+    VIDEO_PATH = %r{\A/(?:w|videos/(?:watch|embed))/([A-Za-z0-9-]+)/?\z}
+
+    def initialize(@api_base : String? = nil)
+    end
+
+    def name : String
+      "peertube"
+    end
+
+    def matches?(url : URI) : Bool
+      return false unless url.scheme == "http" || url.scheme == "https"
+
+      !VIDEO_PATH.match(url.path).nil?
+    end
+
+    def extract(url : URI, client : HTTPClient) : Media
+      identifier = identifier_for(url)
+      unless identifier
+        raise UnsupportedUrlError.new(url.to_s)
+      end
+
+      api_url = URI.parse("#{api_base(url)}/api/v1/videos/#{URI.encode_path_segment(identifier)}")
+      payload = api_payload(client, api_url, url)
+
+      if payload["isLive"]?.try(&.as_bool?)
+        raise MediaUnavailableError.new(url.to_s, "live videos are not supported")
+      end
+
+      formats = formats_from(payload)
+      if formats.empty?
+        raise MediaUnavailableError.new(url.to_s, "video has no downloadable files")
+      end
+
+      Media.new(
+        id: JsonValue.string(payload["uuid"]?) || identifier,
+        title: JsonValue.string(payload["name"]?) || identifier,
+        description: JsonValue.string(payload["description"]?),
+        webpage_url: url,
+        uploader: uploader(payload),
+        duration_seconds: JsonValue.number(payload["duration"]?),
+        thumbnail_url: thumbnail_url(url, payload),
+        upload_date: date_value(payload),
+        extractor: name,
+        formats: formats,
+      )
+    end
+
+    private def identifier_for(url : URI) : String?
+      match = VIDEO_PATH.match(url.path)
+      return nil unless match
+
+      identifier = URI.decode(match[1])
+      identifier.empty? ? nil : identifier
+    end
+
+    private def api_base(url : URI) : String
+      @api_base || begin
+        port = url.port
+        if port
+          "#{url.scheme}://#{url.host}:#{port}"
+        else
+          "#{url.scheme}://#{url.host}"
+        end
+      end
+    end
+
+    private def api_payload(client : HTTPClient, api_url : URI, url : URI) : Hash(String, JSON::Any)
+      response = client.get(api_url)
+      payload = JSON.parse(response.body).as_h?
+      unless payload
+        raise MediaUnavailableError.new(url.to_s, "video metadata is not a JSON object")
+      end
+
+      payload
+    rescue ex : HTTPError
+      case ex.status.code
+      when 401, 403, 404
+        raise MediaUnavailableError.new(url.to_s, "video is unavailable, private, or protected")
+      else
+        raise ex
+      end
+    rescue JSON::ParseException
+      raise MediaUnavailableError.new(url.to_s, "video metadata is not valid JSON")
+    end
+
+    private def formats_from(payload : Hash(String, JSON::Any)) : Array(Format)
+      duration = JsonValue.number(payload["duration"]?)
+      language = language_for(payload)
+      formats = [] of Format
+      seen = Set(String).new
+
+      files = payload["files"]?.try(&.as_a?) || [] of JSON::Any
+      files.each_with_index do |file, index|
+        file_hash = file.as_h?
+        next unless file_hash
+
+        file_url = HttpUrl.absolute(JsonValue.string(file_hash["fileUrl"]?))
+        next unless file_url
+
+        resolution = file_hash["resolution"]?.try(&.as_h?)
+        height = resolution.try(&.["id"]?).try(&.as_i64?)
+        height = nil if height && (height <= 0 || height > 100_000)
+        label = resolution.try(&.["label"]?).try(&.as_s?)
+        audio_only = height.nil?
+
+        format_id = audio_only ? "audio" : "#{height}p"
+        if seen.includes?(format_id)
+          format_id = "#{format_id}-#{index + 1}"
+        end
+        seen << format_id
+
+        size = integer(file_hash["size"]?)
+        formats << Format.new(
+          format_id: format_id,
+          url: file_url,
+          kind: audio_only ? FormatKind::Audio : FormatKind::Combined,
+          container: container_for(file_url),
+          bitrate: bitrate(size, duration),
+          height: audio_only ? nil : height.try(&.to_i32),
+          fps: JsonValue.positive_number(file_hash["fps"]?),
+          filesize: size,
+          language: language,
+          note: label,
+        )
+      end
+
+      if formats.empty?
+        append_hls_formats(payload, formats, language)
+      end
+
+      formats
+    end
+
+    private def append_hls_formats(
+      payload : Hash(String, JSON::Any),
+      formats : Array(Format),
+      language : String?,
+    ) : Nil
+      playlists = payload["streamingPlaylists"]?.try(&.as_a?) || [] of JSON::Any
+
+      playlists.each_with_index do |playlist, index|
+        playlist_hash = playlist.as_h?
+        next unless playlist_hash
+
+        playlist_url = HttpUrl.absolute(JsonValue.string(playlist_hash["playlistUrl"]?))
+        next unless playlist_url
+
+        formats << Format.new(
+          format_id: playlists.size == 1 ? "hls" : "hls-#{index + 1}",
+          url: playlist_url,
+          kind: FormatKind::Combined,
+          protocol: FormatProtocol::Hls,
+          language: language,
+          note: "HLS",
+        )
+      end
+    end
+
+    private def uploader(payload : Hash(String, JSON::Any)) : String?
+      account = payload["account"]?.try(&.as_h?)
+      return nil unless account
+
+      JsonValue.string(account["displayName"]?) || JsonValue.string(account["name"]?)
+    end
+
+    private def thumbnail_url(url : URI, payload : Hash(String, JSON::Any)) : URI?
+      path = JsonValue.string(payload["thumbnailPath"]?) || JsonValue.string(payload["previewPath"]?)
+      return nil unless path
+      return nil unless path.starts_with?('/')
+
+      port = url.port
+      host = port ? "#{url.host}:#{port}" : url.host.to_s
+      HttpUrl.absolute("#{url.scheme}://#{host}#{path}")
+    end
+
+    private def language_for(payload : Hash(String, JSON::Any)) : String?
+      value = payload["language"]?
+      return nil unless value
+
+      if hash = value.as_h?
+        JsonValue.string(hash["id"]?)
+      else
+        value.as_s?
+      end
+    end
+
+    private def date_value(payload : Hash(String, JSON::Any)) : String?
+      value = JsonValue.string(payload["publishedAt"]?)
+      return nil unless value
+      return nil if value.size < 10
+
+      value[0, 10]
+    end
+
+    private def container_for(url : URI) : String?
+      extension = File.extname(url.path).lchop('.').downcase
+      extension.empty? ? nil : extension
+    end
+
+    private def bitrate(size : Int64?, duration : Float64?) : Int64?
+      return nil unless size && duration && duration > 0
+
+      (size * 8 / duration).to_i64
+    end
+
+    private def integer(value : JSON::Any?) : Int64?
+      parsed = JsonValue.integer(value)
+      return nil unless parsed && parsed >= 0
+
+      parsed
+    end
+  end
+end