rill-node 0.0.0-stage → 2026.11.14

package/native-src/src/rill/extractors/internet_archive.cr

@@ -0,0 +1,180 @@
+require "json"
+require "uri"
+
+require "../errors"
+require "../extractor"
+require "../format"
+require "../json_value"
+require "../media"
+require "../timestamp"
+
+module Rill::Extractors
+  class InternetArchive < Extractor
+    BASE_URL     = "https://archive.org"
+    DETAILS_PATH = %r{\A/details/([^/]+)/?\z}
+
+    AUDIO_EXTENSIONS = %w(
+      aac aif aiff ape flac m4a mp3 oga ogg opus shn wav wma
+    )
+
+    MEDIA_EXTENSIONS = {
+      "3gp", "avi", "flv", "m2ts", "mkv", "mov", "mp4", "m4v",
+      "mpg", "mpeg", "ogv", "ts", "webm", "wmv",
+      "aac", "aif", "aiff", "ape", "flac", "m4a", "mp3", "oga",
+      "ogg", "opus", "shn", "wav", "wma",
+    }
+
+    def initialize(@base_url : String = BASE_URL)
+    end
+
+    def name : String
+      "internet_archive"
+    end
+
+    def matches?(url : URI) : Bool
+      return false unless url.scheme == "http" || url.scheme == "https"
+      return false unless archive_host?(url.host)
+
+      !DETAILS_PATH.match(url.path).nil?
+    end
+
+    def extract(url : URI, client : HTTPClient) : Media
+      identifier = identifier_for(url)
+      unless identifier
+        raise UnsupportedUrlError.new(url.to_s)
+      end
+
+      metadata_url = URI.parse("#{@base_url}/metadata/#{URI.encode_path_segment(identifier)}")
+      response = client.get(metadata_url)
+      payload = parse_payload(response.body, url)
+
+      if payload["is_dark"]?.try(&.as_bool?)
+        raise MediaUnavailableError.new(url.to_s, "item is dark")
+      end
+
+      metadata = payload["metadata"]?.try(&.as_h?)
+      unless metadata
+        raise MediaUnavailableError.new(url.to_s, "item metadata is missing")
+      end
+
+      formats = formats_from(payload["files"]?, identifier)
+      if formats.empty?
+        raise MediaUnavailableError.new(url.to_s, "item has no downloadable media")
+      end
+
+      title = JsonValue.string(metadata["title"]?) || identifier
+      duration = Timestamp.clock_seconds(JsonValue.string(metadata["length"]?))
+      duration ||= formats.compact_map(&.duration_seconds).first?
+
+      Media.new(
+        id: JsonValue.string(metadata["identifier"]?) || identifier,
+        title: title,
+        description: JsonValue.string(metadata["description"]?),
+        webpage_url: url,
+        uploader: JsonValue.string(metadata["creator"]?) || JsonValue.string(metadata["uploader"]?),
+        duration_seconds: duration,
+        thumbnail_url: thumbnail_url_for(identifier),
+        upload_date: date_value(metadata),
+        extractor: name,
+        formats: formats,
+      )
+    end
+
+    private def identifier_for(url : URI) : String?
+      match = DETAILS_PATH.match(url.path)
+      return nil unless match
+
+      identifier = URI.decode(match[1])
+      identifier.empty? ? nil : identifier
+    end
+
+    private def archive_host?(host : String?) : Bool
+      case host.try(&.downcase)
+      when "archive.org", "www.archive.org"
+        true
+      else
+        false
+      end
+    end
+
+    private def parse_payload(body : String, url : URI) : Hash(String, JSON::Any)
+      payload = JSON.parse(body).as_h?
+      unless payload
+        raise MediaUnavailableError.new(url.to_s, "item metadata is not a JSON object")
+      end
+
+      payload
+    rescue JSON::ParseException
+      raise MediaUnavailableError.new(url.to_s, "item metadata is not valid JSON")
+    end
+
+    private def formats_from(files_value : JSON::Any?, identifier : String) : Array(Format)
+      files = files_value.try(&.as_a?)
+      return [] of Format unless files
+
+      formats = [] of Format
+
+      files.each do |file|
+        file_hash = file.as_h?
+        next unless file_hash
+
+        file_name = file_hash["name"]?.try(&.as_s?)
+        next unless file_name
+
+        extension = File.extname(file_name).lchop('.').downcase
+        next unless MEDIA_EXTENSIONS.includes?(extension)
+
+        label = JsonValue.string(file_hash["format"]?)
+        format = Format.new(
+          format_id: file_name,
+          url: download_url(identifier, file_name),
+          kind: kind_for(extension),
+          container: extension,
+          width: JsonValue.dimension(file_hash["width"]?),
+          height: JsonValue.dimension(file_hash["height"]?),
+          filesize: int_value(file_hash["size"]?),
+          duration_seconds: Timestamp.clock_seconds(JsonValue.string(file_hash["length"]?)),
+          note: label,
+        )
+        formats << format
+      end
+
+      formats
+    end
+
+    private def download_url(identifier : String, file_name : String) : URI
+      encoded_name = file_name.split('/').reject(&.empty?).map do |segment|
+        URI.encode_path_segment(segment)
+      end.join('/')
+
+      encoded_identifier = URI.encode_path_segment(identifier)
+      URI.parse("#{@base_url}/download/#{encoded_identifier}/#{encoded_name}")
+    end
+
+    private def thumbnail_url_for(identifier : String) : URI
+      URI.parse("#{@base_url}/services/img/#{URI.encode_path_segment(identifier)}")
+    end
+
+    private def kind_for(extension : String) : FormatKind
+      if AUDIO_EXTENSIONS.includes?(extension)
+        FormatKind::Audio
+      else
+        FormatKind::Combined
+      end
+    end
+
+    private def date_value(metadata : Hash(String, JSON::Any)) : String?
+      value = JsonValue.string(metadata["date"]?) || JsonValue.string(metadata["addeddate"]?)
+      return nil unless value
+
+      value.size >= 10 ? value[0, 10] : value
+    end
+
+    private def int_value(value : JSON::Any?) : Int64?
+      parsed = JsonValue.integer(value)
+      return nil unless parsed && parsed >= 0
+
+      parsed
+    end
+  end
+end

package/native-src/src/rill/extractors/m3u.cr

@@ -0,0 +1,81 @@
+require "uri"
+
+require "../errors"
+require "../extractor"
+require "../http_url"
+require "../media"
+require "../playlist"
+
+module Rill::Extractors
+  class M3u < Extractor
+    EXTINF = "#EXTINF:"
+
+    def name : String
+      "m3u"
+    end
+
+    def matches?(url : URI) : Bool
+      return false unless url.scheme == "http" || url.scheme == "https"
+
+      File.extname(url.path).downcase == ".m3u"
+    end
+
+    def extract(url : URI, client : HTTPClient) : Playlist
+      response = client.get(url)
+
+      if response.body.includes?("#EXT-X-")
+        raise UnsupportedManifestError.new("HLS playlists must use a .m3u8 URL")
+      end
+
+      entries = parse_entries(response.body, response.url)
+      if entries.empty?
+        raise MediaUnavailableError.new(url.to_s, "playlist has no entries")
+      end
+
+      Playlist.new(
+        title: title_for(url),
+        webpage_url: url,
+        extractor: name,
+        entries: entries,
+      )
+    end
+
+    private def parse_entries(body : String, base_url : URI) : Array(PlaylistEntry)
+      entries = [] of PlaylistEntry
+      pending_title : String? = nil
+
+      body.lchop('\uFEFF').each_line do |raw|
+        line = raw.strip
+        next if line.empty?
+
+        if line.starts_with?(EXTINF)
+          pending_title = parse_title(line[EXTINF.size..])
+        elsif line.starts_with?('#')
+          next
+        else
+          if entry_url = HttpUrl.resolve(base_url, line)
+            entries << PlaylistEntry.new(url: entry_url, title: pending_title)
+          end
+          pending_title = nil
+        end
+      end
+
+      entries
+    end
+
+    private def parse_title(value : String) : String?
+      _, separator, title = value.partition(',')
+      return nil if separator.empty?
+
+      title = title.strip
+      title.empty? ? nil : title
+    end
+
+    private def title_for(url : URI) : String
+      basename = File.basename(url.path)
+      stem = File.basename(basename, File.extname(basename))
+      decoded = URI.decode(stem)
+      decoded.empty? ? "playlist" : decoded
+    end
+  end
+end

package/native-src/src/rill/extractors/media_ccc_de.cr

@@ -0,0 +1,219 @@
+require "json"
+require "uri"
+
+require "../errors"
+require "../extractor"
+require "../format"
+require "../http_url"
+require "../json_value"
+require "../media"
+require "../mime_type"
+
+module Rill::Extractors
+  class MediaCccDe < Extractor
+    DEFAULT_API_BASE = "https://media.ccc.de"
+    EVENT_PATH       = %r{\A/v/([^/]+)/?\z}
+    CONFERENCE_PATH  = %r{\A/(?:c|conferences)/([^/]+)/?\z}
+
+    def initialize(@api_base : String = DEFAULT_API_BASE)
+    end
+
+    def name : String
+      "media_ccc_de"
+    end
+
+    def matches?(url : URI) : Bool
+      return false unless url.scheme == "http" || url.scheme == "https"
+      return false unless host?(url.host)
+
+      path = url.path
+      !EVENT_PATH.match(path).nil? || !CONFERENCE_PATH.match(path).nil?
+    end
+
+    def extract(url : URI, client : HTTPClient) : Media | Playlist
+      if slug = slug_for(url, EVENT_PATH)
+        extract_event(url, client, slug)
+      elsif acronym = slug_for(url, CONFERENCE_PATH)
+        extract_conference(url, client, acronym)
+      else
+        raise UnsupportedUrlError.new(url.to_s)
+      end
+    end
+
+    private def extract_event(url : URI, client : HTTPClient, slug : String) : Media
+      api_url = URI.parse("#{@api_base}/public/events/#{URI.encode_path_segment(slug)}")
+      payload = api_payload(client, api_url, url, "event")
+
+      formats = formats_from(payload["recordings"]?)
+      if formats.empty?
+        raise MediaUnavailableError.new(url.to_s, "event has no downloadable recordings")
+      end
+
+      Media.new(
+        id: JsonValue.string(payload["guid"]?),
+        title: JsonValue.string(payload["title"]?) || slug,
+        description: JsonValue.string(payload["description"]?),
+        webpage_url: url,
+        uploader: persons(payload["persons"]?),
+        duration_seconds: JsonValue.seconds(payload["duration"]?),
+        thumbnail_url: thumbnail_url(payload),
+        upload_date: date_value(payload),
+        extractor: name,
+        formats: formats,
+      )
+    end
+
+    private def extract_conference(
+      url : URI,
+      client : HTTPClient,
+      acronym : String,
+    ) : Playlist
+      api_url = URI.parse("#{@api_base}/public/conferences/#{URI.encode_path_segment(acronym)}")
+      payload = api_payload(client, api_url, url, "conference")
+
+      entries = [] of PlaylistEntry
+      events = payload["events"]?.try(&.as_a?) || [] of JSON::Any
+
+      events.each do |event|
+        event_hash = event.as_h?
+        next unless event_hash
+
+        entry_url = event_url(event_hash)
+        next unless entry_url
+
+        entries << PlaylistEntry.new(
+          url: entry_url,
+          title: JsonValue.string(event_hash["title"]?),
+          id: JsonValue.string(event_hash["guid"]?),
+        )
+      end
+
+      if entries.empty?
+        raise MediaUnavailableError.new(url.to_s, "conference has no events")
+      end
+
+      Playlist.new(
+        id: acronym,
+        title: JsonValue.string(payload["title"]?) || acronym,
+        webpage_url: url,
+        extractor: name,
+        entries: entries,
+      )
+    end
+
+    private def api_payload(
+      client : HTTPClient,
+      api_url : URI,
+      url : URI,
+      subject : String,
+    ) : Hash(String, JSON::Any)
+      response = client.get(api_url)
+      parse_payload(response.body, url)
+    rescue ex : HTTPError
+      if ex.status.code == 404
+        raise MediaUnavailableError.new(url.to_s, "#{subject} not found")
+      end
+      raise ex
+    end
+
+    private def host?(host : String?) : Bool
+      case host.try(&.downcase)
+      when "media.ccc.de", "www.media.ccc.de"
+        true
+      else
+        false
+      end
+    end
+
+    private def slug_for(url : URI, pattern : Regex) : String?
+      match = pattern.match(url.path)
+      return nil unless match
+
+      slug = URI.decode(match[1])
+      slug.empty? ? nil : slug
+    end
+
+    private def parse_payload(body : String, url : URI) : Hash(String, JSON::Any)
+      payload = JSON.parse(body).as_h?
+      unless payload
+        raise MediaUnavailableError.new(url.to_s, "metadata is not a JSON object")
+      end
+
+      payload
+    rescue JSON::ParseException
+      raise MediaUnavailableError.new(url.to_s, "metadata is not valid JSON")
+    end
+
+    private def formats_from(recordings_value : JSON::Any?) : Array(Format)
+      recordings = recordings_value.try(&.as_a?)
+      return [] of Format unless recordings
+
+      formats = [] of Format
+
+      recordings.each do |recording|
+        recording_hash = recording.as_h?
+        next unless recording_hash
+
+        filename = JsonValue.string(recording_hash["filename"]?)
+        next unless filename
+
+        recording_url = HttpUrl.absolute(JsonValue.string(recording_hash["recording_url"]?))
+        next unless recording_url
+
+        mime_type = JsonValue.string(recording_hash["mime_type"]?) || ""
+        mime = MimeType.parse(mime_type)
+        next if mime.empty?
+
+        audio = mime.audio?
+        formats << Format.new(
+          format_id: filename,
+          url: recording_url,
+          kind: audio ? FormatKind::Audio : FormatKind::Combined,
+          container: mime.container,
+          video_codec: audio ? nil : mime.codecs,
+          audio_codec: audio ? mime.audio_codec : nil,
+          width: JsonValue.dimension(recording_hash["width"]?),
+          height: JsonValue.dimension(recording_hash["height"]?),
+          language: JsonValue.string(recording_hash["language"]?),
+          note: mime_type,
+        )
+      end
+
+      formats
+    end
+
+    private def persons(value : JSON::Any?) : String?
+      array = value.try(&.as_a?)
+      return nil unless array
+
+      names = array.compact_map(&.as_s?)
+      names.empty? ? nil : names.join(", ")
+    end
+
+    private def event_url(event : Hash(String, JSON::Any)) : URI?
+      value = JsonValue.string(event["frontend_link"]?) || JsonValue.string(event["link"]?)
+
+      if value.nil?
+        slug = JsonValue.string(event["slug"]?)
+        value = "#{DEFAULT_API_BASE}/v/#{URI.encode_path_segment(slug)}" if slug
+      end
+
+      return nil unless value
+      value = value.sub("/public/events/", "/v/")
+      HttpUrl.absolute(value)
+    end
+
+    private def thumbnail_url(payload : Hash(String, JSON::Any)) : URI?
+      value = JsonValue.string(payload["thumb_url"]?) || JsonValue.string(payload["poster_url"]?)
+      value ? HttpUrl.absolute(value) : nil
+    end
+
+    private def date_value(payload : Hash(String, JSON::Any)) : String?
+      value = JsonValue.string(payload["date"]?)
+      return nil unless value
+      return nil if value.size < 10
+
+      value[0, 10]
+    end
+  end
+end