rill-node 0.0.0-stage → 2026.11.14

package/native-src/src/rill/c_api/worker_pool.cr

@@ -0,0 +1,403 @@
+require "../cancellation"
+require "../client"
+require "../download_options"
+require "../download_plan"
+require "../download_progress"
+require "../errors"
+require "../extractor_registry"
+require "../format_selector"
+require "../media"
+require "../request_options"
+require "./job"
+require "./memory"
+require "./status"
+require "./sync"
+
+module Rill::CAPI::WorkerPool
+  extend self
+
+  MAX_WORKERS = 64
+
+  @@pool : Pool?
+  @@shutting_down = Atomic(Bool).new(false)
+
+  def start(worker_count : Int32) : Nil
+    @@shutting_down.set(false)
+    pool = Pool.new(worker_count)
+    @@pool = pool
+    pool.start
+  end
+
+  def shutting_down? : Bool
+    @@shutting_down.get
+  end
+
+  def create_job(url : UInt8*, options_json : UInt8*) : JobPtr
+    job = Job.create_extraction(url, options_json)
+    return job if job.null? || Job.done?(job)
+
+    submit(job)
+    job
+  end
+
+  def create_download_job(url : UInt8*, output_path : UInt8*, options_json : UInt8*) : JobPtr
+    job = Job.create_download(url, output_path, options_json)
+    return job if job.null? || Job.done?(job)
+
+    submit(job)
+    job
+  end
+
+  private def submit(job : JobPtr) : Nil
+    pool = @@pool
+    if pool
+      pool.submit(job)
+    else
+      finish_shutdown(job)
+    end
+  end
+
+  def finish_shutdown(job : JobPtr) : Nil
+    if job.value.kind == Rill::CAPI::Kind::DOWNLOAD
+      Job.finish_download(
+        job,
+        Status::SHUTDOWN,
+        Rill::CAPI::DownloadState::CANCELLED,
+        Pointer(UInt8).null,
+        Pointer(UInt8).null,
+      )
+    else
+      Job.finish(
+        job,
+        Status::SHUTDOWN,
+        Pointer(UInt8).null,
+        Pointer(UInt8).null,
+        Pointer(UInt8).null,
+      )
+    end
+  end
+
+  def shutdown : Nil
+    @@shutting_down.set(true)
+    pool = @@pool
+    @@pool = nil
+    pool.try(&.shutdown)
+  end
+
+  class Pool
+    @head : JobPtr
+    @tail : JobPtr
+    @mutex : Sync::Mutex*
+    @cond : Sync::Cond*
+    @done_cond : Sync::Cond*
+
+    def initialize(@worker_count : Int32)
+      @head = JobPtr.null
+      @tail = JobPtr.null
+      @shutdown = false
+      @running = @worker_count
+      @workers = [] of Fiber::ExecutionContext::Isolated
+      @mutex = allocate_mutex
+      @cond = allocate_cond
+      @done_cond = allocate_cond
+
+      if @mutex.null? || @cond.null? || @done_cond.null?
+        raise "failed to allocate sync primitives"
+      end
+    end
+
+    def start : Nil
+      @worker_count.times do |index|
+        @workers << Fiber::ExecutionContext::Isolated.new("rill-worker-#{index}") { work }
+      end
+    end
+
+    def submit(job : JobPtr) : Nil
+      lock
+
+      unless @shutdown
+        enqueue(job)
+        Sync.signal(@cond)
+        unlock
+        return
+      end
+
+      unlock
+      WorkerPool.finish_shutdown(job)
+    end
+
+    def shutdown : Nil
+      lock
+
+      unless @shutdown
+        @shutdown = true
+        Sync.broadcast(@cond)
+        fail_pending
+      end
+
+      while @running > 0
+        Sync.wait(@done_cond, @mutex)
+      end
+
+      unlock
+      destroy_sync
+    end
+
+    private def work : Nil
+      loop do
+        job = pop
+        break if job.null?
+
+        begin
+          process(job)
+        rescue ex
+          finish_error(job, ex)
+        end
+      end
+    ensure
+      worker_finished
+    end
+
+    private def pop : JobPtr
+      lock
+
+      while @head.null? && !@shutdown
+        Sync.wait(@cond, @mutex)
+      end
+
+      job = @head
+      unless job.null?
+        dequeue
+      end
+
+      unlock
+      job
+    end
+
+    private def process(job : JobPtr) : Nil
+      if job.value.kind == Rill::CAPI::Kind::DOWNLOAD
+        process_download(job)
+      else
+        process_extraction(job)
+      end
+    end
+
+    private def process_extraction(job : JobPtr) : Nil
+      options = options_for(job)
+      media = ExtractorRegistry.default.extract(String.new(job.value.url), options)
+      Job.finish(
+        job,
+        Status::OK,
+        Memory.copy_string(media.to_pretty_json),
+        Pointer(UInt8).null,
+        Pointer(UInt8).null,
+      )
+    rescue ex
+      finish_error(job, ex)
+    end
+
+    private def process_download(job : JobPtr) : Nil
+      Job.mark_running(job)
+      cancellation = Cancellation.watch do
+        job.value.cancel_requested != 0 || WorkerPool.shutting_down?
+      end
+      cancellation.check!
+
+      options = DownloadOptions.from_json(optional_string(job.value.options_json))
+      client = Client.new(options.request)
+      result = client.extract(String.new(job.value.url))
+
+      unless result.is_a?(Media)
+        raise DownloadError.new("URL is a playlist")
+      end
+
+      plan = FormatSelector.plan(result.formats, options.format_id, options.preference)
+      unless plan
+        raise DownloadError.new("no downloadable formats")
+      end
+
+      progress = ->(value : DownloadProgress) do
+        Job.update_progress(job, value.downloaded_bytes, value.total_bytes)
+      end
+
+      client.download(
+        plan,
+        String.new(job.value.output_path),
+        cancellation,
+        progress,
+        resume: options.resume,
+        overwrite: options.overwrite,
+        media_backend: options.media_backend,
+      )
+
+      Job.finish_download(
+        job,
+        Status::OK,
+        Rill::CAPI::DownloadState::COMPLETED,
+        Pointer(UInt8).null,
+        Pointer(UInt8).null,
+      )
+    rescue ex : CancelledError
+      Job.finish_download(
+        job,
+        Status::CANCELLED,
+        Rill::CAPI::DownloadState::CANCELLED,
+        Memory.copy_string("cancelled"),
+        Memory.copy_string("cancelled"),
+      )
+    rescue ex
+      Job.finish_download(
+        job,
+        status_for(ex),
+        Rill::CAPI::DownloadState::FAILED,
+        Memory.copy_string(ex.message || ex.class.name),
+        Memory.copy_string(error_code_for(ex)),
+      )
+    end
+
+    private def optional_string(pointer : UInt8*) : String?
+      pointer.null? ? nil : String.new(pointer)
+    end
+
+    private def options_for(job : JobPtr) : RequestOptions
+      if job.value.options_json.null?
+        RequestOptions.new
+      else
+        RequestOptions.from_json(String.new(job.value.options_json))
+      end
+    end
+
+    private def finish_error(job : JobPtr, error : Exception) : Nil
+      message = error.message || error.class.name
+
+      if job.value.kind == Rill::CAPI::Kind::DOWNLOAD
+        Job.finish_download(
+          job,
+          status_for(error),
+          Rill::CAPI::DownloadState::FAILED,
+          Memory.copy_string(message),
+          Memory.copy_string(error_code_for(error)),
+        )
+      else
+        Job.finish(
+          job,
+          status_for(error),
+          Pointer(UInt8).null,
+          Memory.copy_string(message),
+          Memory.copy_string(error_code_for(error)),
+        )
+      end
+    end
+
+    private def status_for(error : Exception) : Int32
+      case error
+      when Rill::CancelledError
+        Status::CANCELLED
+      when Rill::InvalidOptionsError, Rill::InvalidUrlError, ArgumentError
+        Status::INVALID_ARGUMENT
+      when Rill::Error
+        Status::EXTRACTION_ERROR
+      else
+        Status::INTERNAL_ERROR
+      end
+    end
+
+    private def error_code_for(error : Exception) : String
+      case error
+      when Rill::UnsupportedUrlError
+        "unsupported_url"
+      when Rill::MediaUnavailableError
+        "media_unavailable"
+      when Rill::InvalidExtractionError
+        "invalid_extraction"
+      when Rill::HTTPError
+        "http_error"
+      when Rill::InvalidUrlError
+        "invalid_url"
+      when Rill::ResponseTooLargeError
+        "response_too_large"
+      when Rill::TooManyRedirectsError
+        "too_many_redirects"
+      when Rill::NetworkError
+        "network_error"
+      when Rill::InvalidOptionsError, ArgumentError
+        "invalid_options"
+      when Rill::MediaBackendUnavailableError
+        "media_backend_unavailable"
+      when Rill::MediaMuxError
+        "media_mux_error"
+      when Rill::DownloadError
+        "download_error"
+      else
+        "internal"
+      end
+    end
+
+    private def enqueue(job : JobPtr) : Nil
+      job.value.next_job = JobPtr.null
+
+      if @tail.null?
+        @head = job
+      else
+        @tail.value.next_job = job
+      end
+
+      @tail = job
+    end
+
+    private def dequeue : Nil
+      job = @head
+      return if job.null?
+
+      @head = job.value.next_job
+      @tail = JobPtr.null if @head.null?
+    end
+
+    private def fail_pending : Nil
+      loop do
+        job = @head
+        break if job.null?
+
+        dequeue
+
+        Sync.lock(job.value.mutex)
+        job.value.status = Status::SHUTDOWN
+        if job.value.kind == Rill::CAPI::Kind::DOWNLOAD
+          job.value.download_state = Rill::CAPI::DownloadState::CANCELLED
+        end
+        job.value.done = 1
+        Sync.signal(job.value.cond)
+        Sync.unlock(job.value.mutex)
+      end
+    end
+
+    private def worker_finished : Nil
+      lock
+      @running -= 1
+      Sync.broadcast(@done_cond)
+      unlock
+    end
+
+    private def lock : Nil
+      Sync.lock(@mutex)
+    end
+
+    private def unlock : Nil
+      Sync.unlock(@mutex)
+    end
+
+    private def allocate_mutex : Sync::Mutex*
+      Sync.create_mutex
+    end
+
+    private def allocate_cond : Sync::Cond*
+      Sync.create_cond
+    end
+
+    private def destroy_sync : Nil
+      Sync.destroy_cond(@done_cond)
+      Sync.destroy_cond(@cond)
+      Sync.destroy_mutex(@mutex)
+    end
+  end
+end

package/native-src/src/rill/cancellation.cr

@@ -0,0 +1,29 @@
+require "./errors"
+
+module Rill
+  class Cancellation
+    @cancelled : Atomic(Bool)
+    @checker : Proc(Bool)?
+
+    def initialize(@checker : Proc(Bool)? = nil)
+      @cancelled = Atomic(Bool).new(false)
+    end
+
+    def self.watch(&block : -> Bool) : self
+      new(block)
+    end
+
+    def cancel! : Nil
+      @cancelled.set(true)
+    end
+
+    def cancelled? : Bool
+      checker = @checker
+      checker ? checker.call : @cancelled.get
+    end
+
+    def check! : Nil
+      raise CancelledError.new if cancelled?
+    end
+  end
+end

package/native-src/src/rill/client.cr

@@ -0,0 +1,52 @@
+require "path"
+require "uri"
+
+require "./cancellation"
+require "./downloader"
+require "./download_plan"
+require "./download_progress"
+require "./extractor_registry"
+require "./format"
+require "./http_client"
+require "./media"
+require "./media_muxer"
+require "./playlist"
+require "./request_options"
+
+module Rill
+  class Client
+    getter options : RequestOptions
+    getter http : HTTPClient
+    getter registry : ExtractorRegistry
+
+    def initialize(
+      @options : RequestOptions = RequestOptions.new,
+      @registry : ExtractorRegistry = ExtractorRegistry.default,
+    )
+      @http = HTTPClient.new(@options)
+    end
+
+    def extract(url : String | URI) : Media | Playlist
+      @registry.extract(url, @http)
+    end
+
+    def download(
+      source : DownloadPlan | Format | String | URI,
+      target : String | Path,
+      cancellation : Cancellation? = nil,
+      on_progress : Proc(DownloadProgress, Nil)? = nil,
+      resume : Bool = true,
+      overwrite : Bool = false,
+      media_backend : MediaMuxer::Backend = MediaMuxer.backend_from_environment,
+    ) : Path
+      downloader = Downloader.new(
+        @options,
+        @http,
+        resume: resume,
+        overwrite: overwrite,
+        media_muxer: MediaMuxer.new(media_backend),
+      )
+      downloader.download(source, target, cancellation, on_progress)
+    end
+  end
+end

package/native-src/src/rill/cookie_jar.cr

@@ -0,0 +1,130 @@
+require "http/cookie"
+require "uri"
+
+module Rill
+  class CookieJar
+    private struct StoredCookie
+      getter cookie : HTTP::Cookie
+      getter host_only : Bool
+
+      def initialize(@cookie : HTTP::Cookie, @host_only : Bool)
+      end
+    end
+
+    def initialize
+      @cookies = [] of StoredCookie
+    end
+
+    def add(headers : HTTP::Headers, url : URI) : Nil
+      values = headers.get?("Set-Cookie")
+      return if values.nil? || values.empty?
+
+      host = url.host.try(&.downcase)
+      return unless host
+
+      values.each do |value|
+        cookie = parse_cookie(value)
+        next unless cookie
+        next unless cookie.valid?
+
+        domain = normalized_domain(cookie.domain)
+        if domain
+          next unless settable_domain?(host, domain)
+
+          cookie.domain = domain
+          host_only = false
+        else
+          cookie.domain = host
+          host_only = true
+        end
+
+        cookie.path ||= default_path(url.path)
+
+        if cookie.expired?
+          delete_matching(cookie)
+          next
+        end
+
+        store(StoredCookie.new(cookie, host_only))
+      end
+    end
+
+    def cookies_for(url : URI) : Array(HTTP::Cookie)
+      now = Time.utc
+      @cookies.reject!(&.cookie.expired?(now))
+      @cookies.select { |stored| matches?(stored, url) }.map(&.cookie)
+    end
+
+    private def parse_cookie(value : String) : HTTP::Cookie?
+      HTTP::Cookie::Parser.parse_set_cookie(value)
+    rescue
+      nil
+    end
+
+    private def store(stored : StoredCookie) : Nil
+      cookie = stored.cookie
+      delete_matching(cookie)
+      @cookies << stored
+    end
+
+    private def delete_matching(cookie : HTTP::Cookie) : Nil
+      @cookies.reject! do |existing|
+        existing.cookie.name == cookie.name &&
+          existing.cookie.domain.try(&.downcase) == cookie.domain.try(&.downcase) &&
+          existing.cookie.path == cookie.path
+      end
+    end
+
+    private def matches?(stored : StoredCookie, url : URI) : Bool
+      host = url.host.try(&.downcase)
+      return false unless host
+
+      cookie = stored.cookie
+      return false unless domain_matches?(host, cookie.domain, stored.host_only)
+      return false if cookie.secure && url.scheme != "https"
+
+      path_match?(url.path, cookie.path || "/")
+    end
+
+    private def domain_matches?(host : String, domain : String?, host_only : Bool) : Bool
+      return false unless domain
+
+      normalized = normalized_domain(domain)
+      return false unless normalized
+      return host == normalized if host_only
+
+      host == normalized || host.ends_with?(".#{normalized}")
+    end
+
+    private def settable_domain?(host : String, domain : String) : Bool
+      return true if host == domain
+      return false unless host.ends_with?(".#{domain}")
+      return false unless domain.includes?('.')
+
+      true
+    end
+
+    private def normalized_domain(domain : String?) : String?
+      return nil unless domain
+
+      value = domain.strip.downcase.lchop('.')
+      value.empty? ? nil : value
+    end
+
+    private def path_match?(request_path : String, cookie_path : String) : Bool
+      return true if request_path == cookie_path
+      return true if request_path.starts_with?(cookie_path) && cookie_path.ends_with?('/')
+
+      request_path.starts_with?("#{cookie_path}/")
+    end
+
+    private def default_path(path : String) : String
+      return "/" unless path.starts_with?('/')
+
+      index = path.rindex('/')
+      return "/" unless index && index > 0
+
+      path[0, index]
+    end
+  end
+end