redc 0.1.1.dev1__cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl

redc/codes.py

@@ -0,0 +1,151 @@
+status_descriptions = {
+    # Informational (1xx)
+    100: "Continue",
+    101: "Switching Protocols",
+    102: "Processing",
+    103: "Early Hints",
+    # Success (2xx)
+    200: "OK",
+    201: "Created",
+    202: "Accepted",
+    203: "Non-Authoritative Information",
+    204: "No Content",
+    205: "Reset Content",
+    206: "Partial Content",
+    207: "Multi-Status",
+    208: "Already Reported",
+    226: "IM Used",
+    # Redirection (3xx)
+    300: "Multiple Choices",
+    301: "Moved Permanently",
+    302: "Found",
+    303: "See Other",
+    304: "Not Modified",
+    305: "Use Proxy",
+    307: "Temporary Redirect",
+    308: "Permanent Redirect",
+    # Client Errors (4xx)
+    400: "Bad Request",
+    401: "Unauthorized",
+    402: "Payment Required",
+    403: "Forbidden",
+    404: "Not Found",
+    405: "Method Not Allowed",
+    406: "Not Acceptable",
+    407: "Proxy Authentication Required",
+    408: "Request Timeout",
+    409: "Conflict",
+    410: "Gone",
+    411: "Length Required",
+    412: "Precondition Failed",
+    413: "Payload Too Large",
+    414: "URI Too Long",
+    415: "Unsupported Media Type",
+    416: "Range Not Satisfiable",
+    417: "Expectation Failed",
+    418: "I'm a Teapot",
+    421: "Misdirected Request",
+    422: "Unprocessable Entity",
+    423: "Locked",
+    424: "Failed Dependency",
+    425: "Too Early",
+    426: "Upgrade Required",
+    428: "Precondition Required",
+    429: "Too Many Requests",
+    431: "Request Header Fields Too Large",
+    451: "Unavailable For Legal Reasons",
+    # Server Errors (5xx)
+    500: "Internal Server Error",
+    501: "Not Implemented",
+    502: "Bad Gateway",
+    503: "Service Unavailable",
+    504: "Gateway Timeout",
+    505: "HTTP Version Not Supported",
+    506: "Variant Also Negotiates",
+    507: "Insufficient Storage",
+    508: "Loop Detected",
+    510: "Not Extended",
+    511: "Network Authentication Required",
+}
+
+
+class HTTPStatus:
+    # Informational (1xx)
+    CONTINUE = 100
+    SWITCHING_PROTOCOLS = 101
+    PROCESSING = 102
+    EARLY_HINTS = 103
+
+    # Success (2xx)
+    OK = 200
+    CREATED = 201
+    ACCEPTED = 202
+    NON_AUTHORITATIVE_INFORMATION = 203
+    NO_CONTENT = 204
+    RESET_CONTENT = 205
+    PARTIAL_CONTENT = 206
+    MULTI_STATUS = 207
+    ALREADY_REPORTED = 208
+    IM_USED = 226
+
+    # Redirection (3xx)
+    MULTIPLE_CHOICES = 300
+    MOVED_PERMANENTLY = 301
+    FOUND = 302
+    SEE_OTHER = 303
+    NOT_MODIFIED = 304
+    USE_PROXY = 305
+    TEMPORARY_REDIRECT = 307
+    PERMANENT_REDIRECT = 308
+
+    # Client Errors (4xx)
+    BAD_REQUEST = 400
+    UNAUTHORIZED = 401
+    PAYMENT_REQUIRED = 402
+    FORBIDDEN = 403
+    NOT_FOUND = 404
+    METHOD_NOT_ALLOWED = 405
+    NOT_ACCEPTABLE = 406
+    PROXY_AUTHENTICATION_REQUIRED = 407
+    REQUEST_TIMEOUT = 408
+    CONFLICT = 409
+    GONE = 410
+    LENGTH_REQUIRED = 411
+    PRECONDITION_FAILED = 412
+    PAYLOAD_TOO_LARGE = 413
+    URI_TOO_LONG = 414
+    UNSUPPORTED_MEDIA_TYPE = 415
+    RANGE_NOT_SATISFIABLE = 416
+    EXPECTATION_FAILED = 417
+    IM_A_TEAPOT = 418
+    MISDIRECTED_REQUEST = 421
+    UNPROCESSABLE_ENTITY = 422
+    LOCKED = 423
+    FAILED_DEPENDENCY = 424
+    TOO_EARLY = 425
+    UPGRADE_REQUIRED = 426
+    PRECONDITION_REQUIRED = 428
+    TOO_MANY_REQUESTS = 429
+    REQUEST_HEADER_FIELDS_TOO_LARGE = 431
+    UNAVAILABLE_FOR_LEGAL_REASONS = 451
+
+    # Server Errors (5xx)
+    INTERNAL_SERVER_ERROR = 500
+    NOT_IMPLEMENTED = 501
+    BAD_GATEWAY = 502
+    SERVICE_UNAVAILABLE = 503
+    GATEWAY_TIMEOUT = 504
+    HTTP_VERSION_NOT_SUPPORTED = 505
+    VARIANT_ALSO_NEGOTIATES = 506
+    INSUFFICIENT_STORAGE = 507
+    LOOP_DETECTED = 508
+    NOT_EXTENDED = 510
+    NETWORK_AUTHENTICATION_REQUIRED = 511
+
+    @staticmethod
+    def get_description(status_code: int) -> str:
+        """
+        Get the description of a status code
+        """
+
+        return status_descriptions.get(status_code, "")

redc/exceptions/__init__.py

@@ -0,0 +1,22 @@
+from ..codes import HTTPStatus
+
+
+class HTTPError(Exception):
+    """Exception raised for HTTP and CURL-related errors"""
+
+    def __init__(self, status_code: int, curl_error_code: int, curl_error_message: str):
+        self.status_code = status_code
+        self.curl_error_code = curl_error_code
+        self.curl_error_message = curl_error_message
+        self.is_curl_error = status_code == -1
+
+        if self.is_curl_error:
+            super().__init__(f"CURL {self.curl_error_code}: {self.curl_error_message}")
+        else:
+            short_description = HTTPStatus.get_description(self.status_code)
+
+            super().__init__(
+                self.status_code
+                if not short_description
+                else f"{self.status_code} - {short_description}"
+            )

redc/ext/redc.cpp

@@ -0,0 +1,362 @@
+#include "redc.h"
+#include "utils/curl_utils.h"
+#include <iostream>
+#include <stdexcept>
+
+RedC::RedC(const long &buffer) {
+  {
+    acq_gil gil;
+    loop_ = nb::module_::import_("asyncio").attr("get_event_loop")();
+    call_soon_threadsafe_ = loop_.attr("call_soon_threadsafe");
+  }
+
+  static CurlGlobalInit g;
+
+  buffer_size_ = buffer;
+  multi_handle_ = curl_multi_init();
+
+  if (!multi_handle_) {
+    throw std::runtime_error("Failed to create CURL multi handle");
+  }
+
+  try {
+    running_ = true;
+    worker_thread_ = std::thread(&RedC::worker_loop, this);
+  } catch (...) {
+    curl_multi_cleanup(multi_handle_);
+    throw;
+  }
+}
+
+RedC::~RedC() {
+  this->close();
+}
+
+bool RedC::is_running() {
+  return running_;
+}
+
+void RedC::close() {
+  if (running_) {
+    running_ = false;
+
+    if (worker_thread_.joinable()) {
+      curl_multi_wakeup(multi_handle_);
+      worker_thread_.join();
+    }
+
+    cleanup();
+
+    curl_multi_cleanup(multi_handle_);
+  }
+}
+
+py_object RedC::request(const char *method, const char *url, const char *raw_data, const py_object &data,
+                        const py_object &files, const py_object &headers, const long &timeout_ms,
+                        const long &connect_timeout_ms, const bool &allow_redirect, const char *proxy_url,
+                        const bool &verify, const char *ca_cert_path, const py_object &stream_callback,
+                        const py_object &progress_callback, const bool &verbose) {
+  CHECK_RUNNING();
+
+  if (isNullOrEmpty(method) || isNullOrEmpty(url)) {
+    throw std::invalid_argument("method or url must be non-empty");
+  }
+
+  CURL *easy = curl_easy_init();
+  if (!easy) {
+    throw std::runtime_error("Failed to create CURL easy handle");
+  }
+
+  bool is_nobody = (strcmp(method, "HEAD") == 0 || strcmp(method, "OPTIONS") == 0);
+
+  try {
+    curl_easy_setopt(easy, CURLOPT_BUFFERSIZE, buffer_size_);
+    curl_easy_setopt(easy, CURLOPT_URL, url);
+    curl_easy_setopt(easy, CURLOPT_CUSTOMREQUEST, method);
+    curl_easy_setopt(easy, CURLOPT_NOSIGNAL, 1L);
+
+    curl_easy_setopt(easy, CURLOPT_TIMEOUT_MS, timeout_ms);
+
+    curl_easy_setopt(easy, CURLOPT_HEADERFUNCTION, &RedC::header_callback);
+
+    if (verbose) {
+      curl_easy_setopt(easy, CURLOPT_VERBOSE, 1L);
+    }
+
+    if (connect_timeout_ms > 0) {
+      curl_easy_setopt(easy, CURLOPT_CONNECTTIMEOUT_MS, connect_timeout_ms);
+    }
+
+    if (is_nobody) {
+      curl_easy_setopt(easy, CURLOPT_NOBODY, 1L);
+    } else {
+      curl_easy_setopt(easy, CURLOPT_WRITEFUNCTION, &RedC::write_callback);
+    }
+
+    if (allow_redirect) {
+      curl_easy_setopt(easy, CURLOPT_FOLLOWLOCATION, 1L);
+      curl_easy_setopt(easy, CURLOPT_MAXREDIRS, 30L);
+    }
+
+    if (!isNullOrEmpty(proxy_url)) {
+      curl_easy_setopt(easy, CURLOPT_PROXY, proxy_url);
+    }
+
+    if (!verify) {
+      curl_easy_setopt(easy, CURLOPT_SSL_VERIFYPEER, 0);
+      curl_easy_setopt(easy, CURLOPT_SSL_VERIFYHOST, 0);
+    } else if (!isNullOrEmpty(ca_cert_path)) {
+      curl_easy_setopt(easy, CURLOPT_CAINFO, ca_cert_path);
+    }
+
+    CurlMime curl_mime_;
+    if (!isNullOrEmpty(raw_data)) {
+      curl_easy_setopt(easy, CURLOPT_POSTFIELDS, raw_data);
+      curl_easy_setopt(easy, CURLOPT_POSTFIELDSIZE_LARGE, (curl_off_t)strlen(raw_data));
+    } else if (!data.is_none() || !files.is_none()) {
+      curl_mime_.mime = curl_mime_init(easy);
+
+      if (!data.is_none()) {
+        dict dict_obj;
+        try {
+          dict_obj = nb::cast<dict>(data);
+        } catch (...) {
+          throw std::runtime_error("Expected \"data\" to be a dictionary of strings");
+        }
+
+        for (auto const &it : dict_obj) {
+          curl_mimepart *part = curl_mime_addpart(curl_mime_.mime);
+          curl_mime_name(part, nb::str(it.first).c_str());
+          curl_mime_data(part, nb::str(it.second).c_str(), CURL_ZERO_TERMINATED);
+        }
+      }
+
+      if (!files.is_none()) {
+        dict dict_obj;
+        try {
+          dict_obj = nb::cast<dict>(files);
+        } catch (...) {
+          throw std::runtime_error("Expected \"files\" to be a dictionary of strings");
+        }
+
+        for (auto const &it : dict_obj) {
+          curl_mimepart *part = curl_mime_addpart(curl_mime_.mime);
+          curl_mime_name(part, nb::str(it.first).c_str());
+          curl_mime_filedata(part, nb::str(it.second).c_str());
+        }
+      }
+
+      curl_easy_setopt(easy, CURLOPT_MIMEPOST, curl_mime_.mime);
+    }
+
+    CurlSlist slist_headers;
+    if (!headers.is_none()) {
+      for (auto const &it : headers) {
+        slist_headers.slist = curl_slist_append(slist_headers.slist, nb::str(it).c_str());
+      }
+      curl_easy_setopt(easy, CURLOPT_HTTPHEADER, slist_headers.slist);
+    }
+
+    py_object future;
+    {
+      acq_gil gil;
+      future = loop_.attr("create_future")();
+    }
+
+    {
+      std::unique_lock<std::mutex> lock(mutex_);
+      auto &d = transfers_[easy];
+      lock.unlock();
+
+      d.future = future;
+      d.request_headers = std::move(slist_headers);
+      d.curl_mime_ = std::move(curl_mime_);
+
+      curl_easy_setopt(easy, CURLOPT_HEADERDATA, &d);
+      if (!is_nobody) {
+        curl_easy_setopt(easy, CURLOPT_WRITEDATA, &d);
+
+        if (!stream_callback.is_none()) {
+          d.stream_callback = stream_callback;
+          d.has_stream_callback = true;
+        }
+
+        if (!progress_callback.is_none()) {
+          d.progress_callback = progress_callback;
+          d.has_progress_callback = true;
+
+          curl_easy_setopt(easy, CURLOPT_XFERINFODATA, &d);
+          curl_easy_setopt(easy, CURLOPT_NOPROGRESS, 0L);
+          curl_easy_setopt(easy, CURLOPT_XFERINFOFUNCTION, &RedC::progress_callback);
+        } else {
+          curl_easy_setopt(easy, CURLOPT_NOPROGRESS, 1L);
+        }
+      }
+    }
+
+    queue_.enqueue(easy);
+
+    curl_multi_wakeup(multi_handle_);  // thread-safe
+    return future;
+  } catch (...) {
+    curl_easy_cleanup(easy);
+    throw;
+  }
+}
+
+void RedC::worker_loop() {
+  while (running_) {
+    CURL *e;
+    if (queue_.try_dequeue(e)) {
+      CURLMcode res = curl_multi_add_handle(multi_handle_, e);
+      if (res != CURLM_OK) {
+        std::unique_lock<std::mutex> lock(mutex_);
+        auto it = transfers_.find(e);
+        if (it != transfers_.end()) {
+          Data data = std::move(it->second);
+          transfers_.erase(it);
+          lock.unlock();
+          {
+            acq_gil gil;
+            call_soon_threadsafe_(nb::cpp_function([data = std::move(data), res]() {
+              data.future.attr("set_result")(nb::make_tuple(-1, NULL, NULL, (int)res, curl_multi_strerror(res)));
+            }));
+          }
+        }
+        curl_easy_cleanup(e);
+      }
+    } else {
+      int numfds;
+      curl_multi_poll(multi_handle_, nullptr, 0, 30000, &numfds);
+    }
+
+    if (!running_) {
+      return;
+    }
+
+    curl_multi_perform(multi_handle_, &still_running_);
+
+    CURLMsg *msg;
+    int msgs_left;
+    while ((msg = curl_multi_info_read(multi_handle_, &msgs_left))) {
+      if (msg->msg == CURLMSG_DONE) {
+        std::unique_lock<std::mutex> lock(mutex_);
+        auto it = transfers_.find(msg->easy_handle);
+        if (it != transfers_.end()) {
+          Data data = std::move(it->second);
+          transfers_.erase(it);
+          lock.unlock();
+
+          {
+            acq_gil gil;
+
+            CURLcode res = msg->data.result;
+
+            /*
+            * Result is allways Tuple:
+
+            * 0: HTTP response status code.
+            *    If the value is -1, it indicates a cURL error occurred
+            *
+            * 1: Response headers as bytes; can be null
+            *
+            * 2: The actual response data as bytes; can be null
+            *
+            * 3: cURL return code. This indicates the result code of the cURL operation.
+            *    See: https://curl.se/libcurl/c/libcurl-errors.html
+            *
+            * 4: cURL error message string; can be null
+            */
+            py_object result;
+            if (res == CURLE_OK) {
+              short status_code = 0;
+              curl_easy_getinfo(msg->easy_handle, CURLINFO_RESPONSE_CODE, &status_code);
+              result = nb::make_tuple(status_code, py_bytes(data.headers.data(), data.headers.size()),
+                                      py_bytes(data.response.data(), data.response.size()), (int)res, NULL);
+            } else {
+              result = nb::make_tuple(-1, NULL, NULL, (int)res, curl_easy_strerror(res));
+            }
+
+            call_soon_threadsafe_(nb::cpp_function([data = std::move(data), result = std::move(result)]() {
+              data.future.attr("set_result")(std::move(result));
+            }));
+          }
+
+          curl_multi_remove_handle(multi_handle_, msg->easy_handle);
+          curl_easy_cleanup(msg->easy_handle);
+        }
+      }
+    }
+  }
+}
+
+void RedC::cleanup() {
+  std::unique_lock<std::mutex> lock(mutex_);
+  for (auto &[easy, data] : transfers_) {
+    {
+      acq_gil gil;
+      call_soon_threadsafe_(data.future.attr("cancel"));
+    }
+
+    curl_multi_remove_handle(multi_handle_, easy);
+    curl_easy_cleanup(easy);
+  }
+  transfers_.clear();
+}
+
+void RedC::CHECK_RUNNING() {
+  if (!running_) {
+    throw std::runtime_error("RedC can't be used after being closed");
+  }
+}
+
+size_t RedC::header_callback(char *buffer, size_t size, size_t nitems, Data *clientp) {
+  size_t total_size = size * nitems;
+  clientp->headers.insert(clientp->headers.end(), buffer, buffer + total_size);
+
+  return total_size;
+}
+
+size_t RedC::progress_callback(Data *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal,
+                               curl_off_t ulnow) {
+  if (clientp->has_progress_callback) {
+    try {
+      acq_gil
+          gil;  //TODO: this sometimes hangs on exit, which lead to other functions to block such as curl_multi_perform and worker_loop never exit
+      clientp->progress_callback(dltotal, dlnow, ultotal, ulnow);
+    } catch (const std::exception &e) {
+      std::cerr << "Error in progress_callback: " << e.what() << std::endl;
+    }
+  }
+
+  return 0;
+}
+
+size_t RedC::write_callback(char *data, size_t size, size_t nmemb, Data *clientp) {
+  size_t total_size = size * nmemb;
+
+  if (clientp->has_stream_callback) {
+    try {
+      acq_gil gil;
+      clientp->stream_callback(py_bytes(data, total_size), total_size);
+    } catch (const std::exception &e) {
+      std::cerr << "Error in stream_callback: " << e.what() << std::endl;
+    }
+  } else {
+    clientp->response.insert(clientp->response.end(), data, data + total_size);
+  }
+
+  return total_size;
+}
+
+NB_MODULE(redc_ext, m) {
+  nb::class_<RedC>(m, "RedC")
+      .def(nb::init<const long &>())
+      .def("is_running", &RedC::is_running)
+      .def("request", &RedC::request, arg("method"), arg("url"), arg("raw_data") = "", arg("data") = nb::none(),
+           arg("files") = nb::none(), arg("headers") = nb::none(), arg("timeout_ms") = 60 * 1000,
+           arg("connect_timeout_ms") = 0, arg("allow_redirect") = true, arg("proxy_url") = "", arg("verify") = true,
+           arg("ca_cert_path") = "", arg("stream_callback") = nb::none(), arg("progress_callback") = nb::none(),
+           arg("verbose") = false)
+      .def("close", &RedC::close);
+}

redc/ext/redc.h

@@ -0,0 +1,91 @@
+#ifndef REDC_H
+#define REDC_H
+
+#include <atomic>
+#include <cstring>
+#include <map>
+#include <mutex>
+#include <thread>
+#include <vector>
+
+#include <curl/curl.h>
+
+#include <nanobind/nanobind.h>
+#include <nanobind/stl/string.h>
+#include <nanobind/stl/tuple.h>
+
+#include "utils/concurrentqueue.h"
+#include "utils/curl_utils.h"
+
+namespace nb = nanobind;
+using namespace nb::literals;
+
+using acq_gil = nb::gil_scoped_acquire;
+using rel_gil = nb::gil_scoped_release;
+
+using py_object = nb::object;
+using py_bytes = nb::bytes;
+using arg = nb::arg;
+using dict = nb::dict;
+
+bool isNullOrEmpty(const char *str) {
+  return !str || !*str;
+}
+
+struct Data {
+  py_object future;
+  py_object loop;
+  py_object stream_callback{nb::none()};
+  py_object progress_callback{nb::none()};
+
+  bool has_stream_callback{false};
+  bool has_progress_callback{false};
+
+  std::vector<char> headers;
+  CurlSlist request_headers;
+  CurlMime curl_mime_;
+
+  std::vector<char> response;
+};
+
+class RedC {
+ public:
+  RedC(const long &buffer = 16384);
+  ~RedC();
+
+  bool is_running();
+  void close();
+
+  py_object request(const char *method, const char *url, const char *raw_data = "", const py_object &data = nb::none(),
+                    const py_object &files = nb::none(), const py_object &headers = nb::none(),
+                    const long &timeout_ms = 60 * 1000, const long &connect_timeout_ms = 0,
+                    const bool &allow_redirect = true, const char *proxy_url = "", const bool &verify = true,
+                    const char *ca_cert_path = "", const py_object &stream_callback = nb::none(),
+                    const py_object &progress_callback = nb::none(), const bool &verbose = false);
+
+ private:
+  int still_running_{0};
+  long buffer_size_;
+  py_object loop_;
+  py_object call_soon_threadsafe_;
+
+  CURLM *multi_handle_;
+
+  std::map<CURL *, Data> transfers_;
+  std::mutex mutex_;
+  std::thread worker_thread_;
+  std::atomic<bool> running_{false};
+
+  moodycamel::ConcurrentQueue<CURL *> queue_;
+
+  void worker_loop();
+  void cleanup();
+  void CHECK_RUNNING();
+
+  static size_t header_callback(char *buffer, size_t size, size_t nitems, Data *clientp);
+  static size_t progress_callback(Data *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal,
+                                  curl_off_t ulnow);
+  static size_t write_callback(char *data, size_t size, size_t nmemb, Data *clientp);
+};
+
+#endif  // REDC_H