object-storage-proxy 0.3.7__tar.gz → 0.3.9__tar.gz

{object_storage_proxy-0.3.7 → object_storage_proxy-0.3.9}/Cargo.lock

@@ -382,9 +382,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
 
 [[package]]
 name = "bytes"
-version = "1.10.0"
+version = "1.10.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f61dac84819c6588b558454b194026eb1f09c293b9036ae9b159e74e73ab6cf9"
+checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a"
 
 [[package]]
 name = "cc"
@@ -1701,19 +1701,23 @@ dependencies = [
 
 [[package]]
 name = "object-storage-proxy"
-version = "0.3.7"
+version = "0.3.9"
 dependencies = [
+ "async-stream",
  "async-trait",
+ "bytes",
  "chrono",
  "clap 4.5.37",
  "dotenv",
  "env_logger",
  "futures",
+ "futures-core",
  "hex",
  "http",
  "log",
  "nom 8.0.0",
  "percent-encoding",
+ "pin-project-lite",
  "pingora",
  "prometheus 0.14.0",
  "pyo3",
@@ -1725,6 +1729,7 @@ dependencies = [
  "serde_json",
  "sha256",
  "tokio",
+ "tokio-util",
  "tracing",
  "tracing-subscriber",
  "url",
@@ -3256,9 +3261,9 @@ dependencies = [
 
 [[package]]
 name = "tokio-util"
-version = "0.7.13"
+version = "0.7.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d7fcaa8d55a2bdd6b83ace262b016eca0d79ee02818c5c1bcdf0305114081078"
+checksum = "66a539a9ad6d5d281510d5bd368c973d636c02dbf8a67300bfb6b950696ad7df"
 dependencies = [
  "bytes",
  "futures-core",

{object_storage_proxy-0.3.7 → object_storage_proxy-0.3.9}/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "object-storage-proxy"
-version = "0.3.7"
+version = "0.3.9"
 edition = "2024"
 
 [dependencies]
@@ -34,6 +34,11 @@ ring = "0.17.14"
 hex = "0.4.3"
 regex = "1.11.1"
 percent-encoding = "2.3.1"
+bytes = "1.10.1"
+async-stream = "0.3.6"
+tokio-util = "0.7.15"
+futures-core = "0.3.31"
+pin-project-lite = "0.2.16"
 
 # [build-dependencies]
 # openssl-sys = { version = "0.9", features = ["vendored"] }

{object_storage_proxy-0.3.7 → object_storage_proxy-0.3.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: object-storage-proxy
-Version: 0.3.7
+Version: 0.3.9
 Classifier: License :: Other/Proprietary License
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: Implementation :: CPython

{object_storage_proxy-0.3.7 → object_storage_proxy-0.3.9}/src/credentials/signer.rs

@@ -6,6 +6,10 @@ use tracing::{debug, error};
 use std::{collections::{HashMap, HashSet}, fmt};
 use url::Url;
 
+use ring::hmac;
+
+use bytes::{Bytes, BytesMut};
+
 use crate::parsers::{cos_map::CosMapItem, credentials::{parse_credential_scope, parse_token_from_header}};
 
 const SHORT_DATE: &str = "%Y%m%d";
@@ -728,6 +732,70 @@ pub async fn signature_is_valid_for_presigned(
     ).await
 }
 
+
+
+/// Build a stream whose items are *already* wrapped in
+/// “AWS-chunk-signed” envelopes.
+///
+/// * `body`        – raw payload implementing `AsyncRead`  
+/// * `signing_key` – result of the usual `signing_key()` step  
+/// * `scope`       – e.g. `"20250501/eu-west-3/s3/aws4_request"`  
+/// * `ts`          – the `X-Amz-Date` you put in the header (`YYYYMMDDThhmmssZ`)  
+/// * `seed_sig`    – the `Signature=` value you computed for the
+///                   *headers* (the one that goes into `Authorization:`)
+///
+/// ```text
+/// ┌──── header chunk ────┐┌── data ─┐┌─ CRLF ─┐
+/// <hex-len>;chunk-signature=<sig>\r\n<bytes>\r\n
+/// ```
+///
+/// The very last frame is
+/// ```text
+/// 0;chunk-signature=<final-sig>\r\n\r\n
+/// ```
+pub async fn wrap_streaming_body(
+    session: &mut Session,
+    upstream_request: &mut RequestHeader,
+    region: &str,
+    access_key: &str,
+    secret_key: &str,
+) -> Result<(), Box<dyn std::error::Error>> {
+    // 1. pull the COMPLETE body from the client
+    let body: Bytes = session.read_request_body().await.expect("Failed to read request body").unwrap();
+
+    // 2. overwrite the x-amz-* headers so that we can sign UNSIGNED-PAYLOAD
+    upstream_request.insert_header("x-amz-content-sha256", "UNSIGNED-PAYLOAD")?;
+    upstream_request.remove_header("x-amz-decoded-content-length");
+    upstream_request.insert_header("content-length", body.len().to_string())?;
+
+    // 3. resign
+    let ts = chrono::Utc::now();
+    let url = upstream_request.uri.to_string();
+    upstream_request.insert_header("x-amz-date", ts.format("%Y%m%dT%H%M%SZ").to_string())?;
+    let signer = AwsSign::new(
+        upstream_request.method.as_str(),
+        &url,
+        &ts,
+        &upstream_request.headers,
+        region,
+        access_key,
+        secret_key,
+        "s3",
+        b"UNSIGNED-PAYLOAD",
+        None,
+    );
+    let auth = signer.sign();
+    upstream_request.insert_header("authorization", auth)?;
+
+    let end_of_stream: bool = session.is_body_done();
+
+    // 4. finally attach the body
+    session.write_response_body(Some(body), end_of_stream).await?;
+
+    Ok(())
+}
+
+
 #[cfg(test)]
 mod tests {
     use super::*;

{object_storage_proxy-0.3.7 → object_storage_proxy-0.3.9}/src/lib.rs

@@ -1,13 +1,12 @@
 #![warn(clippy::all)]
 use async_trait::async_trait;
-use credentials::signer::{signature_is_valid_for_presigned, signature_is_valid_for_request};
+use credentials::signer::{signature_is_valid_for_presigned, signature_is_valid_for_request, wrap_streaming_body};
 use dotenv::dotenv;
 use http::Uri;
 use http::uri::Authority;
 use parsers::cos_map::{CosMapItem, parse_cos_map};
 use parsers::keystore::parse_hmac_list;
 use pingora::http::ResponseHeader;
-use pingora::protocols::ALPN;
 use pingora::Result;
 use pingora::proxy::{ProxyHttp, Session};
 use pingora::server::Server;
@@ -29,6 +28,9 @@ use tracing::{debug, error, info};
 use tracing_subscriber::EnvFilter;
 use tracing_subscriber::fmt::time::ChronoLocal;
 
+use tokio_util::io::StreamReader;
+use futures::TryStreamExt;
+
 pub mod parsers;
 use parsers::credentials::{parse_presigned_params, parse_token_from_header};
 use parsers::path::parse_path;
@@ -550,7 +552,7 @@ impl ProxyHttp for MyProxy {
 
     async fn upstream_request_filter(
         &self,
-        _session: &mut Session,
+        session: &mut Session,
         upstream_request: &mut pingora::http::RequestHeader,
         ctx: &mut Self::CTX,
     ) -> Result<()> {
@@ -647,26 +649,84 @@ impl ProxyHttp for MyProxy {
             // "x-amz-trailer",
         ];
 
+
         let to_check: Vec<String> = upstream_request
             .headers
             .iter()
             .map(|(name, _)| name.as_str().to_owned())
             .collect();
 
+        // for name in to_check {
+        //     if !allowed.contains(&name.as_str()) {
+        //         let _ = upstream_request.remove_header(&name);
+        //     }
+        // }
+
         for name in to_check {
-            if !allowed.contains(&name.as_str()) {
+            let keep = allowed.contains(&name.as_str())
+                || name.starts_with("x-amz-checksum-");
+            if !keep {
                 let _ = upstream_request.remove_header(&name);
             }
         }
 
         if maybe_hmac {
             debug!("HMAC: Signing request for bucket: {}", hdr_bucket);
-            sign_request(upstream_request, bucket_config.as_ref().unwrap())
-                .await
-                .map_err(|e| {
-                    error!("Failed to sign request for {}: {e}", hdr_bucket);
-                    pingora::Error::new_str("Failed to sign request")
+
+            let streaming = {
+                upstream_request
+                    .headers
+                    .get("x-amz-content-sha256")
+                    .map(|v| v.as_bytes().starts_with(b"STREAMING-AWS4"))
+                    .unwrap_or(false)
+            };
+
+            if streaming {
+                let auth_header = session
+                    .req_header()
+                    .headers
+                    .get("authorization")
+                    .and_then(|h| h.to_str().ok())
+                    .map(ToString::to_string)
+                    .unwrap_or_default();
+
+                let access_key = parse_token_from_header(&auth_header)
+                    .map_err(|_| pingora::Error::new_str("Failed to parse access_key"))?
+                    .1
+                    .to_string();
+                
+                let secret_key = {
+                    let map = ctx.hmac_keystore.read().await;
+                    map.get(&access_key).cloned()
+                };
+
+                if secret_key.is_none() {
+                    error!("No secret key found for access key: {}", access_key);
+                    return Err(pingora::Error::new_str("No secret key found"));
+                }
+                let secret_key = secret_key.unwrap();
+
+                wrap_streaming_body(
+                    session,
+                    upstream_request,
+                    "eu-west-3",               // <- region for the bucket
+                    &access_key,        // <- retrieved from your CosMapItem
+                    &secret_key,
+                )
+                .await.map_err(|e| {
+                    error!("Failed to wrap streaming body: {e}");
+                    pingora::Error::new_str("Failed to wrap streaming body")
                 })?;
+                dbg!("streaming signature!!");
+            } else {
+                sign_request(upstream_request, bucket_config.as_ref().unwrap())
+                    .await
+                    .map_err(|e| {
+                        error!("Failed to sign request for {}: {e}", hdr_bucket);
+                        pingora::Error::new_str("Failed to sign request")
+                    })?;
+            }
+
             debug!("Request signed for bucket: {}", hdr_bucket);
             debug!("{:#?}", &upstream_request.headers);
         } else {