object-storage-proxy 0.2.16__tar.gz → 0.3.1__tar.gz

{object_storage_proxy-0.2.16 → object_storage_proxy-0.3.1}/Cargo.lock

@@ -1701,7 +1701,7 @@ dependencies = [
 
 [[package]]
 name = "object-storage-proxy"
-version = "0.2.16"
+version = "0.3.1"
 dependencies = [
  "async-trait",
  "chrono",
@@ -1713,6 +1713,7 @@ dependencies = [
  "http",
  "log",
  "nom 8.0.0",
+ "percent-encoding",
  "pingora",
  "prometheus 0.14.0",
  "pyo3",

{object_storage_proxy-0.2.16 → object_storage_proxy-0.3.1}/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "object-storage-proxy"
-version = "0.2.16"
+version = "0.3.1"
 edition = "2024"
 
 [dependencies]
@@ -33,6 +33,7 @@ sha256 = "1.6.0"
 ring = "0.17.14"
 hex = "0.4.3"
 regex = "1.11.1"
+percent-encoding = "2.3.1"
 
 # [build-dependencies]
 # openssl-sys = { version = "0.9", features = ["vendored"] }

{object_storage_proxy-0.2.16 → object_storage_proxy-0.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: object-storage-proxy
-Version: 0.2.16
+Version: 0.3.1
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy

{object_storage_proxy-0.2.16 → object_storage_proxy-0.3.1}/src/credentials/signer.rs

@@ -1,11 +1,9 @@
 use chrono::{DateTime, NaiveDateTime, Utc};
 use http::header::HeaderMap;
 use pingora::{http::RequestHeader, proxy::Session};
-use rustls::crypto::hash::Hash;
 use sha256::digest;
-use tokio::sync::RwLock;
-use tracing::{debug, error, info};
-use std::{collections::{HashMap, HashSet}, fmt, sync::Arc};
+use tracing::{debug, error};
+use std::{collections::HashMap, fmt};
 use url::Url;
 
 use crate::parsers::{cos_map::CosMapItem, credentials::{parse_credential_scope, parse_token_from_header}};
@@ -62,6 +60,25 @@ where
     body: &'a [u8],
 }
 
+/// Create a new AwsSign instance
+/// 
+/// # Arguments
+/// 
+/// * `method` - HTTP method (GET, POST, etc.)
+/// * `url` - URL to sign
+/// * `datetime` - Date and time of the request
+/// * `headers` - HTTP headers
+/// * `region` - AWS region
+/// * `access_key` - AWS access key
+/// * `secret_key` - AWS secret key
+/// * `service` - AWS service code
+/// * `body` - Request body
+/// * `signed_headers` - Optional list of signed headers, used to check inbound request signature
+/// 
+/// # Returns
+/// 
+/// A new instance of `AwsSign`
+/// 
 impl<'a> AwsSign<'a, HashMap<String, String>> {
     pub fn new<B: AsRef<[u8]> + ?Sized>(
         method: &'a str,
@@ -75,6 +92,8 @@ impl<'a> AwsSign<'a, HashMap<String, String>> {
         body: &'a B,
         signed_headers: Option<&'a Vec<String>>,
     ) -> Self {
+
+
         let allowed: Vec<&str> = if let Some(sh) = signed_headers {
             sh.iter().map(String::as_str).collect()
         } else {
@@ -86,15 +105,8 @@ impl<'a> AwsSign<'a, HashMap<String, String>> {
                 "x-amz-security-token",
             ]
         };
-        // let allowed = [
-        //     "host",
-        //     "x-amz-date",
-        //     "range",
-        //     "x-amz-content-sha256",
-        //     "x-amz-security-token",
-        // ];
         
-        dbg!(&url);
+        debug!("{:#?}", &url);
         let url: Url = url.parse().unwrap();
         let headers: HashMap<String, String> = headers
             .iter()
@@ -124,6 +136,7 @@ impl<'a> AwsSign<'a, HashMap<String, String>> {
     }
 }
 
+
 /// custom debug implementation to redact secret_key
 impl<'a, T> fmt::Debug for AwsSign<'a, T>
 where
@@ -140,6 +153,7 @@ where
             .field("service", &self.service)
             .field("body", &self.body)
             .field("headers", &self.headers)
+            .field("payload_override", &self.payload_override)
             .finish()
     }
 }
@@ -148,6 +162,10 @@ impl<'a, T> AwsSign<'a, T>
 where
     &'a T: std::iter::IntoIterator<Item = (&'a String, &'a String)>, T: std::fmt::Debug
 {
+    /// for streaming uploads, we need to override the payload hash
+    /// with the actual payload hash
+    /// this is used for the `UNSIGNED-PAYLOAD` case
+    /// and for the `payload_override` case
     pub fn set_payload_override(&mut self, h: String) {
         self.payload_override = Some(h);
     }
@@ -201,14 +219,16 @@ where
         let signature = hex::encode(tag.as_ref());
         let signed_headers = self.signed_header_string();
 
-        format!(
+        let sign_string = format!(
             "AWS4-HMAC-SHA256 Credential={access_key}/{scope},\
              SignedHeaders={signed_headers},Signature={signature}",
             access_key = self.access_key,
             scope = scope_string(self.datetime, self.region, self.service),
             signed_headers = signed_headers,
             signature = signature
-        )
+        );
+        debug!("sign_string: {}", sign_string);
+        sign_string
     }
 }
 
@@ -290,6 +310,14 @@ pub fn signing_key(
     Ok(signing_tag.as_ref().to_vec())
 }
 
+
+/// Sign the request with the AWS V4 signature
+/// # Arguments
+/// * `request` - The request to sign
+/// * `cos_map` - The COS map item containing the credentials
+/// # Returns
+/// * `Ok(())` if the request was signed successfully
+/// * `Err` if there was an error signing the request
 pub(crate) async fn sign_request(
     request: &mut RequestHeader,
     cos_map: &CosMapItem,
@@ -360,22 +388,53 @@ pub(crate) async fn sign_request(
 }
 
 
-pub async fn signature_is_valid(
+/// Core signature validation: compares provided vs computed
+async fn signature_is_valid_core(
+    method: &str,
+    provided_signature: &str,
+    region: &str,
+    service: &str,
+    datetime: DateTime<Utc>,
+    full_url: &str,
+    headers: &HeaderMap,
+    payload_override: Option<String>,
+    access_key: &str,
+    secret_key: &str,
+    signed_headers: &Vec<String>,
+    body_bytes: &[u8],
+) -> Result<bool, Box<dyn std::error::Error>> {
+    // Build AwsSign for authorization header style
+    debug!("{:#?}", &headers);
+    let mut signer = AwsSign::new(
+        method,
+        full_url,
+        &datetime,
+        headers,
+        region,
+        access_key,
+        secret_key,
+        service,
+        body_bytes,
+        Some(&signed_headers),
+    );
+    if let Some(ov) = payload_override {
+        signer.set_payload_override(ov);
+    }
+    let signature = signer.sign();
+    let computed = signature.split("Signature=").nth(1).unwrap_or_default();
+    debug!("Provided signature: {}", provided_signature);
+    debug!("Computed signature: {}", computed);
+    Ok(computed == provided_signature)
+}
+
+/// Validate standard S3 Authorization header
+pub async fn signature_is_valid_for_request(
     auth_header: &str,
     session: &Session,
     secret_key: &str,
 ) -> Result<bool, Box<dyn std::error::Error>> {
 
-    let signed_headers_str = auth_header
-        .split("SignedHeaders=")
-        .nth(1)
-        .and_then(|s| s.split(',').next())
-        .unwrap_or("");
-    let signed_headers: Vec<String> =
-        signed_headers_str.split(';').map(|s| s.to_lowercase()).collect();
-
-
-    // extract access key from Authorization header
+    
     let (_, local_access_key) = parse_token_from_header(auth_header)
         .map_err(|_| pingora::Error::new_str("Failed to parse token"))?;
     let local_access_key = local_access_key.to_string();
@@ -383,49 +442,26 @@ pub async fn signature_is_valid(
         error!("Missing access key");
         return Ok(false);
     }
-
-    // extract provided signature
     let provided_signature = auth_header
         .split("Signature=")
         .nth(1)
-        .ok_or_else(|| pingora::Error::new_str("Invalid Authorization header: no Signature"))?
-        .to_string();
-
-    // parse x-amz-date header
-    let dt_header = session
-        .req_header()
-        .headers
-        .get("x-amz-date")
-        .ok_or_else(|| pingora::Error::new_str("Missing x-amz-date header"))?
-        .to_str()?;
-
-    // use NaiveDateTime then assign Utc timezone (format: %Y%m%dT%H%M%SZ)
-    let naive = NaiveDateTime::parse_from_str(dt_header, LONG_DATETIME)
-        .map_err(|_| {
-            pingora::Error::new_str("invalid date")
-        })?;
-    let datetime = naive.and_utc();
-
-
-    info!("parsing the region and service");
+        .ok_or("Missing Signature")?;
 
     let (_, (region, service)) = parse_credential_scope(auth_header)
         .map_err(|_| pingora::Error::new_str("Invalid Credential scope"))?;
 
-    dbg!(&region);
-    dbg!(&service);
+    let method = session.req_header().method.to_string();
+    // Parse date header
+    let dt_header = session.req_header().headers.get("x-amz-date").unwrap().to_str()?;
+    let datetime = NaiveDateTime::parse_from_str(dt_header, LONG_DATETIME)?.and_utc();
+
 
-    // determine payload hash header
     let content_sha256 = session
         .req_header()
         .headers
         .get("x-amz-content-sha256")
         .and_then(|h| h.to_str().ok())
         .ok_or_else(|| pingora::Error::new_str("Missing x-amz-content-sha256 header"))?;
-    // let body_bytes: &[u8] = match content_sha256 {
-    //     "UNSIGNED-PAYLOAD" => b"UNSIGNED-PAYLOAD",
-    //     _ => &[],
-    // };
 
     let (body_bytes, payload_override) = if content_sha256 == "UNSIGNED-PAYLOAD" {
         (b"UNSIGNED-PAYLOAD" as &[u8], None)
@@ -435,6 +471,7 @@ pub async fn signature_is_valid(
         (&[] as &[u8], Some(content_sha256.to_owned()))
     };
 
+    // Build full URL
     let original_uri = session.req_header().uri.to_string();
     let full_url = if original_uri.starts_with('/') {
         let host = session
@@ -448,36 +485,159 @@ pub async fn signature_is_valid(
         original_uri
     };
 
-    // construct AwsSign and compute signature
-    let method = session.req_header().method.to_string();
-    let mut signer = AwsSign::new(
+    // Signed headers list
+    let signed_headers_str = auth_header
+        .split("SignedHeaders=")
+        .nth(1)
+        .unwrap()
+        .split(',')
+        .next()
+        .unwrap();
+
+    let signed_headers: Vec<String> = signed_headers_str.split(';').map(str::to_string).collect();
+
+    signature_is_valid_core(
         &method,
+        provided_signature,
+        region,
+        service,
+        datetime,
         &full_url,
-        &datetime,
         &session.req_header().headers,
-        region,
+        payload_override,
         &local_access_key,
-        &secret_key,
-        service,
+        secret_key,
+        &signed_headers,
         body_bytes,
-        Some(&signed_headers),
-    );
+    )
+    .await
+}
 
-    if let Some(ov) = payload_override {
-        signer.set_payload_override(ov);
+
+/// Validate presigned URL signature
+pub async fn signature_is_valid_for_presigned(
+    session: &Session,
+    secret_key: &str,
+) -> Result<bool, Box<dyn std::error::Error>> {
+    // Extract query params from the URL
+
+    let uri = session.req_header().uri.to_string();
+    let full_uri = if uri.starts_with('/') {
+        // build an absolute URL: scheme://host + path?query
+        let host = session
+            .req_header()
+            .headers
+            .get("host")
+            .ok_or("Missing host header")?
+            .to_str()?;
+        format!("https://{}{}", host, uri)
+    } else {
+        uri
+    };
+
+    
+    let mut url = Url::parse(&full_uri)?; 
+    debug!("full_url: {}", url);
+    let mut provided_signature = None;
+    let mut qp: Vec<(String,String)> = vec![];
+    for (k, v) in url.query_pairs() {
+        if k == "X-Amz-Signature" {
+            provided_signature = Some(v.into_owned());
+        } else {
+            qp.push((k.into_owned(), v.into_owned()));
+        }
+    }
+    let provided_signature = provided_signature.ok_or("Missing X-Amz-Signature")?;
+    
+    // rebuild query string without the signature
+    qp.sort();
+    let new_query = qp.iter()
+                      .map(|(k,v)| format!("{k}={v}"))
+                      .collect::<Vec<_>>()
+                      .join("&");
+    url.set_query(Some(&new_query));
+    
+    // params map (also without the signature)
+    let params: HashMap<_, _> = qp.into_iter().collect();
+    debug!("params: {:?}", params);
+    debug!("url: {:?}", url);
+
+    debug!("provided signature: {}", provided_signature);
+    let credential = params
+        .get("X-Amz-Credential")
+        .ok_or("Missing X-Amz-Credential")?;
+
+    debug!("credential: {}", credential);
+
+    // Parse credential: <access_key>/<date>/<region>/<service>/aws4_request
+    let mut parts = credential.split('/');
+    let access_key = parts.next().ok_or("Malformed Credential")?;
+    let _credential_date = parts.next().ok_or("Malformed Credential")?;
+    let region = parts.next().ok_or("Malformed Credential")?;
+    let service = parts.next().ok_or("Malformed Credential")?;
+
+    debug!("access_key: {}", access_key);
+    debug!("region: {}", region);
+    debug!("service: {}", service);
+
+    // Parse date from query
+    let date_str = params
+        .get("X-Amz-Date")
+        .ok_or("Missing X-Amz-Date")?;
+    let datetime = NaiveDateTime::parse_from_str(date_str, LONG_DATETIME)?.and_utc();
+
+    debug!("datetime: {}", datetime);
+
+    let body_bytes: &[u8] = b"UNSIGNED-PAYLOAD";
+    let payload_override = None;  
+
+    debug!("body_bytes: {:?}", body_bytes);
+
+    // Collect signed headers list
+    let signed_headers = params
+        .get("X-Amz-SignedHeaders")
+        .unwrap()
+        .split(';')
+        .map(str::to_string)
+        .collect::<Vec<_>>();
+
+    let mut signed_hdrs = HeaderMap::new();
+
+    let host_header = match url.port_or_known_default() {
+        Some(443) | Some(80) | None => url.host_str().unwrap().to_owned(),
+        Some(p)                    => format!("{}:{}", url.host_str().unwrap(), p),
+    };
+
+    signed_hdrs.insert("host", host_header.parse()?);
+    
+    // copy any additional headers that appear in X-Amz-SignedHeaders (rare)
+    for h in &["x-amz-date", "x-amz-content-sha256", "range", "x-amz-security-token"] {
+        if signed_headers.contains(&h.to_string()) {
+            if let Some(v) = session.req_header().headers.get(*h) {
+                signed_hdrs.insert(*h, v.clone());
+            }
+        }
     }
 
-    let signature = signer.sign();
-    let computed_signature = signature
-        .split("Signature=")
-        .nth(1)
-        .unwrap_or_default();
 
-    info!("Provided signature: {}", provided_signature);
-    info!("Computed signature: {}", computed_signature);
-    Ok(computed_signature == provided_signature)
-}
 
+    debug!("signed_headers: {:?}", signed_headers);
+    // Delegate to core validator
+    signature_is_valid_core(
+        session.req_header().method.as_str(),
+        &provided_signature,
+        region,
+        service,
+        datetime,
+        url.as_str(),
+        &signed_hdrs,
+        payload_override,
+        access_key,
+        secret_key,
+        &signed_headers,
+        body_bytes,
+    ).await
+}
 
 #[cfg(test)]
 mod tests {
@@ -671,4 +831,5 @@ mod tests {
         let qs = canonical_query_string(&parsed);
         assert_eq!(qs, "a=1%20space&b=2");
     }
+    
 }

{object_storage_proxy-0.2.16 → object_storage_proxy-0.3.1}/src/lib.rs

@@ -1,11 +1,12 @@
 #![warn(clippy::all)]
 use async_trait::async_trait;
-use credentials::signer::signature_is_valid;
+use credentials::signer::{signature_is_valid_for_presigned, signature_is_valid_for_request};
 use dotenv::dotenv;
 use http::Uri;
 use http::uri::Authority;
 use parsers::cos_map::{CosMapItem, parse_cos_map};
 use parsers::keystore::parse_hmac_list;
+use pingora::http::ResponseHeader;
 use pingora::Result;
 use pingora::proxy::{ProxyHttp, Session};
 use pingora::server::Server;
@@ -28,7 +29,7 @@ use tracing_subscriber::EnvFilter;
 use tracing_subscriber::fmt::time::ChronoLocal;
 
 pub mod parsers;
-use parsers::credentials::parse_token_from_header;
+use parsers::credentials::{parse_presigned_params, parse_token_from_header};
 use parsers::path::parse_path;
 
 pub mod credentials;
@@ -197,6 +198,7 @@ pub struct MyCtx {
     validator: Option<PyObject>,
     bucket_creds_fetcher: Option<PyObject>,
     hmac_fetcher: Option<PyObject>,
+    is_presigned: Option<bool>,
     
 }
 
@@ -221,6 +223,7 @@ impl ProxyHttp for MyProxy {
                 .hmac_fetcher
                 .as_ref()
                 .map(|v| Python::with_gil(|py| v.clone_ref(py))),
+            is_presigned: None,
         }
     }
 
@@ -311,19 +314,30 @@ impl ProxyHttp for MyProxy {
             let map = ctx.cos_mapping.read().await;
             map.get(bucket).and_then(|c| c.ttl).unwrap_or(0)
         };
+        let mut access_key: String = String::new();
+
+        if auth_header.is_empty() {
+            if let Some(q) = session.req_header().uri.query() {
+                if q.contains("X-Amz-Credential") {
+                    let (_, p) = parse_presigned_params(&format!("?{q}"))
+                        .map_err(|_| pingora::Error::new_str("Failed to parse presigned params"))?;
+                    access_key = p.access_key.clone();
+                }
+            }
+        } else {
+            access_key = parse_token_from_header(&auth_header)
+            .map_err(|_| pingora::Error::new_str("Failed to parse access_key"))?
+            .1
+            .to_string();
+        }
 
         let is_authorized = if let Some(py_cb) = &ctx.validator {
-            let access_key = parse_token_from_header(&auth_header)
-                .map_err(|_| pingora::Error::new_str("Failed to parse access_key"))?
-                .1
-                .to_string();
-
-                let is_multipart = session
-                    .req_header()
-                    .uri
-                    .query()
-                    .map_or(false, |q| q.contains("uploadId="));
 
+            let is_multipart = session
+                .req_header()
+                .uri
+                .query()
+                .map_or(false, |q| q.contains("uploadId="));
 
 
             info!("CHECKING SIGNATURE");
@@ -333,6 +347,62 @@ impl ProxyHttp for MyProxy {
                     // continue
                     
                 } else {
+                    // presigned
+                    info!("Checking presigned signature");
+                    let uri_q = session.req_header().uri.query().unwrap_or("");
+
+                    if auth_header.is_empty() && uri_q.contains("X-Amz-Signature") {
+                        ctx.is_presigned = Some(true);
+
+                        // ensure we have the secret_key in the keystore
+                        if !ctx.hmac_keystore.read().await.contains_key(&access_key) {
+                            debug!("No key in keystore, trying to fetch via hmac_fetcher for ->{}<-", access_key);
+                            // fetch via hmac_fetcher exactly as you do below…
+                            if let Some(py_fetcher) = &ctx.hmac_fetcher {
+                                // call Python callback
+                                let cb = py_fetcher;
+                                let secret: PyResult<String> = Python::with_gil(|py| {
+                                    cb.call1(py, (&access_key,))
+                                      .and_then(|r| r.extract(py))
+                                });
+                                debug!("Got secret: {:#?}", secret);
+                                match secret {
+                                    Ok(secret_key) => {
+                                        debug!("got key and inserting into keystore");
+                                        ctx.hmac_keystore.write().await.insert(access_key.clone().to_string(), secret_key);
+                                    }
+                                    Err(_) => {
+                                        // no key → unauthorized
+                                        session.respond_error(401).await?;
+                                        return Ok(true);
+                                    }
+                                }
+                            } else {
+                                session.respond_error(401).await?;
+                                return Ok(true);
+                            }
+     
+                        }
+                        debug!("now checking if the signature is valid for presigned...");
+                        let sk = ctx.hmac_keystore.read().await.get(&access_key).unwrap().clone();
+                        debug!("got secret {} from keystore", sk);
+                        debug!("RAW_PATH       = {}", &session.req_header().uri);
+                        debug!("RAW_HOST_HDR   = {:?}", &session.req_header().headers.get("host"));
+                        let ok = match signature_is_valid_for_presigned(&session, &sk).await {
+                            Ok(b)  => b,
+                            Err(e) => {
+                                error!("presigned-URL validation error: {e}");   // <-- keep the info
+                                return Err(pingora::Error::new_str("Failed to check signature"));
+                            }
+                        };                       
+                        info!("is signature valid?: {}", ok);
+                        if !ok {
+                            session.respond_error(401).await?;
+                            return Ok(true);
+                        }
+                    } else {
+                    info!("processing a regular request");
+
                     let has_key = {
                         let map = ctx.hmac_keystore.read().await;
                         map.contains_key(&access_key)
@@ -361,12 +431,12 @@ impl ProxyHttp for MyProxy {
                         }
                     }
                     let secret_key = {
-                                            let map = ctx.hmac_keystore.read().await;
-                                            map.get(&access_key).cloned()
-                                        };
+                                let map = ctx.hmac_keystore.read().await;
+                                map.get(&access_key).cloned()
+                            };
 
                     info!("Checking signature");
-                     let sig_ok = match signature_is_valid(
+                     let sig_ok = match signature_is_valid_for_request(
                          &auth_header,
                          &session,
                          &secret_key.unwrap(),
@@ -389,17 +459,19 @@ impl ProxyHttp for MyProxy {
                          session.respond_error(401).await?;
                          return Ok(true);
                      }
+                    }
                 }
             }
-
-            let cache_key = format!("{}:{}", access_key, bucket);
+            info!("Signature check passed, continuing now onto the bespoke validation");
+            let cache_key = format!("{}:{}", &access_key, bucket);
+            debug!("Cache key: {}", cache_key);
 
             let bucket_clone = bucket.to_string();
             let callback_clone: PyObject = Python::with_gil(|py| py_cb.clone_ref(py));
-
+            let move_access_key = access_key.clone();
             ctx.auth_cache
                 .get_or_validate(&cache_key, Duration::from_secs(ttl), move || {
-                    let tk = access_key.clone();
+                    let tk = move_access_key.clone();
                     let bu = bucket_clone.clone();
                     let cb = Python::with_gil(|py| callback_clone.clone_ref(py));
                     async move {
@@ -423,10 +495,8 @@ impl ProxyHttp for MyProxy {
             let map = ctx.cos_mapping.read().await;
             map.get(&hdr_bucket).cloned()
         };
-        let access_key = parse_token_from_header(&auth_header)
-            .map_err(|_| pingora::Error::new_str("Failed to parse access_key"))?
-            .1
-            .to_string();
+
+        debug!("Access key: {}", &access_key);
 
         // we have to check for some available credentials here to be able to return unauthorized already if not
         match bucket_config.clone() {
@@ -473,6 +543,31 @@ impl ProxyHttp for MyProxy {
         ctx: &mut Self::CTX,
     ) -> Result<()> {
 
+        if let Some(presigned) = ctx.is_presigned {
+            if presigned {
+                debug!("upstream_request_filter::presigned");
+                let cleaned_q = upstream_request
+                    .uri
+                    .query()
+                    .unwrap_or("")
+                    .split('&')
+                    .filter(|kv| !kv.starts_with("X-Amz-"))
+                    .collect::<Vec<_>>()
+                    .join("&");
+
+            let _ = upstream_request.remove_header("authorization");
+        
+            let new_path_and_query = if cleaned_q.is_empty() {
+                upstream_request.uri.path().to_owned()
+            } else {
+                format!("{}?{}", upstream_request.uri.path(), cleaned_q)
+            };
+        
+            upstream_request.set_uri(new_path_and_query.try_into().unwrap());
+ 
+            }
+        };
+
         let _ = upstream_request.remove_header("accept-encoding");
 
         debug!("upstream_request_filter::start");
@@ -587,6 +682,19 @@ impl ProxyHttp for MyProxy {
         Ok(())
     }
 
+    async fn response_filter(
+        &self,
+        _session: &mut Session,
+        resp: &mut ResponseHeader,
+        _ctx: &mut Self::CTX,
+    ) -> Result<()> {
+        let _ = resp.remove_header("server");
+
+        let _ = resp.insert_header("Server", "Object-Storage-Proxy");
+
+        Ok(())
+    }
+
 }
 
 pub fn init_tracing() {
@@ -618,7 +726,7 @@ pub fn run_server(py: Python, run_args: &ProxyServerConfig) {
         parse_hmac_list(py, &run_args.hmac_keystore).unwrap_or(HashMap::new())
     };
 
-    info!("HMAC keys: {:#?}", &local_hmac_map);
+    debug!("HMAC keys: {:#?}", &local_hmac_map);
 
     let cosmap = Arc::new(RwLock::new(parse_cos_map(py, &run_args.cos_map).unwrap()));
     let hmac_keystore = Arc::new(RwLock::new(local_hmac_map));

object_storage_proxy-0.3.1/src/parsers/credentials.rs

@@ -0,0 +1,234 @@
+use std::collections::HashMap;
+
+use nom::{
+    bytes::complete::{tag, take_until, take_while1}, multi::separated_list1, sequence::{preceded, separated_pair}, IResult, Parser
+};
+
+use nom::character::complete::char as nomchar;
+use percent_encoding::percent_decode_str;
+use nom::error::{Error, ErrorKind, make_error};
+
+fn miss<'a>(i: &'a str) -> nom::Err<Error<&'a str>> {
+    nom::Err::Error(make_error(i, ErrorKind::Tag))
+}
+
+pub fn parse_token_from_header(header: &str) -> IResult<&str, &str> {
+    let (_, token) =
+        (preceded(tag("AWS4-HMAC-SHA256 Credential="), take_until("/"))).parse(header)?;
+
+    Ok(("", token))
+}
+
+pub fn parse_credential_scope(input: &str) -> IResult<&str, (&str, &str)> {
+    let (input, _) = take_until("Credential=")(input)?;
+    let (remaining, (_, _, _, _, _, region, _, service, _)) = (
+        tag("Credential="),          // prefix
+        take_until("/"),            // access key
+        tag("/"),
+        take_until("/"),            // date
+        tag("/"),
+        take_until("/"),            // region
+        tag("/"),
+        take_until("/aws4_request"),// service
+        tag("/aws4_request"),       // trailing
+    ).parse(input)?;
+    Ok((remaining, (region, service)))
+}
+
+#[derive(Debug, PartialEq)]
+pub struct PresignedParams {
+    pub algorithm: String,
+    pub access_key: String,
+    pub credential_date: String,
+    pub region: String,
+    pub service: String,
+    pub amz_date: String,
+    pub expires: String,
+    pub signed_headers: String,
+    pub signature: String,
+}
+
+/// key chars: letters, digits, dash, dot
+fn is_key_char(c: char) -> bool {
+    c.is_alphanumeric() || c == '-' || c == '.'
+}
+/// val chars: anything except `&`
+fn is_val_char(c: char) -> bool {
+    c != '&'
+}
+
+/// Parse the `?` and then a list of `key=val` pairs separated by `&`
+fn query_pairs(input: &str) -> IResult<&str, Vec<(&str, &str)>> {
+    // skip everything up to the '?'
+    let (input, _) = if let Some(i) = input.find('?') {
+        // consume everything up to – and including – the first '?'
+        nom::bytes::complete::take::<_, _, nom::error::Error<_>>(i + 1usize)(input)?
+    } else {
+        // the input *is* the query string, start parsing immediately
+        ("", input)
+    };    // then parse key=val (& key=val)* until the end
+    separated_list1(
+        nomchar('&'),
+        separated_pair(
+            take_while1(is_key_char),
+            nomchar('='),
+            take_while1(is_val_char),
+        ),
+    ).parse(input)
+}
+
+/// Top-level parser
+pub fn parse_presigned_params(input: &str) -> IResult<&str, PresignedParams> {
+    let (rest, pairs) = query_pairs(input)?;
+    // build a little map
+    let mut m = HashMap::new();
+    for (k, v) in pairs {
+        // percent-decode the value
+        let val = percent_decode_str(v).decode_utf8_lossy().into_owned();
+        m.insert(k, val);
+    }
+
+    // pull out each required field (error if missing)
+    let algorithm      = m.remove("X-Amz-Algorithm"     ).ok_or_else(|| miss(rest))?;
+    let credential_raw = m.remove("X-Amz-Credential"    ).ok_or_else(|| miss(rest))?;
+    let amz_date       = m.remove("X-Amz-Date"          ).ok_or_else(|| miss(rest))?;
+    let expires        = m.remove("X-Amz-Expires"       ).ok_or_else(|| miss(rest))?;
+    let signed_headers = m.remove("X-Amz-SignedHeaders" ).ok_or_else(|| miss(rest))?;
+    let signature      = m.remove("X-Amz-Signature"     ).ok_or_else(|| miss(rest))?;
+
+    // split the credential into its components
+    // format is: ACCESSKEY/DATE/REGION/SERVICE/aws4_request
+    let mut parts = credential_raw.split('/');
+    let access_key     = parts.next().unwrap_or("").to_string();
+    let credential_date= parts.next().unwrap_or("").to_string();
+    let region         = parts.next().unwrap_or("").to_string();
+    let service        = parts.next().unwrap_or("").to_string();
+    // ignore the final “aws4_request”
+
+    Ok((rest, PresignedParams {
+        algorithm,
+        access_key,
+        credential_date,
+        region,
+        service,
+        amz_date,
+        expires,
+        signed_headers,
+        signature,
+    }))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use nom::Err;
+
+    #[test]
+    fn test_parse_token_from_header() {
+        let input = "AWS4-HMAC-SHA256 Credential=MYLOCAL123/20250417/eu-west-3/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=ec323a7db4d0b8bd27eced3b2bb0d59f9b9dd";
+        let result = parse_token_from_header(input);
+        assert_eq!(result, Ok(("", ("MYLOCAL123"))));
+    }
+
+    #[test]
+    fn parse_token_from_header_success_and_error() {
+        let input = "AWS4-HMAC-SHA256 Credential=TOKEN123/20250417/eu-west-1/s3/aws4_request, SignedHeaders=host,Signature=abc";
+        let result = parse_token_from_header(input);
+        assert!(result.is_ok());
+        let (remaining, token) = result.unwrap();
+        assert_eq!(token, "TOKEN123");
+        assert_eq!(remaining, "");
+
+        let bad = "NoCredentialHere";
+        assert!(parse_token_from_header(bad).is_err());
+    }
+
+
+    #[test]
+    fn test_parse_valid_scope() {
+        let header = "Credential=AKIAEXAMPLE/20250425/us-west-2/s3/aws4_request, SignedHeaders=host;x-amz-date";
+        let (rem, (region, service)) = parse_credential_scope(header).expect("parse failed");
+        assert_eq!(region, "us-west-2");
+        assert_eq!(service, "s3");
+        assert!(rem.starts_with(", SignedHeaders"));
+    }
+
+    #[test]
+    fn test_parse_invalid_scope() {
+        let header = "Credential=AKIAEXAMPLE/20250425/us-west-2/s3/some_request";
+        assert!(matches!(parse_credential_scope(header), Err(Err::Error(_))));
+    }
+
+    #[test]
+    fn test_parse_with_prefix() {
+        let header = "Authorization: AWS4-HMAC-SHA256 Credential=XYZ/20250425/eu-central-1/dynamodb/aws4_request/extra";
+        let idx = header.find("Credential=").unwrap();
+        let substr = &header[idx..];
+        let (rem, (region, service)) = parse_credential_scope(substr).expect("parse failed");
+        assert_eq!(region, "eu-central-1");
+        assert_eq!(service, "dynamodb");
+        assert!(rem.starts_with("/extra"));
+    }
+
+    #[test]
+    fn parses_all_fields() {
+        let url = "http://localhost:6190/proxy-aws-bucket01/mandelbrot/?\
+            X-Amz-Algorithm=AWS4-HMAC-SHA256&\
+            X-Amz-Credential=MYLOCAL123%2F20250426%2Feu-west-3%2Fs3%2Faws4_request&\
+            X-Amz-Date=20250426T143249Z&\
+            X-Amz-Expires=3600&\
+            X-Amz-SignedHeaders=host&\
+            X-Amz-Signature=53cb3d8a12c8c1078fba3fcd55ced9c93fcdc8e2f98184e9ffea50245f4ebea5";
+
+        let (_, p) = parse_presigned_params(url).unwrap();
+        assert_eq!(p.algorithm, "AWS4-HMAC-SHA256");
+        assert_eq!(p.access_key, "MYLOCAL123");
+        assert_eq!(p.credential_date, "20250426");
+        assert_eq!(p.region, "eu-west-3");
+        assert_eq!(p.service, "s3");
+        assert_eq!(p.amz_date, "20250426T143249Z");
+        assert_eq!(p.expires, "3600");
+        assert_eq!(p.signed_headers, "host");
+        assert_eq!(p.signature, "53cb3d8a12c8c1078fba3fcd55ced9c93fcdc8e2f98184e9ffea50245f4ebea5");
+    }
+
+    #[test]
+    fn fails_if_missing_signature() {
+        let url = "https://example.com/?X-Amz-Credential=AK/20250426/us-west-2/s3/aws4_request";
+        assert!(parse_presigned_params(url).is_err());
+    }
+
+    #[test]
+    fn parses_presigned_params_from_raw_query() {
+        // the very same query string that shows up in the log
+        let q = "X-Amz-Algorithm=AWS4-HMAC-SHA256&\
+                 X-Amz-Credential=MYLOCAL123%2F20250426%2Feu-west-3%2Fs3%2Faws4_request&\
+                 X-Amz-Date=20250426T143249Z&\
+                 X-Amz-Expires=3600&\
+                 X-Amz-SignedHeaders=host&\
+                 X-Amz-Signature=53cb3d8a12c8c1078fba3fcd55ced9c93fcdc8e2f98184e9ffea50245f4ebea5";
+    
+        // ❌ This is what the proxy does today — and it should FAIL until we fix the parser.
+        assert!(
+            parse_presigned_params(q).is_err(),
+            "the parser should reject a query that has no leading '?'"
+        );
+    
+        // ✅ This is what the proxy *should* do (or what the parser should accept):
+        let wrapped = format!("?{q}");
+        let (_, p) = parse_presigned_params(&wrapped)
+            .expect("parser must succeed when a leading '?' is present");
+    
+        assert_eq!(p.algorithm,        "AWS4-HMAC-SHA256");
+        assert_eq!(p.access_key,       "MYLOCAL123");
+        assert_eq!(p.credential_date,  "20250426");
+        assert_eq!(p.region,           "eu-west-3");
+        assert_eq!(p.service,          "s3");
+        assert_eq!(p.amz_date,         "20250426T143249Z");
+        assert_eq!(p.expires,          "3600");
+        assert_eq!(p.signed_headers,   "host");
+        assert_eq!(p.signature,        "53cb3d8a12c8c1078fba3fcd55ced9c93fcdc8e2f98184e9ffea50245f4ebea5");
+    }
+}
+
+

{object_storage_proxy-0.2.16 → object_storage_proxy-0.3.1}/src/parsers/keystore.rs

@@ -1,6 +1,6 @@
 use std::collections::HashMap;
 
-use pyo3::{exceptions, prelude::*, types::{PyDict, PyList}};
+use pyo3::{exceptions, prelude::*};
 
 
 pub fn parse_hmac_list(

object_storage_proxy-0.3.1/test_integration.sh

@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+
+cd /Users/jeroen/projects/mandelbrot
+
+echo "generate testfile"
+dd if=/dev/random of=/Users/jeroen/projects/mandelbrot/testfile_10 bs=1M count=10
+
+echo "uploading 10MB file."
+aws s3 cp testfile_10 s3://proxy-aws-bucket01/mandelbrot/testfile_10bis --profile osp
+if [ $? -eq 0 ]; then
+    echo -e "\033[1;32mOK\033[0m"
+else
+    echo -e "\033[1;31mERROR\033[0m"
+fi
+echo "listing .."
+aws s3 ls s3://proxy-aws-bucket01/mandelbrot/testfile_10bis --profile osp
+if [ $? -eq 0 ]; then
+    echo -e "\033[1;32mOK\033[0m"
+else
+    echo -e "\033[1;31mERROR\033[0m"
+fi
+
+echo "download a 10MB file"
+aws s3 cp s3://proxy-aws-bucket01/mandelbrot/testfile_10bis testfile_10bis --profile osp
+if [ $? -eq 0 ]; then
+    echo -e "\033[1;32mOK\033[0m"
+else
+    echo -e "\033[1;31mERROR\033[0m"
+fi
+
+
+echo "listing .."
+ls -latrh testfile_10bis
+
+if [ $? -eq 0 ]; then
+    echo -e "\033[1;32mOK\033[0m"
+else
+    echo -e "\033[1;31mERROR\033[0m"
+fi
+
+
+echo "deleting .."
+aws s3 rm s3://proxy-aws-bucket01/mandelbrot/testfile_10bis --profile osp
+if [ $? -eq 0 ]; then
+    echo -e "\033[1;32mOK\033[0m"
+else
+    echo -e "\033[1;31mERROR\033[0m"
+fi
+
+
+echo "listing .."
+aws s3 ls s3://proxy-aws-bucket01/mandelbrot/testfile_10bis --profile osp
+if [ $? -eq 1 ]; then
+    echo -e "\033[1;32mOK\033[0m"
+else
+    echo -e "\033[1;31mERROR\033[0m"
+fi

object_storage_proxy-0.2.16/src/parsers/credentials.rs

@@ -1,82 +0,0 @@
-use nom::{
-    bytes::complete::{tag, take_until}, sequence::{preceded, tuple}, IResult, Parser
-};
-
-pub fn parse_token_from_header(header: &str) -> IResult<&str, &str> {
-    let (_, token) =
-        (preceded(tag("AWS4-HMAC-SHA256 Credential="), take_until("/"))).parse(header)?;
-
-    Ok(("", token))
-}
-
-pub fn parse_credential_scope(input: &str) -> IResult<&str, (&str, &str)> {
-    let (input, _) = take_until("Credential=")(input)?;
-    let (remaining, (_, _, _, _, _, region, _, service, _)) = (
-        tag("Credential="),          // prefix
-        take_until("/"),            // access key
-        tag("/"),
-        take_until("/"),            // date
-        tag("/"),
-        take_until("/"),            // region
-        tag("/"),
-        take_until("/aws4_request"),// service
-        tag("/aws4_request"),       // trailing
-    ).parse(input)?;
-    Ok((remaining, (region, service)))
-}
-
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use nom::Err;
-
-    #[test]
-    fn test_parse_token_from_header() {
-        let input = "AWS4-HMAC-SHA256 Credential=MYLOCAL123/20250417/eu-west-3/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=ec323a7db4d0b8bd27eced3b2bb0d59f9b9dd";
-        let result = parse_token_from_header(input);
-        assert_eq!(result, Ok(("", ("MYLOCAL123"))));
-    }
-
-    #[test]
-    fn parse_token_from_header_success_and_error() {
-        let input = "AWS4-HMAC-SHA256 Credential=TOKEN123/20250417/eu-west-1/s3/aws4_request, SignedHeaders=host,Signature=abc";
-        let result = parse_token_from_header(input);
-        assert!(result.is_ok());
-        let (remaining, token) = result.unwrap();
-        assert_eq!(token, "TOKEN123");
-        assert_eq!(remaining, "");
-
-        let bad = "NoCredentialHere";
-        assert!(parse_token_from_header(bad).is_err());
-    }
-
-
-    #[test]
-    fn test_parse_valid_scope() {
-        let header = "Credential=AKIAEXAMPLE/20250425/us-west-2/s3/aws4_request, SignedHeaders=host;x-amz-date";
-        let (rem, (region, service)) = parse_credential_scope(header).expect("parse failed");
-        assert_eq!(region, "us-west-2");
-        assert_eq!(service, "s3");
-        assert!(rem.starts_with(", SignedHeaders"));
-    }
-
-    #[test]
-    fn test_parse_invalid_scope() {
-        let header = "Credential=AKIAEXAMPLE/20250425/us-west-2/s3/some_request";
-        assert!(matches!(parse_credential_scope(header), Err(Err::Error(_))));
-    }
-
-    #[test]
-    fn test_parse_with_prefix() {
-        let header = "Authorization: AWS4-HMAC-SHA256 Credential=XYZ/20250425/eu-central-1/dynamodb/aws4_request/extra";
-        let idx = header.find("Credential=").unwrap();
-        let substr = &header[idx..];
-        let (rem, (region, service)) = parse_credential_scope(substr).expect("parse failed");
-        assert_eq!(region, "eu-central-1");
-        assert_eq!(service, "dynamodb");
-        assert!(rem.starts_with("/extra"));
-    }
-}
-
-