object-storage-proxy 0.2.15__tar.gz → 0.2.16__tar.gz

{object_storage_proxy-0.2.15 → object_storage_proxy-0.2.16}/Cargo.lock

@@ -1701,7 +1701,7 @@ dependencies = [
 
 [[package]]
 name = "object-storage-proxy"
-version = "0.2.15"
+version = "0.2.16"
 dependencies = [
  "async-trait",
  "chrono",

{object_storage_proxy-0.2.15 → object_storage_proxy-0.2.16}/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "object-storage-proxy"
-version = "0.2.15"
+version = "0.2.16"
 edition = "2024"
 
 [dependencies]

{object_storage_proxy-0.2.15 → object_storage_proxy-0.2.16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: object-storage-proxy
-Version: 0.2.15
+Version: 0.2.16
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy
@@ -30,11 +30,16 @@ Project-URL: BugTracker, https://github.com/opensourceworks-org/object-storage-p
 
 A fast and safe in-process reverse proxy server, based on Cloudflare's [pingora](https://github.com/cloudflare/pingora?tab=readme-ov-file), to reverse proxy AWS and IBM Cloud Object Storage buckets and integrate your Authentication and Authorization services.
 
+Decouples frontend from backend authentication and authorization. 
+
+
 - [x] Takes a Python authorization callable (allows you to plug in your own authorization services) and api_key fetch callback function and cos bucket dictionary.
 - [x] The validation is cached with optional ttl (default 5min, keep it short). 
 - [x] The apikey is used to authenticate against IBM's IAM endpoint and is cached and renewed on expiration. (IBM only)
-- [x] If no apikey is provided, a Python function can be passed in to fetch the apikey or hmac keys for any given bucket (run once).
-- [x] HMAC support: passing in access and secret id keys (or as json string from python credentials callable), will be used to sign the request (AWS/IBM/..)
+- [x] If no apikey or hmac keypair is provided, a Python function can be passed in to fetch the apikey or hmac keys for any given bucket (run once).
+- [x] HMAC support: passing in access and secret id keys (or as json string from python credentials callable), will be used to sign the upstream request (AWS/IBM/..)
+- [x] frontend request is signed with your own custom keypair
+- [x] supports running in cloud environments / proxy headers for request signature validation
 
 The bucket dict contains for each bucket:
 
@@ -85,6 +90,8 @@ The Python callables take two arguments:
     def your_request_authorizer(token: str, bucket: str) -> bool
 ```
 
+Proxy Configuration
+
 
 
 
@@ -120,7 +127,7 @@ Pass in a function which maps bucket to instance (credentials), and a function t
 The advantage is we can plug in a python authentication function and another function for authorization, allowing for fine-grained control.
 
 ## authentication
-We use the standard aws hmac header.
+We use the standard aws hmac header and aws v4 request signing algorithm.
 
 ## authorization
 Pass in a callable from python which will be called from rust.  This will be cached (ttl) for consecutive requests.
@@ -177,7 +184,8 @@ def strtobool(val: str) -> bool:
     raise ValueError(f"invalid truth value {val!r}")
 
 
-def do_api_creds(bucket) -> str:
+def do_api_creds(token: str, bucket: str) -> str:
+    """Fetch credentials (ro, rw, access_denied) for the given bucket, depending on the token. """
     apikey = os.getenv("COS_API_KEY")
     if not apikey:
         raise ValueError("COS_API_KEY environment variable not set")
@@ -186,11 +194,13 @@ def do_api_creds(bucket) -> str:
     return apikey
 
 
-def do_hmac_creds(bucket) -> str:
+def do_hmac_creds(token: str, bucket: str) -> str:
+    """ Fetch HMAC credentials (ro, rw, access_denied) for the given bucket, depending on the token """
     access_key = os.getenv("ACCESS_KEY")
     secret_key = os.getenv("SECRET_KEY")
     if not access_key or not secret_key:
         raise ValueError("ACCESS_KEY or SECRET_KEY environment variable not set")
+        
     print(f"Fetching HMAC credentials for {bucket}...")
 
     return json.dumps({
@@ -198,8 +208,27 @@ def do_hmac_creds(bucket) -> str:
         "secret_key": secret_key
     })
 
+def lookup_secret_key(access_key: str) -> str | None:
+    # get all environment variables ending in ACCESS_KEY
+    access_keys = [{key:value} for key, value in os.environ.items() if key.endswith("ACCESS_KEY") and value==access_key ]
+
+    if len(access_keys) > 0:
+        access_key_var = next((k for k, v in access_keys[0].items() if v == access_key), None)
+
+        secret_key_var = access_key_var.replace("ACCESS_KEY", "SECRET_KEY")
+        return os.getenv(secret_key_var, None)
+    else:
+        print(f"no access keys found for : {access_key}")
+
 
 def do_validation(token: str, bucket: str) -> bool:
+    """ Authorize the request based on token for the given bucket. 
+        You can plug in your own authorization service here.
+        The token is the authorization token passed in the request.
+        The bucket is the bucket name.
+        The function should return True if the request is authorized, False otherwise.
+    """
+
     print(f"PYTHON: Validating headers: {token} for {bucket}...")
     # return random.choice([True, False])
     return True
@@ -214,7 +243,6 @@ def main() -> None:
         osp.enable_request_counting()
         print("Request counting enabled")
 
-
     apikey = os.getenv("COS_API_KEY")
     if not apikey:
         raise ValueError("COS_API_KEY environment variable not set")
@@ -240,16 +268,48 @@ def main() -> None:
             # "secret_key": os.getenv("SECRET_KEY"),
             "port": 443,
             "ttl": 300
+        },
+        "proxy-bucket05": {
+            "host": "s3.eu-de.cloud-object-storage.appdomain.cloud",
+            "region": "eu-de",
+            "access_key": os.getenv("PROXY_BUCKET05_ACCESS_KEY"),
+            "secret_key": os.getenv("PROXY_BUCKET05_SECRET_KEY"),
+            "port": 443,
+            "ttl": 300
+        },
+        "proxy-aws-bucket01": {
+            "host": "s3.eu-west-3.amazonaws.com",
+            "region": "eu-west-3",
+            "access_key": os.getenv("AWS_ACCESS_KEY"),
+            "secret_key": os.getenv("AWS_SECRET_KEY"),
+            "port": 443,
+            "ttl": 300
         }
     }
 
+    hmac_keys= [
+        # {
+        #     "access_key": os.getenv("LOCAL_ACCESS_KEY"),
+        #     "secret_key": os.getenv("LOCAL_SECRET_KEY")
+        # },
+        {
+            "access_key": os.getenv("LOCAL2_ACCESS_KEY"),
+            "secret_key": os.getenv("LOCAL2_SECRET_KEY")
+        },
+
+    ]
+
     ra = ProxyServerConfig(
         cos_map=cos_map,
-        bucket_creds_fetcher=do_hmac_creds,  # or: do_api_creds
+        bucket_creds_fetcher=do_hmac_creds,
         validator=do_validation,
         http_port=6190,
-        https_port=8443,
+        # https_port=8443,
         threads=1,
+        # verify=False,
+        hmac_keystore=hmac_keys,
+        skip_signature_validation=False,
+        hmac_fetcher=lookup_secret_key
     )
 
     start_server(ra)
@@ -258,9 +318,6 @@ def main() -> None:
 if __name__ == "__main__":
     main()
 
-
-
-
 ```
 
 Run with [aws-cli](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html) (but could be anything compatible with the aws s3 api like polars, spark, presto, ...):

{object_storage_proxy-0.2.15 → object_storage_proxy-0.2.16}/README.md

@@ -11,11 +11,16 @@
 
 A fast and safe in-process reverse proxy server, based on Cloudflare's [pingora](https://github.com/cloudflare/pingora?tab=readme-ov-file), to reverse proxy AWS and IBM Cloud Object Storage buckets and integrate your Authentication and Authorization services.
 
+Decouples frontend from backend authentication and authorization. 
+
+
 - [x] Takes a Python authorization callable (allows you to plug in your own authorization services) and api_key fetch callback function and cos bucket dictionary.
 - [x] The validation is cached with optional ttl (default 5min, keep it short). 
 - [x] The apikey is used to authenticate against IBM's IAM endpoint and is cached and renewed on expiration. (IBM only)
-- [x] If no apikey is provided, a Python function can be passed in to fetch the apikey or hmac keys for any given bucket (run once).
-- [x] HMAC support: passing in access and secret id keys (or as json string from python credentials callable), will be used to sign the request (AWS/IBM/..)
+- [x] If no apikey or hmac keypair is provided, a Python function can be passed in to fetch the apikey or hmac keys for any given bucket (run once).
+- [x] HMAC support: passing in access and secret id keys (or as json string from python credentials callable), will be used to sign the upstream request (AWS/IBM/..)
+- [x] frontend request is signed with your own custom keypair
+- [x] supports running in cloud environments / proxy headers for request signature validation
 
 The bucket dict contains for each bucket:
 
@@ -66,6 +71,8 @@ The Python callables take two arguments:
     def your_request_authorizer(token: str, bucket: str) -> bool
 ```
 
+Proxy Configuration
+
 
 
 
@@ -101,7 +108,7 @@ Pass in a function which maps bucket to instance (credentials), and a function t
 The advantage is we can plug in a python authentication function and another function for authorization, allowing for fine-grained control.
 
 ## authentication
-We use the standard aws hmac header.
+We use the standard aws hmac header and aws v4 request signing algorithm.
 
 ## authorization
 Pass in a callable from python which will be called from rust.  This will be cached (ttl) for consecutive requests.
@@ -158,7 +165,8 @@ def strtobool(val: str) -> bool:
     raise ValueError(f"invalid truth value {val!r}")
 
 
-def do_api_creds(bucket) -> str:
+def do_api_creds(token: str, bucket: str) -> str:
+    """Fetch credentials (ro, rw, access_denied) for the given bucket, depending on the token. """
     apikey = os.getenv("COS_API_KEY")
     if not apikey:
         raise ValueError("COS_API_KEY environment variable not set")
@@ -167,11 +175,13 @@ def do_api_creds(bucket) -> str:
     return apikey
 
 
-def do_hmac_creds(bucket) -> str:
+def do_hmac_creds(token: str, bucket: str) -> str:
+    """ Fetch HMAC credentials (ro, rw, access_denied) for the given bucket, depending on the token """
     access_key = os.getenv("ACCESS_KEY")
     secret_key = os.getenv("SECRET_KEY")
     if not access_key or not secret_key:
         raise ValueError("ACCESS_KEY or SECRET_KEY environment variable not set")
+        
     print(f"Fetching HMAC credentials for {bucket}...")
 
     return json.dumps({
@@ -179,8 +189,27 @@ def do_hmac_creds(bucket) -> str:
         "secret_key": secret_key
     })
 
+def lookup_secret_key(access_key: str) -> str | None:
+    # get all environment variables ending in ACCESS_KEY
+    access_keys = [{key:value} for key, value in os.environ.items() if key.endswith("ACCESS_KEY") and value==access_key ]
+
+    if len(access_keys) > 0:
+        access_key_var = next((k for k, v in access_keys[0].items() if v == access_key), None)
+
+        secret_key_var = access_key_var.replace("ACCESS_KEY", "SECRET_KEY")
+        return os.getenv(secret_key_var, None)
+    else:
+        print(f"no access keys found for : {access_key}")
+
 
 def do_validation(token: str, bucket: str) -> bool:
+    """ Authorize the request based on token for the given bucket. 
+        You can plug in your own authorization service here.
+        The token is the authorization token passed in the request.
+        The bucket is the bucket name.
+        The function should return True if the request is authorized, False otherwise.
+    """
+
     print(f"PYTHON: Validating headers: {token} for {bucket}...")
     # return random.choice([True, False])
     return True
@@ -195,7 +224,6 @@ def main() -> None:
         osp.enable_request_counting()
         print("Request counting enabled")
 
-
     apikey = os.getenv("COS_API_KEY")
     if not apikey:
         raise ValueError("COS_API_KEY environment variable not set")
@@ -221,16 +249,48 @@ def main() -> None:
             # "secret_key": os.getenv("SECRET_KEY"),
             "port": 443,
             "ttl": 300
+        },
+        "proxy-bucket05": {
+            "host": "s3.eu-de.cloud-object-storage.appdomain.cloud",
+            "region": "eu-de",
+            "access_key": os.getenv("PROXY_BUCKET05_ACCESS_KEY"),
+            "secret_key": os.getenv("PROXY_BUCKET05_SECRET_KEY"),
+            "port": 443,
+            "ttl": 300
+        },
+        "proxy-aws-bucket01": {
+            "host": "s3.eu-west-3.amazonaws.com",
+            "region": "eu-west-3",
+            "access_key": os.getenv("AWS_ACCESS_KEY"),
+            "secret_key": os.getenv("AWS_SECRET_KEY"),
+            "port": 443,
+            "ttl": 300
         }
     }
 
+    hmac_keys= [
+        # {
+        #     "access_key": os.getenv("LOCAL_ACCESS_KEY"),
+        #     "secret_key": os.getenv("LOCAL_SECRET_KEY")
+        # },
+        {
+            "access_key": os.getenv("LOCAL2_ACCESS_KEY"),
+            "secret_key": os.getenv("LOCAL2_SECRET_KEY")
+        },
+
+    ]
+
     ra = ProxyServerConfig(
         cos_map=cos_map,
-        bucket_creds_fetcher=do_hmac_creds,  # or: do_api_creds
+        bucket_creds_fetcher=do_hmac_creds,
         validator=do_validation,
         http_port=6190,
-        https_port=8443,
+        # https_port=8443,
         threads=1,
+        # verify=False,
+        hmac_keystore=hmac_keys,
+        skip_signature_validation=False,
+        hmac_fetcher=lookup_secret_key
     )
 
     start_server(ra)
@@ -239,9 +299,6 @@ def main() -> None:
 if __name__ == "__main__":
     main()
 
-
-
-
 ```
 
 Run with [aws-cli](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html) (but could be anything compatible with the aws s3 api like polars, spark, presto, ...):

object_storage_proxy-0.2.16/src/credentials/hmac_keystore.rs

@@ -0,0 +1,34 @@
+use pyo3::pyclass;
+
+#[pyclass]
+#[derive(Debug, Clone)]
+pub struct HmacKeyStore {
+    access_key: String,
+    secret_key: String,
+
+}
+
+impl HmacKeyStore {
+    pub fn new(access_key: String, secret_key: String) -> Self {
+        HmacKeyStore {
+            access_key,
+            secret_key,
+        }
+    }
+
+    pub fn get_access_key(&self) -> &str {
+        &self.access_key
+    }
+
+    pub fn get_secret_key(&self) -> &str {
+        &self.secret_key
+    }
+}
+impl Default for HmacKeyStore {
+    fn default() -> Self {
+        HmacKeyStore {
+            access_key: String::new(),
+            secret_key: String::new(),
+        }
+    }
+}

{object_storage_proxy-0.2.15 → object_storage_proxy-0.2.16}/src/credentials/mod.rs

@@ -1,3 +1,4 @@
 pub mod models;
 pub mod secrets_proxy;
 pub mod signer;
+pub mod hmac_keystore;

{object_storage_proxy-0.2.15 → object_storage_proxy-0.2.16}/src/credentials/signer.rs

@@ -1,17 +1,19 @@
-use chrono::{DateTime, Utc};
+use chrono::{DateTime, NaiveDateTime, Utc};
 use http::header::HeaderMap;
-use pingora::http::RequestHeader;
+use pingora::{http::RequestHeader, proxy::Session};
+use rustls::crypto::hash::Hash;
 use sha256::digest;
-use tracing::debug;
-use std::{collections::HashMap, fmt};
+use tokio::sync::RwLock;
+use tracing::{debug, error, info};
+use std::{collections::{HashMap, HashSet}, fmt, sync::Arc};
 use url::Url;
 
-use crate::parsers::cos_map::CosMapItem;
+use crate::parsers::{cos_map::CosMapItem, credentials::{parse_credential_scope, parse_token_from_header}};
 
 const SHORT_DATE: &str = "%Y%m%d";
 const LONG_DATETIME: &str = "%Y%m%dT%H%M%SZ";
 
-// AwsSign copied and slightly modified from https://github.com/psnszsn/aws-sign-v4
+// AwsSign copied and modified from https://github.com/psnszsn/aws-sign-v4
 
 pub struct AwsSign<'a, T: 'a>
 where
@@ -24,6 +26,7 @@ where
     access_key: &'a str,
     secret_key: &'a str,
     headers: T,
+    payload_override: Option<String>,
 
     /*
     service is the <aws-service-code> that can be found in the service-quotas api.
@@ -70,15 +73,28 @@ impl<'a> AwsSign<'a, HashMap<String, String>> {
         secret_key: &'a str,
         service: &'a str,
         body: &'a B,
+        signed_headers: Option<&'a Vec<String>>,
     ) -> Self {
-        let allowed = [
-            "host",
-            "content-length",
-            "x-amz-date",
-            "x-amz-content-sha256",
-            "x-amz-security-token",
-        ];
-
+        let allowed: Vec<&str> = if let Some(sh) = signed_headers {
+            sh.iter().map(String::as_str).collect()
+        } else {
+            vec![
+                "host",
+                "x-amz-date",
+                "range",
+                "x-amz-content-sha256",
+                "x-amz-security-token",
+            ]
+        };
+        // let allowed = [
+        //     "host",
+        //     "x-amz-date",
+        //     "range",
+        //     "x-amz-content-sha256",
+        //     "x-amz-security-token",
+        // ];
+        
+        dbg!(&url);
         let url: Url = url.parse().unwrap();
         let headers: HashMap<String, String> = headers
             .iter()
@@ -103,6 +119,7 @@ impl<'a> AwsSign<'a, HashMap<String, String>> {
             headers,
             service,
             body: body.as_ref(),
+            payload_override: None,
         }
     }
 }
@@ -131,6 +148,10 @@ impl<'a, T> AwsSign<'a, T>
 where
     &'a T: std::iter::IntoIterator<Item = (&'a String, &'a String)>, T: std::fmt::Debug
 {
+    pub fn set_payload_override(&mut self, h: String) {
+        self.payload_override = Some(h);
+    }
+
     pub fn canonical_header_string(&'a self) -> String {
         let mut keyvalues = self
             .headers
@@ -153,7 +174,9 @@ where
 
     pub fn canonical_request(&'a self) -> String {
         let url: &str = self.url.path().into();
-        let payload_line = if self.body == b"UNSIGNED-PAYLOAD" {
+        let payload_line = if let Some(ov) = &self.payload_override {
+            ov.clone()
+        } else if self.body == b"UNSIGNED-PAYLOAD" {
             "UNSIGNED-PAYLOAD".into()
         } else {
             digest(self.body)
@@ -321,6 +344,7 @@ pub(crate) async fn sign_request(
         secret_key,
         "s3",
         body_bytes,
+        None,
     );
     debug!("{:#?}", &auth_header);
 
@@ -335,6 +359,126 @@ pub(crate) async fn sign_request(
     Ok(())
 }
 
+
+pub async fn signature_is_valid(
+    auth_header: &str,
+    session: &Session,
+    secret_key: &str,
+) -> Result<bool, Box<dyn std::error::Error>> {
+
+    let signed_headers_str = auth_header
+        .split("SignedHeaders=")
+        .nth(1)
+        .and_then(|s| s.split(',').next())
+        .unwrap_or("");
+    let signed_headers: Vec<String> =
+        signed_headers_str.split(';').map(|s| s.to_lowercase()).collect();
+
+
+    // extract access key from Authorization header
+    let (_, local_access_key) = parse_token_from_header(auth_header)
+        .map_err(|_| pingora::Error::new_str("Failed to parse token"))?;
+    let local_access_key = local_access_key.to_string();
+    if local_access_key.is_empty() {
+        error!("Missing access key");
+        return Ok(false);
+    }
+
+    // extract provided signature
+    let provided_signature = auth_header
+        .split("Signature=")
+        .nth(1)
+        .ok_or_else(|| pingora::Error::new_str("Invalid Authorization header: no Signature"))?
+        .to_string();
+
+    // parse x-amz-date header
+    let dt_header = session
+        .req_header()
+        .headers
+        .get("x-amz-date")
+        .ok_or_else(|| pingora::Error::new_str("Missing x-amz-date header"))?
+        .to_str()?;
+
+    // use NaiveDateTime then assign Utc timezone (format: %Y%m%dT%H%M%SZ)
+    let naive = NaiveDateTime::parse_from_str(dt_header, LONG_DATETIME)
+        .map_err(|_| {
+            pingora::Error::new_str("invalid date")
+        })?;
+    let datetime = naive.and_utc();
+
+
+    info!("parsing the region and service");
+
+    let (_, (region, service)) = parse_credential_scope(auth_header)
+        .map_err(|_| pingora::Error::new_str("Invalid Credential scope"))?;
+
+    dbg!(&region);
+    dbg!(&service);
+
+    // determine payload hash header
+    let content_sha256 = session
+        .req_header()
+        .headers
+        .get("x-amz-content-sha256")
+        .and_then(|h| h.to_str().ok())
+        .ok_or_else(|| pingora::Error::new_str("Missing x-amz-content-sha256 header"))?;
+    // let body_bytes: &[u8] = match content_sha256 {
+    //     "UNSIGNED-PAYLOAD" => b"UNSIGNED-PAYLOAD",
+    //     _ => &[],
+    // };
+
+    let (body_bytes, payload_override) = if content_sha256 == "UNSIGNED-PAYLOAD" {
+        (b"UNSIGNED-PAYLOAD" as &[u8], None)
+    } else {
+        // we don't have the raw body here, but we do have its hash:
+        // tell AwsSign to use this string directly
+        (&[] as &[u8], Some(content_sha256.to_owned()))
+    };
+
+    let original_uri = session.req_header().uri.to_string();
+    let full_url = if original_uri.starts_with('/') {
+        let host = session
+            .req_header()
+            .headers
+            .get("host")
+            .ok_or_else(|| pingora::Error::new_str("Missing host header"))?
+            .to_str()?;
+        format!("https://{}{}", host, original_uri)
+    } else {
+        original_uri
+    };
+
+    // construct AwsSign and compute signature
+    let method = session.req_header().method.to_string();
+    let mut signer = AwsSign::new(
+        &method,
+        &full_url,
+        &datetime,
+        &session.req_header().headers,
+        region,
+        &local_access_key,
+        &secret_key,
+        service,
+        body_bytes,
+        Some(&signed_headers),
+    );
+
+    if let Some(ov) = payload_override {
+        signer.set_payload_override(ov);
+    }
+
+    let signature = signer.sign();
+    let computed_signature = signature
+        .split("Signature=")
+        .nth(1)
+        .unwrap_or_default();
+
+    info!("Provided signature: {}", provided_signature);
+    info!("Computed signature: {}", computed_signature);
+    Ok(computed_signature == provided_signature)
+}
+
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -349,7 +493,7 @@ mod tests {
         let datetime = chrono::Utc::now();
         let url: &str = "https://hi.s3.us-east-1.amazonaws.com/Prod/graphql";
         let map: HeaderMap = HeaderMap::new();
-        let aws_sign = AwsSign::new("GET", url, &datetime, &map, "us-east-1", "a", "b", "s3", "");
+        let aws_sign = AwsSign::new("GET", url, &datetime, &map, "us-east-1", "a", "b", "s3", "", None);
         let s = aws_sign.canonical_request();
         assert_eq!(
             s,
@@ -372,6 +516,7 @@ mod tests {
             "b",
             "s3",
             "".as_bytes(),
+            None,
         );
         let s = aws_sign.canonical_request();
         assert_eq!(
@@ -396,6 +541,7 @@ mod tests {
             "b",
             "s3",
             &body,
+            None,
         );
         let s = aws_sign.canonical_request();
         assert_eq!(