object-storage-proxy 0.2.14__tar.gz → 0.2.16__tar.gz

{object_storage_proxy-0.2.14 → object_storage_proxy-0.2.16}/Cargo.lock

@@ -1701,7 +1701,7 @@ dependencies = [
 
 [[package]]
 name = "object-storage-proxy"
-version = "0.2.14"
+version = "0.2.16"
 dependencies = [
  "async-trait",
  "chrono",

{object_storage_proxy-0.2.14 → object_storage_proxy-0.2.16}/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "object-storage-proxy"
-version = "0.2.14"
+version = "0.2.16"
 edition = "2024"
 
 [dependencies]
@@ -40,6 +40,7 @@ regex = "1.11.1"
 [lib]
 name = "object_storage_proxy"
 crate-type = ["cdylib"]
+path = "src/lib.rs"
 
 [package.metadata.maturin]
 bindings = "pyo3"

{object_storage_proxy-0.2.14 → object_storage_proxy-0.2.16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: object-storage-proxy
-Version: 0.2.14
+Version: 0.2.16
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy
@@ -30,11 +30,16 @@ Project-URL: BugTracker, https://github.com/opensourceworks-org/object-storage-p
 
 A fast and safe in-process reverse proxy server, based on Cloudflare's [pingora](https://github.com/cloudflare/pingora?tab=readme-ov-file), to reverse proxy AWS and IBM Cloud Object Storage buckets and integrate your Authentication and Authorization services.
 
-- [x] Takes a Python authorization callable function (allows you to plug in your own authorization services) and api_key fetch callback function and cos bucket dictionary.
+Decouples frontend from backend authentication and authorization. 
+
+
+- [x] Takes a Python authorization callable (allows you to plug in your own authorization services) and api_key fetch callback function and cos bucket dictionary.
 - [x] The validation is cached with optional ttl (default 5min, keep it short). 
 - [x] The apikey is used to authenticate against IBM's IAM endpoint and is cached and renewed on expiration. (IBM only)
-- [x] If no apikey is provided, a Python function can be passed in to fetch the apikey or hmac keys for any given bucket (run once).
-- [x] HMAC support: passing in access and secret id keys (or as json string from python credentials callable), will be used to sign the request (AWS/IBM/..)
+- [x] If no apikey or hmac keypair is provided, a Python function can be passed in to fetch the apikey or hmac keys for any given bucket (run once).
+- [x] HMAC support: passing in access and secret id keys (or as json string from python credentials callable), will be used to sign the upstream request (AWS/IBM/..)
+- [x] frontend request is signed with your own custom keypair
+- [x] supports running in cloud environments / proxy headers for request signature validation
 
 The bucket dict contains for each bucket:
 
@@ -85,6 +90,8 @@ The Python callables take two arguments:
     def your_request_authorizer(token: str, bucket: str) -> bool
 ```
 
+Proxy Configuration
+
 
 
 
@@ -120,7 +127,7 @@ Pass in a function which maps bucket to instance (credentials), and a function t
 The advantage is we can plug in a python authentication function and another function for authorization, allowing for fine-grained control.
 
 ## authentication
-We use the standard aws hmac header.
+We use the standard aws hmac header and aws v4 request signing algorithm.
 
 ## authorization
 Pass in a callable from python which will be called from rust.  This will be cached (ttl) for consecutive requests.
@@ -177,7 +184,8 @@ def strtobool(val: str) -> bool:
     raise ValueError(f"invalid truth value {val!r}")
 
 
-def do_api_creds(bucket) -> str:
+def do_api_creds(token: str, bucket: str) -> str:
+    """Fetch credentials (ro, rw, access_denied) for the given bucket, depending on the token. """
     apikey = os.getenv("COS_API_KEY")
     if not apikey:
         raise ValueError("COS_API_KEY environment variable not set")
@@ -186,11 +194,13 @@ def do_api_creds(bucket) -> str:
     return apikey
 
 
-def do_hmac_creds(bucket) -> str:
+def do_hmac_creds(token: str, bucket: str) -> str:
+    """ Fetch HMAC credentials (ro, rw, access_denied) for the given bucket, depending on the token """
     access_key = os.getenv("ACCESS_KEY")
     secret_key = os.getenv("SECRET_KEY")
     if not access_key or not secret_key:
         raise ValueError("ACCESS_KEY or SECRET_KEY environment variable not set")
+        
     print(f"Fetching HMAC credentials for {bucket}...")
 
     return json.dumps({
@@ -198,8 +208,27 @@ def do_hmac_creds(bucket) -> str:
         "secret_key": secret_key
     })
 
+def lookup_secret_key(access_key: str) -> str | None:
+    # get all environment variables ending in ACCESS_KEY
+    access_keys = [{key:value} for key, value in os.environ.items() if key.endswith("ACCESS_KEY") and value==access_key ]
+
+    if len(access_keys) > 0:
+        access_key_var = next((k for k, v in access_keys[0].items() if v == access_key), None)
+
+        secret_key_var = access_key_var.replace("ACCESS_KEY", "SECRET_KEY")
+        return os.getenv(secret_key_var, None)
+    else:
+        print(f"no access keys found for : {access_key}")
+
 
 def do_validation(token: str, bucket: str) -> bool:
+    """ Authorize the request based on token for the given bucket. 
+        You can plug in your own authorization service here.
+        The token is the authorization token passed in the request.
+        The bucket is the bucket name.
+        The function should return True if the request is authorized, False otherwise.
+    """
+
     print(f"PYTHON: Validating headers: {token} for {bucket}...")
     # return random.choice([True, False])
     return True
@@ -214,7 +243,6 @@ def main() -> None:
         osp.enable_request_counting()
         print("Request counting enabled")
 
-
     apikey = os.getenv("COS_API_KEY")
     if not apikey:
         raise ValueError("COS_API_KEY environment variable not set")
@@ -240,16 +268,48 @@ def main() -> None:
             # "secret_key": os.getenv("SECRET_KEY"),
             "port": 443,
             "ttl": 300
+        },
+        "proxy-bucket05": {
+            "host": "s3.eu-de.cloud-object-storage.appdomain.cloud",
+            "region": "eu-de",
+            "access_key": os.getenv("PROXY_BUCKET05_ACCESS_KEY"),
+            "secret_key": os.getenv("PROXY_BUCKET05_SECRET_KEY"),
+            "port": 443,
+            "ttl": 300
+        },
+        "proxy-aws-bucket01": {
+            "host": "s3.eu-west-3.amazonaws.com",
+            "region": "eu-west-3",
+            "access_key": os.getenv("AWS_ACCESS_KEY"),
+            "secret_key": os.getenv("AWS_SECRET_KEY"),
+            "port": 443,
+            "ttl": 300
         }
     }
 
+    hmac_keys= [
+        # {
+        #     "access_key": os.getenv("LOCAL_ACCESS_KEY"),
+        #     "secret_key": os.getenv("LOCAL_SECRET_KEY")
+        # },
+        {
+            "access_key": os.getenv("LOCAL2_ACCESS_KEY"),
+            "secret_key": os.getenv("LOCAL2_SECRET_KEY")
+        },
+
+    ]
+
     ra = ProxyServerConfig(
         cos_map=cos_map,
-        bucket_creds_fetcher=do_hmac_creds,  # or: do_api_creds
+        bucket_creds_fetcher=do_hmac_creds,
         validator=do_validation,
         http_port=6190,
-        https_port=8443,
+        # https_port=8443,
         threads=1,
+        # verify=False,
+        hmac_keystore=hmac_keys,
+        skip_signature_validation=False,
+        hmac_fetcher=lookup_secret_key
     )
 
     start_server(ra)
@@ -258,9 +318,6 @@ def main() -> None:
 if __name__ == "__main__":
     main()
 
-
-
-
 ```
 
 Run with [aws-cli](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html) (but could be anything compatible with the aws s3 api like polars, spark, presto, ...):

{object_storage_proxy-0.2.14 → object_storage_proxy-0.2.16}/README.md

@@ -11,11 +11,16 @@
 
 A fast and safe in-process reverse proxy server, based on Cloudflare's [pingora](https://github.com/cloudflare/pingora?tab=readme-ov-file), to reverse proxy AWS and IBM Cloud Object Storage buckets and integrate your Authentication and Authorization services.
 
-- [x] Takes a Python authorization callable function (allows you to plug in your own authorization services) and api_key fetch callback function and cos bucket dictionary.
+Decouples frontend from backend authentication and authorization. 
+
+
+- [x] Takes a Python authorization callable (allows you to plug in your own authorization services) and api_key fetch callback function and cos bucket dictionary.
 - [x] The validation is cached with optional ttl (default 5min, keep it short). 
 - [x] The apikey is used to authenticate against IBM's IAM endpoint and is cached and renewed on expiration. (IBM only)
-- [x] If no apikey is provided, a Python function can be passed in to fetch the apikey or hmac keys for any given bucket (run once).
-- [x] HMAC support: passing in access and secret id keys (or as json string from python credentials callable), will be used to sign the request (AWS/IBM/..)
+- [x] If no apikey or hmac keypair is provided, a Python function can be passed in to fetch the apikey or hmac keys for any given bucket (run once).
+- [x] HMAC support: passing in access and secret id keys (or as json string from python credentials callable), will be used to sign the upstream request (AWS/IBM/..)
+- [x] frontend request is signed with your own custom keypair
+- [x] supports running in cloud environments / proxy headers for request signature validation
 
 The bucket dict contains for each bucket:
 
@@ -66,6 +71,8 @@ The Python callables take two arguments:
     def your_request_authorizer(token: str, bucket: str) -> bool
 ```
 
+Proxy Configuration
+
 
 
 
@@ -101,7 +108,7 @@ Pass in a function which maps bucket to instance (credentials), and a function t
 The advantage is we can plug in a python authentication function and another function for authorization, allowing for fine-grained control.
 
 ## authentication
-We use the standard aws hmac header.
+We use the standard aws hmac header and aws v4 request signing algorithm.
 
 ## authorization
 Pass in a callable from python which will be called from rust.  This will be cached (ttl) for consecutive requests.
@@ -158,7 +165,8 @@ def strtobool(val: str) -> bool:
     raise ValueError(f"invalid truth value {val!r}")
 
 
-def do_api_creds(bucket) -> str:
+def do_api_creds(token: str, bucket: str) -> str:
+    """Fetch credentials (ro, rw, access_denied) for the given bucket, depending on the token. """
     apikey = os.getenv("COS_API_KEY")
     if not apikey:
         raise ValueError("COS_API_KEY environment variable not set")
@@ -167,11 +175,13 @@ def do_api_creds(bucket) -> str:
     return apikey
 
 
-def do_hmac_creds(bucket) -> str:
+def do_hmac_creds(token: str, bucket: str) -> str:
+    """ Fetch HMAC credentials (ro, rw, access_denied) for the given bucket, depending on the token """
     access_key = os.getenv("ACCESS_KEY")
     secret_key = os.getenv("SECRET_KEY")
     if not access_key or not secret_key:
         raise ValueError("ACCESS_KEY or SECRET_KEY environment variable not set")
+        
     print(f"Fetching HMAC credentials for {bucket}...")
 
     return json.dumps({
@@ -179,8 +189,27 @@ def do_hmac_creds(bucket) -> str:
         "secret_key": secret_key
     })
 
+def lookup_secret_key(access_key: str) -> str | None:
+    # get all environment variables ending in ACCESS_KEY
+    access_keys = [{key:value} for key, value in os.environ.items() if key.endswith("ACCESS_KEY") and value==access_key ]
+
+    if len(access_keys) > 0:
+        access_key_var = next((k for k, v in access_keys[0].items() if v == access_key), None)
+
+        secret_key_var = access_key_var.replace("ACCESS_KEY", "SECRET_KEY")
+        return os.getenv(secret_key_var, None)
+    else:
+        print(f"no access keys found for : {access_key}")
+
 
 def do_validation(token: str, bucket: str) -> bool:
+    """ Authorize the request based on token for the given bucket. 
+        You can plug in your own authorization service here.
+        The token is the authorization token passed in the request.
+        The bucket is the bucket name.
+        The function should return True if the request is authorized, False otherwise.
+    """
+
     print(f"PYTHON: Validating headers: {token} for {bucket}...")
     # return random.choice([True, False])
     return True
@@ -195,7 +224,6 @@ def main() -> None:
         osp.enable_request_counting()
         print("Request counting enabled")
 
-
     apikey = os.getenv("COS_API_KEY")
     if not apikey:
         raise ValueError("COS_API_KEY environment variable not set")
@@ -221,16 +249,48 @@ def main() -> None:
             # "secret_key": os.getenv("SECRET_KEY"),
             "port": 443,
             "ttl": 300
+        },
+        "proxy-bucket05": {
+            "host": "s3.eu-de.cloud-object-storage.appdomain.cloud",
+            "region": "eu-de",
+            "access_key": os.getenv("PROXY_BUCKET05_ACCESS_KEY"),
+            "secret_key": os.getenv("PROXY_BUCKET05_SECRET_KEY"),
+            "port": 443,
+            "ttl": 300
+        },
+        "proxy-aws-bucket01": {
+            "host": "s3.eu-west-3.amazonaws.com",
+            "region": "eu-west-3",
+            "access_key": os.getenv("AWS_ACCESS_KEY"),
+            "secret_key": os.getenv("AWS_SECRET_KEY"),
+            "port": 443,
+            "ttl": 300
         }
     }
 
+    hmac_keys= [
+        # {
+        #     "access_key": os.getenv("LOCAL_ACCESS_KEY"),
+        #     "secret_key": os.getenv("LOCAL_SECRET_KEY")
+        # },
+        {
+            "access_key": os.getenv("LOCAL2_ACCESS_KEY"),
+            "secret_key": os.getenv("LOCAL2_SECRET_KEY")
+        },
+
+    ]
+
     ra = ProxyServerConfig(
         cos_map=cos_map,
-        bucket_creds_fetcher=do_hmac_creds,  # or: do_api_creds
+        bucket_creds_fetcher=do_hmac_creds,
         validator=do_validation,
         http_port=6190,
-        https_port=8443,
+        # https_port=8443,
         threads=1,
+        # verify=False,
+        hmac_keystore=hmac_keys,
+        skip_signature_validation=False,
+        hmac_fetcher=lookup_secret_key
     )
 
     start_server(ra)
@@ -239,9 +299,6 @@ def main() -> None:
 if __name__ == "__main__":
     main()
 
-
-
-
 ```
 
 Run with [aws-cli](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html) (but could be anything compatible with the aws s3 api like polars, spark, presto, ...):

{object_storage_proxy-0.2.14 → object_storage_proxy-0.2.16}/pyproject.toml

@@ -26,6 +26,9 @@ dependencies = [
 [tool.maturin]
 features = ["pyo3/extension-module"]
 
+[tool.uv.workspace]
+members = ["integration"]
+
 [dependency-groups]
 dev = []
 

object_storage_proxy-0.2.16/src/credentials/hmac_keystore.rs

@@ -0,0 +1,34 @@
+use pyo3::pyclass;
+
+#[pyclass]
+#[derive(Debug, Clone)]
+pub struct HmacKeyStore {
+    access_key: String,
+    secret_key: String,
+
+}
+
+impl HmacKeyStore {
+    pub fn new(access_key: String, secret_key: String) -> Self {
+        HmacKeyStore {
+            access_key,
+            secret_key,
+        }
+    }
+
+    pub fn get_access_key(&self) -> &str {
+        &self.access_key
+    }
+
+    pub fn get_secret_key(&self) -> &str {
+        &self.secret_key
+    }
+}
+impl Default for HmacKeyStore {
+    fn default() -> Self {
+        HmacKeyStore {
+            access_key: String::new(),
+            secret_key: String::new(),
+        }
+    }
+}

{object_storage_proxy-0.2.14 → object_storage_proxy-0.2.16}/src/credentials/mod.rs

@@ -1,3 +1,4 @@
 pub mod models;
 pub mod secrets_proxy;
 pub mod signer;
+pub mod hmac_keystore;

object_storage_proxy-0.2.16/src/credentials/models.rs

@@ -0,0 +1,58 @@
+pub enum BucketCredential {
+    Hmac {
+        access_key: String,
+        secret_key: String,
+    },
+    ApiKey(String),
+}
+
+impl BucketCredential {
+    pub fn parse(raw: &str) -> Self {
+        if let Ok(json_val) = serde_json::from_str::<serde_json::Value>(raw) {
+            if let (Some(ak), Some(sk)) = (json_val.get("access_key"), json_val.get("secret_key")) {
+                return BucketCredential::Hmac {
+                    access_key: ak.as_str().unwrap().to_owned(),
+                    secret_key: sk.as_str().unwrap().to_owned(),
+                };
+            }
+            if let Some(apikey) = json_val.get("api_key").or_else(|| json_val.get("apikey")) {
+                return BucketCredential::ApiKey(apikey.as_str().unwrap().to_owned());
+            }
+        }
+
+        BucketCredential::ApiKey(raw.to_owned())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+
+    #[test]
+    fn parse_bucket_credential_variants() {
+        let hmac_json = r#"{ "access_key": "AK", "secret_key": "SK" }"#;
+        match BucketCredential::parse(hmac_json) {
+            BucketCredential::Hmac { access_key, secret_key } => {
+                assert_eq!(access_key, "AK");
+                assert_eq!(secret_key, "SK");
+            }
+            _ => panic!("Expected Hmac variant"),
+        }
+
+        let api_json = r#"{ "api_key": "APIKEY" }"#;
+        if let BucketCredential::ApiKey(k) = BucketCredential::parse(api_json) {
+            assert_eq!(k, "APIKEY");
+        } else {
+            panic!("Expected ApiKey variant");
+        }
+
+        let raw = "raw_token";
+        if let BucketCredential::ApiKey(k) = BucketCredential::parse(raw) {
+            assert_eq!(k, raw);
+        } else {
+            panic!("Expected fallback ApiKey variant");
+        }
+    }
+
+}

{object_storage_proxy-0.2.14 → object_storage_proxy-0.2.16}/src/credentials/secrets_proxy.rs

@@ -170,6 +170,8 @@ pub(crate) async fn get_credential_for_bucket(
 
 #[cfg(test)]
 mod tests {
+    use std::time::{SystemTime, UNIX_EPOCH};
+
     use super::*;
     use wiremock::matchers::{method, path};
     use wiremock::{Mock, MockServer, ResponseTemplate};
@@ -267,4 +269,56 @@ mod tests {
             Err(format!("Failed to get token: {}", err_text).into())
         }
     }
+
+    #[tokio::test]
+    async fn secrets_cache_hit_returns_cached_value() {
+        let cache = SecretsCache::new();
+        let key = "test".to_string();
+
+        let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs();
+        cache.insert(key.clone(), "cached_token".to_string(), now + 3600);
+
+
+        let fetcher = || async { panic!("Should not be called on cache hit") };
+
+        let result = cache.get(&key, fetcher).await;
+        assert_eq!(result, Some("cached_token".to_string()));
+    }
+
+    #[tokio::test]
+    async fn secrets_cache_expired_renews_token() {
+        let cache = SecretsCache::new();
+        let key = "test2".to_string();
+        // expired token
+        let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs();
+        cache.insert(key.clone(), "old_token".to_string(), now);
+    
+        // fetcher returns new token
+        let fetcher = move || async {
+            let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs();
+            Ok(IamResponse { access_token: "new_token".into(), expires_in: 3600, expiration: now + 7200 })
+        };
+    
+        let result = cache.get(&key, fetcher).await;
+        assert_eq!(result, Some("new_token".to_string()));
+    }
+    
+    #[tokio::test]
+    async fn secrets_cache_invalidate_works() {
+        let cache = SecretsCache::new();
+        let key = "test3".to_string();
+        let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs();
+        cache.insert(key.clone(), "token".to_string(), now + 3600);
+    
+        cache.invalidate(&key);
+    
+        // now fetcher must be called
+        let fetcher = move || async {
+            let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs();
+            Ok(IamResponse { access_token: "fresh_token".into(), expires_in: 3600, expiration: now + 3600 })
+        };
+        let result = cache.get(&key, fetcher).await;
+        assert_eq!(result, Some("fresh_token".to_string()));
+    }
+
 }