object-storage-proxy 0.4.1__tar.gz → 0.4.2__tar.gz

{object_storage_proxy-0.4.1 → object_storage_proxy-0.4.2}/Cargo.lock

@@ -1757,7 +1757,7 @@ dependencies = [
 
 [[package]]
 name = "object-storage-proxy"
-version = "0.4.1"
+version = "0.4.2"
 dependencies = [
  "async-stream",
  "async-trait",

{object_storage_proxy-0.4.1 → object_storage_proxy-0.4.2}/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "object-storage-proxy"
-version = "0.4.1"
+version = "0.4.2"
 edition = "2024"
 
 [dependencies]

{object_storage_proxy-0.4.1 → object_storage_proxy-0.4.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: object-storage-proxy
-Version: 0.4.1
+Version: 0.4.2
 Classifier: License :: Other/Proprietary License
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: Implementation :: CPython
@@ -104,12 +104,14 @@ cos_map = {
 }
 ```
 
-The Python callables take two arguments: 
-    TODO: add prefix for fine-grained validation
+The Python callables take two or three arguments: 
 
     - token: parsed from the original aws request's authorization header
     - bucket: parsed from the uri path
 
+    the validator may also take a third optional argument
+    - request: dict of the original request (method, path, query, ...)
+
 ```python
     def your_credentials_fetcher(token: str, bucket: str) -> str
     def your_request_authorizer(token: str, bucket: str) -> bool

{object_storage_proxy-0.4.1 → object_storage_proxy-0.4.2}/README.md

@@ -85,12 +85,14 @@ cos_map = {
 }
 ```
 
-The Python callables take two arguments: 
-    TODO: add prefix for fine-grained validation
+The Python callables take two or three arguments: 
 
     - token: parsed from the original aws request's authorization header
     - bucket: parsed from the uri path
 
+    the validator may also take a third optional argument
+    - request: dict of the original request (method, path, query, ...)
+
 ```python
     def your_credentials_fetcher(token: str, bucket: str) -> str
     def your_request_authorizer(token: str, bucket: str) -> bool

{object_storage_proxy-0.4.1 → object_storage_proxy-0.4.2}/integration/presto/query.sql

@@ -54,7 +54,7 @@ WITH order_stats AS (
     o.o_custkey   AS custkey,
     COUNT(*)      AS order_count,
     MAX(o.o_orderdate) AS last_order_date
-  FROM hive.default.orders AS o
+  FROM hive.default.orders_minio AS o
   GROUP BY
     o.o_custkey
   ORDER BY
@@ -67,7 +67,7 @@ SELECT
   os.order_count,
   os.last_order_date
 FROM order_stats AS os
-JOIN hive.default.customer AS c
+JOIN hive.default.customer_minio AS c
   ON os.custkey = c.c_custkey
 ORDER BY
   os.order_count DESC

{object_storage_proxy-0.4.1 → object_storage_proxy-0.4.2}/src/credentials/signer.rs

@@ -160,7 +160,7 @@ impl<'a> AwsSign<'a, HashMap<String, String>> {
         //     })
         //     .collect();
         
-        debug!("{:#?}", &url);
+        dbg!("{:#?}", &url);
         let url: Url = url.parse().unwrap();
         // let headers: HashMap<String, String> = headers
         //     .iter()
@@ -1263,6 +1263,8 @@ mod tests {
             port: 443,
             api_key: None,
             ttl: None,
+            tls: Some(true),
+            addressing_style: Some("path".to_string()),
         }
     }
 

{object_storage_proxy-0.4.1 → object_storage_proxy-0.4.2}/src/lib.rs

@@ -20,6 +20,8 @@ use std::sync::{
     atomic::{AtomicBool, AtomicUsize, Ordering},
 };
 
+// use utils::functions::inspect_callable_signature;
+
 
 use std::collections::HashMap;
 use std::fmt::Debug;
@@ -252,13 +254,14 @@ impl ProxyHttp for MyProxy {
 
         let path = session.req_header().uri.path();
 
+
         let parse_path_result = parse_path(path);
         if parse_path_result.is_err() {
             error!("Failed to parse path: {:?}", parse_path_result);
             return Err(pingora::Error::new_str("Failed to parse path"));
         }
 
-        let (_, (bucket, _)) = parse_path(path).unwrap();
+        let (_, (bucket, _)) = parse_path_result.unwrap();
 
         let hdr_bucket = bucket.to_owned();
 
@@ -266,20 +269,39 @@ impl ProxyHttp for MyProxy {
             let map = ctx.cos_mapping.read().await;
             map.get(&hdr_bucket).cloned()
         };
+
+        let addressing_style = bucket_config.clone()
+            .and_then(|config| config.addressing_style)
+            .unwrap_or("virtual".to_string());
+
         let endpoint = match bucket_config.clone() {
-            Some(config) => format!("{}.{}", bucket, config.host.to_owned()),
+            Some(config) => {
+                if addressing_style == "path" {
+                    config.host.to_owned()
+                } else {
+                    format!("{}.{}", bucket, config.host)
+                }            
+            },
             None => {
                 format!("{}.{}", bucket, self.cos_endpoint)
             }
         };
 
-        let port = bucket_config
+        let port = bucket_config.clone()
             .and_then(|config| Some(config.port))
             .unwrap_or(443);
 
         let addr = (endpoint.clone(), port);
 
-        let mut peer = Box::new(HttpPeer::new(addr, true, endpoint.clone()));
+        let endpoint_is_tls = bucket_config
+            .and_then(|config| config.tls)
+            .unwrap_or(true);
+        
+        dbg!("is_tls: {}", endpoint_is_tls);
+        dbg!("endpoint: {}", &endpoint);
+
+        let mut peer = Box::new(HttpPeer::new(addr, endpoint_is_tls, endpoint.clone()));
+        dbg!("peer: {:#?}", &peer);
 
         // todo: make ths configurable
 
@@ -322,21 +344,36 @@ impl ProxyHttp for MyProxy {
         info!("request method : {}", session.req_header().method);
 
         let parsed_query_result = parse_query(request_query);
+
         if parsed_query_result.is_err() {
             error!("Failed to parse query: {:?}", parsed_query_result);
             return Err(pingora::Error::new_str("Failed to parse query"));
         }
-        let (rest, query_dict) = parsed_query_result.unwrap();
+        let (rest, mut query_dict) = parsed_query_result.unwrap();
         if rest.is_empty() {
             info!("Parsed query: {:#?}", query_dict);
         } else {
             error!("Failed to parse query: {}", rest);
         }
 
-        info!("Parsed query: {:#?}", query_dict);
+        query_dict.insert("method".to_string(), session.req_header().method.to_string());
+        query_dict.insert("path".to_string(), session.req_header().uri.path().to_string());
+        // insert source
+        query_dict.insert(
+            "source".to_string(),
+            session
+                .req_header()
+                .headers
+                .get("x-forwarded-for")
+                .and_then(|h| h.to_str().ok())
+                .unwrap_or_default()
+                .to_string(),
+        );
+
 
 
 
+        info!("---> Parsed query: {:#?}", query_dict);
 
         if session
             .req_header()
@@ -357,7 +394,7 @@ impl ProxyHttp for MyProxy {
             return Err(pingora::Error::new_str("Failed to parse path"));
         }
 
-        let (_, (bucket, _uri_path)) = parse_path(path).unwrap();
+        let (_, (bucket, _uri_path)) = parse_path_result.unwrap();
 
         let hdr_bucket = bucket.to_owned();
 
@@ -522,21 +559,27 @@ impl ProxyHttp for MyProxy {
                 }
             }
             info!("Signature check passed, continuing now onto the bespoke validation");
-            let cache_key = format!("{}:{}", &access_key, bucket);
+            let cache_key = format!("{}:{}:{:?}", &access_key, bucket, &query_dict);
             debug!("Cache key: {}", cache_key);
 
             let bucket_clone = bucket.to_string();
             let callback_clone: PyObject = Python::with_gil(|py| py_cb.clone_ref(py));
+
             let move_access_key = access_key.clone();
+            let req = query_dict.clone();
+
             ctx.auth_cache
                 .get_or_validate(&cache_key, Duration::from_secs(ttl), move || {
                     let tk = move_access_key.clone();
                     let bu = bucket_clone.clone();
                     let cb = Python::with_gil(|py| callback_clone.clone_ref(py));
-                    async move {
-                        validate_request(&tk, &bu, cb)
-                            .await
-                            .map_err(|_| pingora::Error::new_str("Validator error"))
+                    {
+                        let req_value = req.clone();
+                        async move {
+                            validate_request(&tk, &bu, &req_value, cb)
+                                .await
+                                .map_err(|_| pingora::Error::new_str("Validator error"))
+                        }
                     }
                 })
                 .await?
@@ -630,8 +673,14 @@ impl ProxyHttp for MyProxy {
         let _ = upstream_request.remove_header("accept-encoding");
 
         debug!("upstream_request_filter::start");
+
+
         let (_, (bucket, my_updated_url)) = parse_path(upstream_request.uri.path()).unwrap();
 
+        dbg!(&my_updated_url);
+
+
+
         let hdr_bucket = bucket.to_string();
 
         let my_query = match upstream_request.uri.query() {
@@ -644,12 +693,33 @@ impl ProxyHttp for MyProxy {
             map.get(&hdr_bucket).cloned()
         };
 
+        let addressing_style = bucket_config
+            .clone()
+            .and_then(|config| config.addressing_style)
+            .unwrap_or("virtual".to_string());
+
+
+        let this_url = match addressing_style.as_str() {
+            "virtual" => my_updated_url,
+            _ => {
+
+                let u_url = format!("/{}{}", bucket, my_updated_url);
+                dbg!("u_url: {}", &u_url);
+                &u_url.clone()
+            },
+        };
+
+
         let endpoint = match bucket_config.clone() {
             Some(cfg) => {
+                let this_host = match addressing_style.as_str() {
+                    "path" => cfg.host.to_owned(),
+                    _ => format!("{}.{}", bucket, cfg.host),
+                };
                 if cfg.port == 443 {
-                    format!("{}.{}", bucket, cfg.host)
+                    this_host
                 } else {
-                    format!("{}.{}:{}", bucket, cfg.host, cfg.port)
+                    format!("{}:{}", this_host, cfg.port)
                 }
             }
             None => format!("{}.{}", bucket, self.cos_endpoint),
@@ -659,15 +729,17 @@ impl ProxyHttp for MyProxy {
 
         // Box:leak the temporary string to get a static reference which will outlive the function
         let authority = Authority::from_static(Box::leak(endpoint.clone().into_boxed_str()));
+        // if addressing_style == "virtual" {
 
         let new_uri = Uri::builder()
             .scheme("https")
             .authority(authority.clone())
-            .path_and_query(my_updated_url.to_owned() + &my_query)
+            .path_and_query(this_url.to_owned() + &my_query)
             .build()
             .expect("should build a valid URI");
 
         upstream_request.set_uri(new_uri.clone());
+        // }
         upstream_request.insert_header("host", authority.as_str())?;
 
         let (maybe_hmac, maybe_api_key) = match &bucket_config {
@@ -837,7 +909,7 @@ impl ProxyHttp for MyProxy {
             upstream_request.insert_header("Authorization", format!("Bearer {bearer_token}"))?;
         }
 
-        debug!("Sending request to upstream: {}", &new_uri);
+        // debug!("Sending request to upstream: {}", &new_uri);
 
         debug!("Request sent to upstream.");
         debug!("upstream_request_filter::end");

{object_storage_proxy-0.4.1 → object_storage_proxy-0.4.2}/src/parsers/cos_map.rs

@@ -28,6 +28,8 @@ pub struct CosMapItem {
     pub access_key: Option<String>,
     pub secret_key: Option<String>,
     pub ttl: Option<u64>,
+    pub tls: Option<bool>,
+    pub addressing_style: Option<String>,
 }
 
 impl CosMapItem {
@@ -119,6 +121,13 @@ pub(crate) fn parse_cos_map(
             None
         };
 
+        let tls = inner_map
+            .get("tls")
+            .or_else(|| inner_map.get("is_tls_enabled"))
+            .map(|v| v.extract(py))
+            .transpose()?;
+           
+
         let access_key = inner_map
             .get("access_key")
             .or_else(|| inner_map.get("accessKey"))
@@ -131,6 +140,12 @@ pub(crate) fn parse_cos_map(
             .map(|v| v.extract(py))
             .transpose()?;
 
+        let addressing_style = inner_map
+            .get("addressing_style")
+            .or_else(|| inner_map.get("addressingStyle"))
+            .map(|v| v.extract(py))
+            .transpose()?;
+
         map.insert(
             bucket.clone(),
             CosMapItem {
@@ -141,6 +156,8 @@ pub(crate) fn parse_cos_map(
                 access_key,
                 secret_key,
                 ttl,
+                tls,
+                addressing_style,
             },
         );
     }

object_storage_proxy-0.4.2/src/utils/functions.rs

@@ -0,0 +1,27 @@
+use pyo3::prelude::*;
+use pyo3::types::{IntoPyDict, PyAny, PyFunction};
+use tracing::debug;
+
+pub(crate) fn callable_accepts_request(py: Python<'_>, callable: &PyObject) -> PyResult<bool> {
+
+    let inspect = py.import("inspect")?;
+    let signature = inspect.call_method1("signature", (callable.to_owned(),))?;
+    let parameters = signature.getattr("parameters")?;
+    dbg!(&parameters);
+    let parameters = parameters.call_method0("items")?;
+
+    
+    for p in parameters.try_iter()? {
+        let (name, param) = p?.extract::<(String, PyObject)>()?;
+        let annotation = param.getattr(py, "annotation")?;
+        debug!("Param: {}", name);
+        let arg_type = annotation.to_string();
+        debug!("Annotation: {}", &arg_type);
+        if name == "request" && arg_type.contains("dict"){
+            return Ok(true)
+        }
+
+    }
+
+    Ok(false)
+}

{object_storage_proxy-0.4.1 → object_storage_proxy-0.4.2}/src/utils/mod.rs

@@ -1 +1,2 @@
 pub mod validator;
+pub mod functions;

{object_storage_proxy-0.4.1 → object_storage_proxy-0.4.2}/src/utils/validator.rs

@@ -1,6 +1,8 @@
-use pyo3::{PyObject, Python};
+use pyo3::{types::IntoPyDict, PyObject, Python};
+use rustls::crypto::hash::Hash;
 use tokio::{sync::Mutex, task};
 use tracing::{debug, error};
+use tracing_subscriber::field::debug;
 
 use std::{
     collections::HashMap,
@@ -8,6 +10,8 @@ use std::{
     time::{Duration, Instant},
 };
 
+use crate::utils::functions::callable_accepts_request;
+
 #[derive(Clone, Debug)]
 struct AuthEntry {
     authorized: bool,
@@ -107,26 +111,67 @@ impl AuthCache {
 pub async fn validate_request(
     token: &str,
     bucket: &str,
+    request: &HashMap<String, String>,
     callback: PyObject,
 ) -> Result<bool, String> {
     let token = token.to_string();
     let bucket = bucket.to_string();
 
-    let authorized = task::spawn_blocking(move || {
-        Python::with_gil(
-            |py| match callback.call1(py, (token.as_str(), bucket.as_str())) {
-                Ok(result_obj) => result_obj
-                    .extract::<bool>(py)
-                    .map_err(|_| "Failed to extract boolean".to_string()),
-                Err(e) => {
-                    error!("Python callback error: {:?}", e);
-                    Err("Inner Python exception".to_string())
-                }
-            },
-        )
-    })
-    .await
-    .map_err(|e| format!("Join error: {:?}", e))??;
+    let req = request
+        .into_iter()
+        .map(|(k, v)| (k.clone(), v.clone()))
+        .collect::<HashMap<String, String>>();
+
+    debug!("request details sent to Python callable: {:?}", &req);
+
+    let takes_request = Python::with_gil(|py| {
+        let sig = callable_accepts_request(py, &callback);
+        if sig.is_err() {
+            return Err(format!("Invalid callable signature: {:?}", sig));
+        }
+        Ok(sig.unwrap())
+    });
+
+    if takes_request.is_err() {
+        return Err(format!("Invalid callable signature: {:?}", takes_request));
+    }
+    let takes_request = takes_request.unwrap();
+
+    debug!("Python callable can take request: {:?}", &takes_request);
+
+    let authorized = if takes_request {
+        task::spawn_blocking(move || {
+            Python::with_gil(
+                |py| match callback.call1(py, (token.as_str(), bucket.as_str(), &req)) {
+                    Ok(result_obj) => result_obj
+                        .extract::<bool>(py)
+                        .map_err(|_| "Failed to extract boolean".to_string()),
+                    Err(e) => {
+                        error!("Python callback error: {:?}", e);
+                        Err("Inner Python exception".to_string())
+                    }
+                },
+            )
+        })
+        .await
+        .map_err(|e| format!("Join error: {:?}", e))??
+    } else {
+        task::spawn_blocking(move || {
+            Python::with_gil(
+                |py| match callback.call1(py, (token.as_str(), bucket.as_str())) {
+                    Ok(result_obj) => result_obj
+                        .extract::<bool>(py)
+                        .map_err(|_| "Failed to extract boolean".to_string()),
+                    Err(e) => {
+                        error!("Python callback error: {:?}", e);
+                        Err("Inner Python exception".to_string())
+                    }
+                },
+            )
+        })
+        .await
+        .map_err(|e| format!("Join error: {:?}", e))??
+    };
 
     Ok(authorized)
 }

{object_storage_proxy-0.4.1 → object_storage_proxy-0.4.2}/test_server.py

@@ -8,7 +8,7 @@ from dotenv import load_dotenv
 from object_storage_proxy import start_server, ProxyServerConfig
 
 
-_TRUES  = {"y", "yes", "t", "true", "on", "1"}
+_TRUES = {"y", "yes", "t", "true", "on", "1"}
 _FALSES = {"n", "no", "f", "false", "off", "0"}
 
 
@@ -23,35 +23,39 @@ def strtobool(val: str) -> bool:
 
 
 def do_api_creds(token: str, bucket: str) -> str:
-    """Fetch credentials (ro, rw, access_denied) for the given bucket, depending on the token. """
+    """Fetch credentials (ro, rw, access_denied) for the given bucket, depending on the token."""
     apikey = os.getenv("COS_API_KEY")
     if not apikey:
         raise ValueError("COS_API_KEY environment variable not set")
-    
+
     print(f"Fetching credentials for {bucket}...")
     return apikey
 
 
 def do_hmac_creds(token: str, bucket: str) -> str:
-    """ Fetch HMAC credentials (ro, rw, access_denied) for the given bucket, depending on the token """
+    """Fetch HMAC credentials (ro, rw, access_denied) for the given bucket, depending on the token"""
     access_key = os.getenv("ACCESS_KEY")
     secret_key = os.getenv("SECRET_KEY")
     if not access_key or not secret_key:
         raise ValueError("ACCESS_KEY or SECRET_KEY environment variable not set")
-        
+
     print(f"Fetching HMAC credentials for {bucket}...")
 
-    return json.dumps({
-        "access_key": access_key,
-        "secret_key": secret_key
-    })
+    return json.dumps({"access_key": access_key, "secret_key": secret_key})
+
 
 def lookup_secret_key(access_key: str) -> str | None:
     # get all environment variables ending in ACCESS_KEY
-    access_keys = [{key:value} for key, value in os.environ.items() if key.endswith("ACCESS_KEY") and value==access_key ]
+    access_keys = [
+        {key: value}
+        for key, value in os.environ.items()
+        if key.endswith("ACCESS_KEY") and value == access_key
+    ]
 
     if len(access_keys) > 0:
-        access_key_var = next((k for k, v in access_keys[0].items() if v == access_key), None)
+        access_key_var = next(
+            (k for k, v in access_keys[0].items() if v == access_key), None
+        )
 
         secret_key_var = access_key_var.replace("ACCESS_KEY", "SECRET_KEY")
         return os.getenv(secret_key_var, None)
@@ -59,16 +63,14 @@ def lookup_secret_key(access_key: str) -> str | None:
         print(f"no access keys found for : {access_key}")
 
 
-def do_validation(token: str, bucket: str) -> bool:
-    """ Authorize the request based on token for the given bucket. 
-        You can plug in your own authorization service here.
-        The token is the authorization token passed in the request.
-        The bucket is the bucket name.
-        The function should return True if the request is authorized, False otherwise.
+def do_validation(token: str, bucket: str, request: dict) -> bool:
+    """Authorize the request based on token for the given bucket.
+    You can plug in your own authorization service here.
+    The token is the authorization token passed in the request.
+    The bucket is the bucket name.
+    The function should return True if the request is authorized, False otherwise.
     """
-
-    print(f"PYTHON: Validating headers: {token} for {bucket}...")
-    # return random.choice([True, False])
+    print(f"------> From Python: Validating headers: {token=}, {bucket=}, {request=}")
     return True
 
 
@@ -86,18 +88,27 @@ def main() -> None:
         raise ValueError("COS_API_KEY environment variable not set")
 
     cos_map = {
+        "tpch": {
+            "host": "biggie",
+            "region": "eu-de",
+            "port": 9000,
+            "access_key": "localkey",
+            "secret_key": "localpass",
+            "addressing_style": "path",
+            "is_tls_enabled": False,
+        },
         "bucket1": {
             "host": "s3.eu-de.cloud-object-storage.appdomain.cloud",
             "region": "eu-de",
             "port": 443,
             "apikey": apikey,
-            "ttl": 0
+            "ttl": 0,
         },
         "bucket2": {
             "host": "s3.eu-de.cloud-object-storage.appdomain.cloud",
             "region": "eu-de",
             "port": 443,
-            "apikey": apikey
+            "apikey": apikey,
         },
         "proxy-bucket01": {
             "host": "s3.eu-de.cloud-object-storage.appdomain.cloud",
@@ -105,7 +116,7 @@ def main() -> None:
             # "access_key": os.getenv("ACCESS_KEY"),
             # "secret_key": os.getenv("SECRET_KEY"),
             "port": 443,
-            "ttl": 300
+            "ttl": 300,
         },
         "proxy-bucket05": {
             "host": "s3.eu-de.cloud-object-storage.appdomain.cloud",
@@ -113,7 +124,7 @@ def main() -> None:
             "access_key": os.getenv("PROXY_BUCKET05_ACCESS_KEY"),
             "secret_key": os.getenv("PROXY_BUCKET05_SECRET_KEY"),
             "port": 443,
-            "ttl": 300
+            "ttl": 300,
         },
         "proxy-aws-bucket01": {
             "host": "s3.eu-west-3.amazonaws.com",
@@ -121,20 +132,19 @@ def main() -> None:
             "access_key": os.getenv("AWS_ACCESS_KEY"),
             "secret_key": os.getenv("AWS_SECRET_KEY"),
             "port": 443,
-            "ttl": 300
-        }
+            "ttl": 300,
+        },
     }
 
-    hmac_keys= [
+    hmac_keys = [
         # {
         #     "access_key": os.getenv("LOCAL_ACCESS_KEY"),
         #     "secret_key": os.getenv("LOCAL_SECRET_KEY")
         # },
         {
             "access_key": os.getenv("LOCAL2_ACCESS_KEY"),
-            "secret_key": os.getenv("LOCAL2_SECRET_KEY")
+            "secret_key": os.getenv("LOCAL2_SECRET_KEY"),
         },
-
     ]
 
     ra = ProxyServerConfig(
@@ -147,7 +157,7 @@ def main() -> None:
         # verify=False,
         hmac_keystore=hmac_keys,
         skip_signature_validation=False,
-        hmac_fetcher=lookup_secret_key
+        hmac_fetcher=lookup_secret_key,
     )
 
     start_server(ra)
@@ -155,6 +165,3 @@ def main() -> None:
 
 if __name__ == "__main__":
     main()
-
-
-