object-storage-proxy 0.4.2__tar.gz → 0.5.0__tar.gz

object_storage_proxy-0.5.0/.env.example

@@ -0,0 +1,52 @@
+# Environment variables
+
+Copy this file to .env and fill in your values.
+The .env file is excluded from version control.
+
+# ---------------------------------------------------------------------------
+# IBM Cloud Object Storage
+# ---------------------------------------------------------------------------
+
+# IAM API key used to fetch bearer tokens for COS buckets that use API key auth.
+COS_API_KEY=
+
+# Default HMAC keypair (used by do_hmac_creds fallback).
+ACCESS_KEY=
+SECRET_KEY=
+
+# Per-bucket HMAC keypairs. The naming convention is <PREFIX>_ACCESS_KEY / <PREFIX>_SECRET_KEY.
+# The hmac_fetcher callable resolves the secret key from the access key by scanning
+# all environment variables that follow this pattern.
+LOCAL2_ACCESS_KEY=
+LOCAL2_SECRET_KEY=
+
+PROXY_BUCKET05_ACCESS_KEY=
+PROXY_BUCKET05_SECRET_KEY=
+
+# ---------------------------------------------------------------------------
+# AWS
+# ---------------------------------------------------------------------------
+
+AWS_ACCESS_KEY=
+AWS_SECRET_KEY=
+
+# Required for AWS CLI >= 2.x to avoid checksum errors with the proxy.
+AWS_REQUEST_CHECKSUM_CALCULATION=WHEN_REQUIRED
+
+# ---------------------------------------------------------------------------
+# TLS (HTTPS frontend)
+# ---------------------------------------------------------------------------
+
+# Full paths to the TLS certificate and private key used by the HTTPS listener.
+# Generate self-signed certs for local development:
+#   openssl req -x509 -nodes -days 365 -newkey rsa:4096 \
+#     -keyout key.pem -out cert.pem -config localhost.cnf
+TLS_CERT_PATH=
+TLS_KEY_PATH=
+
+# ---------------------------------------------------------------------------
+# Proxy behaviour
+# ---------------------------------------------------------------------------
+
+# Set to "true" to enable per-URL request counting (exposed via get_request_count()).
+OSP_ENABLE_REQUEST_COUNTING=false

object_storage_proxy-0.5.0/.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,37 @@
+---
+name: Bug report
+about: Report a reproducible defect
+labels: bug
+---
+
+**Describe the bug**
+
+A clear and concise description of what the bug is.
+
+**To reproduce**
+
+Steps to reproduce the behaviour:
+
+1. Configure `ProxyServerConfig` with ...
+2. Send request ...
+3. See error ...
+
+**Expected behaviour**
+
+What you expected to happen.
+
+**Actual behaviour**
+
+What actually happened. Include relevant log output (redact any credentials).
+
+**Environment**
+
+- object-storage-proxy version:
+- Python version:
+- Rust version (`rustc --version`):
+- OS:
+- Client (aws-cli / boto3 / polars / other):
+
+**Additional context**
+
+Add any other context about the problem here.

object_storage_proxy-0.5.0/.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,21 @@
+---
+name: Feature request
+about: Suggest a new capability or improvement
+labels: enhancement
+---
+
+**Is your feature request related to a problem?**
+
+A clear and concise description of what the problem is.
+
+**Describe the solution you'd like**
+
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+
+Any alternative solutions or features you have considered.
+
+**Additional context**
+
+Add any other context, links, or screenshots here.

object_storage_proxy-0.5.0/.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,20 @@
+## Summary
+
+Briefly describe what this PR does and why.
+
+## Related issue
+
+Closes #
+
+## Changes
+
+- 
+- 
+
+## Checklist
+
+- [ ] Tests added or updated for every changed behaviour
+- [ ] `cargo fmt` passes
+- [ ] `cargo clippy -- -D warnings` passes with no new warnings
+- [ ] `CHANGELOG.md` updated under `Unreleased`
+- [ ] No credentials, personal paths, or debug prints introduced

{object_storage_proxy-0.4.2 → object_storage_proxy-0.5.0}/.github/workflows/ci.yml

@@ -14,8 +14,22 @@ permissions:
   contents: read
 
 jobs:
-  test:
-    name: Test
+  lint:
+    name: Lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install Rust stable
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          components: rustfmt, clippy
+      - name: Check formatting
+        run: cargo fmt --check
+      - name: Clippy
+        run: cargo clippy -- -D warnings
+
+  test_rust:
+    name: Test (Rust)
     runs-on: ubuntu-latest
 
     steps:
@@ -25,6 +39,27 @@ jobs:
       - name: Run cargo tests
         run: cargo test --all-features
 
+  test_python:
+    name: Test (Python)
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+      - name: Build and install extension
+        uses: PyO3/maturin-action@v1
+        with:
+          args: --out dist --find-interpreter
+      - name: Install wheel and test deps
+        run: |
+          uv pip install --system dist/*.whl pytest
+      - name: Run pytest
+        run: pytest tests/
+
 
   linux:
     runs-on: ${{ matrix.platform.runner }}
@@ -34,8 +69,8 @@ jobs:
         platform:
           - runner: ubuntu-22.04
             target: x86_64
-          # - runner: ubuntu-22.04
-          #   target: aarch64
+          - runner: ubuntu-22.04
+            target: aarch64
           # - runner: ubuntu-22.04
           #   target: armv7
 
@@ -50,6 +85,7 @@ jobs:
       #   run: echo "CC_aarch64_unknown_linux_gnu=aarch64-linux-gnu-gcc" >> $GITHUB_ENV
 
       - name: Install AArch64 cross-compiler
+        if: matrix.platform.target == 'aarch64'
         run: |
           sudo apt-get update
           sudo apt-get install -y gcc-aarch64-linux-gnu g++-aarch64-linux-gnu
@@ -57,18 +93,21 @@ jobs:
       - name: Build wheels
         uses: PyO3/maturin-action@v1
         env:
-          # point maturin's cc-rs at the aarch64 cross-compiler
-          CC_aarch64_unknown_linux_gnu: gcc-aarch64-linux-gnu
-          CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER: gcc-aarch64-linux-gnu
-          # force the assembler to know it's ARMv8-A
-          CFLAGS_aarch64_unknown_linux_gnu: "-march=armv8-a -D__ARM_ARCH=8"
+          # point maturin's cc-rs and cargo at the aarch64 cross-compiler
+          CC_aarch64_unknown_linux_gnu: aarch64-linux-gnu-gcc
+          CXX_aarch64_unknown_linux_gnu: aarch64-linux-gnu-g++
+          CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER: aarch64-linux-gnu-gcc
         with:
           target: ${{ matrix.platform.target }}
           args: --release --out dist --find-interpreter
           sccache: 'true'
           manylinux: auto
           before-script-linux: |
-            # If we're running on rhel centos, install needed packages.
+            # Install aarch64 cross-compiler inside the manylinux container
+            if command -v apt-get &> /dev/null; then
+                apt-get update -y && apt-get install -y gcc-aarch64-linux-gnu g++-aarch64-linux-gnu
+            fi
+            # If we're running on rhel/centos, install needed packages.
             if command -v yum &> /dev/null; then
                 yum update -y && yum install -y perl-core openssl openssl-devel pkgconfig libatomic perl-CPAN
 
@@ -196,8 +235,6 @@ jobs:
     strategy:
       matrix:
         platform:
-          - runner: macos-13
-            target: x86_64
           - runner: macos-14
             target: aarch64
     steps:
@@ -236,7 +273,7 @@ jobs:
     name: Release
     runs-on: ubuntu-latest
     # if: ${{ startsWith(github.ref, 'refs/tags/') || github.event_name == 'workflow_dispatch' }}
-    needs: [test, linux, musllinux, macos, sdist]
+    needs: [lint, test_rust, test_python, linux, musllinux, macos, sdist]
     permissions:
       # Use to sign the release artifacts
       id-token: write

{object_storage_proxy-0.4.2 → object_storage_proxy-0.5.0}/.gitignore

@@ -1,4 +1,6 @@
 /target
+dist/
+target/wheels/
 .ibm_cos_creds
 .Idea/
 .env

object_storage_proxy-0.5.0/BUILD.md

@@ -0,0 +1,135 @@
+# Build Instructions
+
+`object-storage-proxy` is a mixed Rust/Python project built with [maturin](https://github.com/PyO3/maturin). The Rust library exposes a Python extension module via [PyO3](https://pyo3.rs).
+
+## Prerequisites
+
+| Tool | Purpose |
+|------|---------|
+| Rust (stable) | compile the core library |
+| Python ≥ 3.10 | host the extension module |
+| [uv](https://docs.astral.sh/uv/) | Python package/venv manager |
+| `maturin` | build the PyO3 extension wheel |
+
+### Install Rust
+
+```bash
+curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
+```
+
+### Install uv
+
+```bash
+curl -LsSf https://astral.sh/uv/install.sh | sh
+```
+
+### Nix (optional)
+
+A `flake.nix` is provided that sets up the full Rust + Python toolchain in an isolated shell:
+
+```bash
+nix develop
+```
+
+---
+
+## Development build (editable install)
+
+`maturin develop` compiles the Rust extension and installs it directly into the active virtual environment as an editable package — the fastest workflow during development.
+
+```bash
+# Create/activate a virtual environment and install dev dependencies
+uv sync
+
+# Compile the Rust extension and install it into .venv
+uv run maturin develop
+
+# With release optimisations (faster runtime, slower build)
+uv run maturin develop --release
+```
+
+The compiled `.so` is placed inside `.venv` and importable immediately as `import object_storage_proxy`.
+
+---
+
+## Build a wheel
+
+```bash
+# Debug wheel (quick, for local testing)
+uv run maturin build
+
+# Release wheel (optimised, for distribution)
+uv run maturin build --release
+```
+
+Wheels are written to `target/wheels/`.
+
+---
+
+## Install from wheel
+
+```bash
+pip install target/wheels/object_storage_proxy-*.whl
+```
+
+---
+
+## Run the test server
+
+`test_server.py` starts a local proxy server. It requires several environment variables (see `.env.example` or the table below).
+
+### Required environment variables
+
+| Variable | Description |
+|----------|-------------|
+| `COS_API_KEY` | IBM COS API key (used for `bucket1`, `bucket2`, `proxy-bucket01`) |
+| `ACCESS_KEY` / `SECRET_KEY` | Default HMAC keypair |
+| `LOCAL2_ACCESS_KEY` / `LOCAL2_SECRET_KEY` | HMAC keypair added to the keystore |
+| `PROXY_BUCKET05_ACCESS_KEY` / `PROXY_BUCKET05_SECRET_KEY` | HMAC keypair for `proxy-bucket05` |
+| `AWS_ACCESS_KEY` / `AWS_SECRET_KEY` | AWS keypair for `proxy-aws-bucket01` |
+
+Copy `.env.example` to `.env` (if provided) or export the variables manually, then run:
+
+```bash
+# Build the extension first (if not already done)
+uv run maturin develop
+
+# Start the proxy server
+uv run python test_server.py
+```
+
+The server listens on:
+- HTTP  → `0.0.0.0:6190`
+- HTTPS → `0.0.0.0:8443`
+
+---
+
+## Cargo commands (pure Rust)
+
+```bash
+# Build the Rust library
+cargo build
+
+# Run Rust unit tests
+cargo test
+
+# Run tests with nextest
+cargo nextest run
+
+# Watch for changes and recompile
+cargo watch -x build
+```
+
+---
+
+## Linting & formatting
+
+```bash
+# Rust
+cargo fmt
+cargo clippy -- -D warnings
+
+# Python
+uv run ruff check .
+uv run ruff format .
+```

object_storage_proxy-0.5.0/CHANGELOG.md

@@ -0,0 +1,69 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+### Added
+- `flake.nix` for reproducible Rust + Python development environment via Nix.
+- `Taskfile.yml` with tasks for build, run, test, lint, and clean.
+- `BUILD.md` with detailed build and run instructions.
+- `CONTRIBUTING.md`, `CHANGELOG.md`, `CODE_OF_CONDUCT.md`, `SECURITY.md`.
+- GitHub issue templates and pull request template.
+- `.env.example` documenting all required environment variables.
+
+### Changed
+- Fixed `pyproject.toml` license classifier from Proprietary to MIT.
+- Added `license`, `homepage`, `repository`, and `description` to `Cargo.toml`.
+- Updated `object_storage_proxy.pyi` stub with all `ProxyServerConfig` parameters.
+- Commented out broken `[tool.uv.workspace]` entry (integration/ has no pyproject.toml).
+
+### Fixed
+- Unused import compiler warnings resolved via `cargo fix`.
+
+## [0.4.3] - 2025-04-19
+
+### Added
+- Configurable `max_presign_url_usage_attempts` for presigned URL access control.
+- `server_name` field on `ProxyServerConfig`.
+- `hmac_fetcher` callable for dynamic secret key lookup by access key.
+
+### Changed
+- Migrated from `pingora 0.4` (OpenSSL) to `pingora 0.5` (rustls).
+- Switched from `openssl` to `rustls` + `aws-lc-rs` throughout.
+
+## [0.4.0] - 2025-03-01
+
+### Added
+- Configurable request counting (`enable_request_counting`, `disable_request_counting`, `get_request_count`).
+- `skip_signature_validation` option for development use.
+- `verify` option to disable upstream TLS certificate verification.
+- `hmac_keystore` support for multi-credential HMAC key management.
+
+### Changed
+- `ProxyServerConfig` now accepts `hmac_keystore` as a list of access/secret key dicts.
+
+## [0.3.0] - 2025-01-15
+
+### Added
+- Python callable for authorization (`validator`).
+- TTL-based authorization cache.
+- HTTP/2 support on the HTTPS frontend.
+
+## [0.2.0] - 2024-11-01
+
+### Added
+- Python callable for credential fetching (`bucket_creds_fetcher`).
+- IBM COS IAM bearer token cache with configurable TTL.
+- Path-style to virtual-style address translation.
+
+## [0.1.0] - 2024-09-01
+
+### Added
+- Initial release.
+- Pingora-based reverse proxy for AWS S3 and IBM Cloud Object Storage.
+- AWS SigV4 request re-signing.
+- `ProxyServerConfig` Python class with `cos_map`, `http_port`, `https_port`, `threads`.

object_storage_proxy-0.5.0/CODE_OF_CONDUCT.md

@@ -0,0 +1,41 @@
+# Contributor Covenant Code of Conduct
+
+## Our pledge
+
+We as members, contributors, and leaders pledge to make participation in our community a harassment-free experience for everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, caste, colour, religion, or sexual identity and orientation.
+
+We pledge to act and interact in ways that contribute to an open, welcoming, diverse, inclusive, and healthy community.
+
+## Our standards
+
+Examples of behaviour that contributes to a positive environment:
+
+- Demonstrating empathy and kindness toward other people
+- Being respectful of differing opinions, viewpoints, and experiences
+- Giving and gracefully accepting constructive feedback
+- Accepting responsibility and apologising to those affected by our mistakes, and learning from the experience
+- Focusing on what is best not just for us as individuals, but for the overall community
+
+Examples of unacceptable behaviour:
+
+- The use of sexualised language or imagery, and sexual attention or advances of any kind
+- Trolling, insulting or derogatory comments, and personal or political attacks
+- Public or private harassment
+- Publishing others' private information, such as a physical or email address, without their explicit permission
+- Other conduct which could reasonably be considered inappropriate in a professional setting
+
+## Enforcement responsibilities
+
+Community leaders are responsible for clarifying and enforcing our standards of acceptable behaviour and will take appropriate and fair corrective action in response to any behaviour that they deem inappropriate, threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies within all community spaces, and also applies when an individual is officially representing the community in public spaces.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behaviour may be reported to the community leaders responsible for enforcement at jeroen@flexworks.eu. All complaints will be reviewed and investigated promptly and fairly.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant](https://www.contributor-covenant.org), version 2.1.

object_storage_proxy-0.5.0/CONTRIBUTING.md

@@ -0,0 +1,101 @@
+# Contributing
+
+Thank you for considering a contribution to object-storage-proxy. The following guidelines help keep the process smooth for everyone.
+
+## Code of conduct
+
+This project follows the [Contributor Covenant](CODE_OF_CONDUCT.md). Please read it before participating.
+
+## Getting started
+
+1. Fork the repository and clone your fork.
+2. Create a feature or fix branch from `main`:
+   ```bash
+   git checkout -b feat/my-feature
+   # or
+   git checkout -b fix/my-bug
+   ```
+3. Set up the development environment (see [BUILD.md](BUILD.md)).
+4. Make your changes, add tests, and ensure everything passes.
+5. Open a pull request against `main`.
+
+## Development setup
+
+```bash
+# Install Rust (stable)
+curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
+
+# Install uv
+curl -LsSf https://astral.sh/uv/install.sh | sh
+
+# Install dependencies and build the extension
+uv sync
+uv run maturin develop
+
+# Or use the Nix flake
+nix develop
+```
+
+## Running tests
+
+```bash
+# Rust unit tests
+cargo test
+
+# With nextest
+cargo nextest run
+
+# Python tests
+uv run pytest
+```
+
+## Code style
+
+### Rust
+
+- Format with `cargo fmt` before committing.
+- All `cargo clippy -- -D warnings` findings must be resolved.
+- Prefer `?` over `.unwrap()` in non-test code.
+- Use `tracing::{debug, info, warn, error}` instead of `println!` or `dbg!`.
+
+### Python
+
+- Type annotations are required on all public functions.
+- Docstrings on all public symbols.
+
+## Commit messages
+
+Use the [Conventional Commits](https://www.conventionalcommits.org/) format:
+
+```
+<type>(<scope>): <short summary>
+
+[optional body]
+
+[optional footer]
+```
+
+Common types: `feat`, `fix`, `docs`, `refactor`, `test`, `chore`, `ci`.
+
+Examples:
+
+```
+feat(proxy): add path-style to virtual-style translation for AWS
+fix(signer): handle missing authorization header gracefully
+docs: add CONTRIBUTING guide
+```
+
+## Pull request checklist
+
+- [ ] Tests added or updated for every changed behaviour.
+- [ ] `cargo fmt` and `cargo clippy` pass with no warnings.
+- [ ] `CHANGELOG.md` updated under `Unreleased`.
+- [ ] PR description explains the motivation and approach.
+
+## Reporting bugs
+
+Use the [bug report template](.github/ISSUE_TEMPLATE/bug_report.md).
+
+## Requesting features
+
+Use the [feature request template](.github/ISSUE_TEMPLATE/feature_request.md).