maskflow-gateway 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- maskflow_gateway-0.1.0/.dockerignore +8 -0
- maskflow_gateway-0.1.0/.gitignore +16 -0
- maskflow_gateway-0.1.0/PKG-INFO +143 -0
- maskflow_gateway-0.1.0/README.md +116 -0
- maskflow_gateway-0.1.0/deploy/Dockerfile +44 -0
- maskflow_gateway-0.1.0/deploy/docker-compose.yml +36 -0
- maskflow_gateway-0.1.0/deploy/fly.toml +32 -0
- maskflow_gateway-0.1.0/deploy/helm/maskflow-gateway/Chart.yaml +15 -0
- maskflow_gateway-0.1.0/deploy/helm/maskflow-gateway/templates/NOTES.txt +20 -0
- maskflow_gateway-0.1.0/deploy/helm/maskflow-gateway/templates/_helpers.tpl +28 -0
- maskflow_gateway-0.1.0/deploy/helm/maskflow-gateway/templates/deployment.yaml +90 -0
- maskflow_gateway-0.1.0/deploy/helm/maskflow-gateway/templates/hpa.yaml +22 -0
- maskflow_gateway-0.1.0/deploy/helm/maskflow-gateway/templates/ingress.yaml +35 -0
- maskflow_gateway-0.1.0/deploy/helm/maskflow-gateway/templates/pdb.yaml +13 -0
- maskflow_gateway-0.1.0/deploy/helm/maskflow-gateway/templates/secret.yaml +16 -0
- maskflow_gateway-0.1.0/deploy/helm/maskflow-gateway/templates/service.yaml +14 -0
- maskflow_gateway-0.1.0/deploy/helm/maskflow-gateway/templates/servicemonitor.yaml +19 -0
- maskflow_gateway-0.1.0/deploy/helm/maskflow-gateway/values.yaml +81 -0
- maskflow_gateway-0.1.0/deploy/railway.json +14 -0
- maskflow_gateway-0.1.0/deploy/render.yaml +29 -0
- maskflow_gateway-0.1.0/loadtest/README.md +55 -0
- maskflow_gateway-0.1.0/loadtest/locustfile.py +91 -0
- maskflow_gateway-0.1.0/loadtest/mock_upstream.py +67 -0
- maskflow_gateway-0.1.0/pyproject.toml +65 -0
- maskflow_gateway-0.1.0/src/maskflow_gateway/__init__.py +6 -0
- maskflow_gateway-0.1.0/src/maskflow_gateway/__main__.py +32 -0
- maskflow_gateway-0.1.0/src/maskflow_gateway/app.py +73 -0
- maskflow_gateway-0.1.0/src/maskflow_gateway/config.py +103 -0
- maskflow_gateway-0.1.0/src/maskflow_gateway/errors.py +62 -0
- maskflow_gateway-0.1.0/src/maskflow_gateway/masking.py +88 -0
- maskflow_gateway-0.1.0/src/maskflow_gateway/observability/__init__.py +0 -0
- maskflow_gateway-0.1.0/src/maskflow_gateway/observability/logging.py +65 -0
- maskflow_gateway-0.1.0/src/maskflow_gateway/observability/metrics.py +69 -0
- maskflow_gateway-0.1.0/src/maskflow_gateway/providers/__init__.py +5 -0
- maskflow_gateway-0.1.0/src/maskflow_gateway/providers/anthropic.py +223 -0
- maskflow_gateway-0.1.0/src/maskflow_gateway/providers/openai.py +247 -0
- maskflow_gateway-0.1.0/src/maskflow_gateway/py.typed +0 -0
- maskflow_gateway-0.1.0/src/maskflow_gateway/ratelimit.py +79 -0
- maskflow_gateway-0.1.0/src/maskflow_gateway/routes/__init__.py +9 -0
- maskflow_gateway-0.1.0/src/maskflow_gateway/routes/_proxy.py +218 -0
- maskflow_gateway-0.1.0/src/maskflow_gateway/routes/chat.py +26 -0
- maskflow_gateway-0.1.0/src/maskflow_gateway/routes/embeddings.py +31 -0
- maskflow_gateway-0.1.0/src/maskflow_gateway/routes/mask.py +76 -0
- maskflow_gateway-0.1.0/src/maskflow_gateway/routes/messages.py +26 -0
- maskflow_gateway-0.1.0/src/maskflow_gateway/routes/meta.py +56 -0
- maskflow_gateway-0.1.0/src/maskflow_gateway/sessions.py +239 -0
- maskflow_gateway-0.1.0/src/maskflow_gateway/streaming/__init__.py +12 -0
- maskflow_gateway-0.1.0/src/maskflow_gateway/streaming/bytestream.py +43 -0
- maskflow_gateway-0.1.0/src/maskflow_gateway/streaming/sse.py +71 -0
- maskflow_gateway-0.1.0/src/maskflow_gateway/streaming/unmask.py +185 -0
- maskflow_gateway-0.1.0/src/maskflow_gateway/upstream.py +98 -0
- maskflow_gateway-0.1.0/tests/conftest.py +29 -0
- maskflow_gateway-0.1.0/tests/helpers.py +19 -0
- maskflow_gateway-0.1.0/tests/streaming/__init__.py +0 -0
- maskflow_gateway-0.1.0/tests/streaming/test_sse.py +43 -0
- maskflow_gateway-0.1.0/tests/streaming/test_unmask_fuzz.py +194 -0
- maskflow_gateway-0.1.0/tests/test_config.py +40 -0
- maskflow_gateway-0.1.0/tests/test_masking.py +114 -0
- maskflow_gateway-0.1.0/tests/test_meta.py +28 -0
- maskflow_gateway-0.1.0/tests/test_no_pii_leak.py +84 -0
- maskflow_gateway-0.1.0/tests/test_routes_chat.py +313 -0
- maskflow_gateway-0.1.0/tests/test_routes_messages.py +133 -0
- maskflow_gateway-0.1.0/tests/test_routes_misc.py +91 -0
- maskflow_gateway-0.1.0/tests/test_sessions.py +100 -0
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: maskflow-gateway
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: MaskFlow Gateway: a drop-in OpenAI/Anthropic-compatible proxy that masks PII before it reaches an LLM provider and restores it in the response -- streaming included.
|
|
5
|
+
License: MIT
|
|
6
|
+
Requires-Python: >=3.10
|
|
7
|
+
Requires-Dist: fastapi>=0.111
|
|
8
|
+
Requires-Dist: httpx>=0.27
|
|
9
|
+
Requires-Dist: maskflow-sdk<0.8,>=0.7.0
|
|
10
|
+
Requires-Dist: prometheus-client>=0.20
|
|
11
|
+
Requires-Dist: pydantic-settings>=2.3
|
|
12
|
+
Requires-Dist: pydantic>=2.7
|
|
13
|
+
Requires-Dist: uvicorn[standard]>=0.30
|
|
14
|
+
Provides-Extra: dev
|
|
15
|
+
Requires-Dist: cryptography>=42; extra == 'dev'
|
|
16
|
+
Requires-Dist: fakeredis>=2.23; extra == 'dev'
|
|
17
|
+
Requires-Dist: hypothesis>=6.100; extra == 'dev'
|
|
18
|
+
Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
|
|
19
|
+
Requires-Dist: pytest>=8.0; extra == 'dev'
|
|
20
|
+
Requires-Dist: respx>=0.21; extra == 'dev'
|
|
21
|
+
Provides-Extra: loadtest
|
|
22
|
+
Requires-Dist: locust>=2.29; extra == 'loadtest'
|
|
23
|
+
Provides-Extra: redis
|
|
24
|
+
Requires-Dist: cryptography>=42; extra == 'redis'
|
|
25
|
+
Requires-Dist: redis>=5.0; extra == 'redis'
|
|
26
|
+
Description-Content-Type: text/markdown
|
|
27
|
+
|
|
28
|
+
# maskflow-gateway
|
|
29
|
+
|
|
30
|
+
A drop-in **OpenAI / Anthropic-compatible proxy** that detects PII in a
|
|
31
|
+
request, replaces it with reversible typed placeholders before the request
|
|
32
|
+
reaches the provider, and restores the originals in the response —
|
|
33
|
+
**including mid-stream**.
|
|
34
|
+
|
|
35
|
+
Point your existing client's base URL at the gateway. No SDK change, no
|
|
36
|
+
code change.
|
|
37
|
+
|
|
38
|
+
```python
|
|
39
|
+
from openai import OpenAI
|
|
40
|
+
|
|
41
|
+
client = OpenAI(
|
|
42
|
+
base_url="http://localhost:8000/v1", # <- the gateway
|
|
43
|
+
api_key="sk-...", # <- your real OpenAI key, passed straight through
|
|
44
|
+
)
|
|
45
|
+
client.chat.completions.create(
|
|
46
|
+
model="gpt-4o",
|
|
47
|
+
messages=[
|
|
48
|
+
{"role": "user", "content": "Email Rahul at rahul.sharma@example.com about PAN ABCPE1234F"}
|
|
49
|
+
],
|
|
50
|
+
)
|
|
51
|
+
# OpenAI sees: "Email Rahul at <EMAIL_1> about PAN <PAN_1>"
|
|
52
|
+
# You get back: the model's reply with <EMAIL_1>/<PAN_1> restored to the real values
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
## Endpoints
|
|
56
|
+
|
|
57
|
+
| Route | What it does |
|
|
58
|
+
|---|---|
|
|
59
|
+
| `POST /v1/chat/completions` | OpenAI chat, streaming + non-streaming, tool calls |
|
|
60
|
+
| `POST /v1/messages` | Anthropic Messages, streaming + non-streaming, tool use |
|
|
61
|
+
| `POST /v1/embeddings` | masks each input **before** it is embedded — the RAG path |
|
|
62
|
+
| `POST /v1/mask` / `POST /v1/unmask` | direct masking, no upstream call |
|
|
63
|
+
| `GET /healthz` | liveness (never touches Redis) |
|
|
64
|
+
| `GET /readyz` | readiness — **503 fail-closed** if Redis `maxmemory-policy != noeviction` |
|
|
65
|
+
| `GET /metrics` | Prometheus |
|
|
66
|
+
| `GET /v1/entities` | every PII type the loaded packs detect |
|
|
67
|
+
|
|
68
|
+
## Streaming unmask
|
|
69
|
+
|
|
70
|
+
The model streams the reply in arbitrary chunks and a placeholder like
|
|
71
|
+
`<PERSON_NAME_1>` can be split across two SSE frames. The gateway parses
|
|
72
|
+
the provider's SSE, keeps a rolling buffer + a trie of the session's active
|
|
73
|
+
placeholders, and emits the longest prefix that is *certain* — a completed
|
|
74
|
+
placeholder (replaced) or a character that cannot begin any placeholder —
|
|
75
|
+
retaining only the tail that could still grow into one. A two-layer
|
|
76
|
+
decoder handles chunk splits mid-UTF-8. Property: for **any** chunking of a
|
|
77
|
+
masked reply, the concatenated stream equals the non-streaming
|
|
78
|
+
`unmask` result (fuzz-tested at every byte boundary).
|
|
79
|
+
|
|
80
|
+
## Sessions (multi-turn / tool loops)
|
|
81
|
+
|
|
82
|
+
Send `X-Maskflow-Session: <your opaque id>` to keep `<PHONE_1>` meaning the
|
|
83
|
+
same number across every turn and tool call of one agent run. Without the
|
|
84
|
+
header each request is masked and unmasked in isolation.
|
|
85
|
+
|
|
86
|
+
- `X-Maskflow-Session-TTL: <seconds>` overrides the default (3600), capped
|
|
87
|
+
at `MASKFLOW_GATEWAY_SESSION_TTL_MAX_SECONDS` (86400).
|
|
88
|
+
- Keyed sessions need Redis (`MASKFLOW_GATEWAY_REDIS_URL`). Mappings are
|
|
89
|
+
encrypted with **AES-256-GCM** (`MASKFLOW_GATEWAY_SESSION_KEY`, 32 random
|
|
90
|
+
bytes hex) before they touch Redis, with a mandatory TTL.
|
|
91
|
+
- Redis **must** run `maxmemory-policy noeviction` — an evicted session
|
|
92
|
+
mid-conversation means unmask finds nothing and a user sees raw
|
|
93
|
+
`<PERSON_NAME_1>` text. `/readyz` returns 503 until this is fixed.
|
|
94
|
+
|
|
95
|
+
## Configuration
|
|
96
|
+
|
|
97
|
+
Every setting is an environment variable prefixed `MASKFLOW_GATEWAY_`:
|
|
98
|
+
|
|
99
|
+
| Variable | Default | Notes |
|
|
100
|
+
|---|---|---|
|
|
101
|
+
| `OPENAI_BASE_URL` | `https://api.openai.com/v1` | must include the version segment |
|
|
102
|
+
| `ANTHROPIC_BASE_URL` | `https://api.anthropic.com/v1` | |
|
|
103
|
+
| `UPSTREAM_API_KEY` | *(unset)* | set → gateway injects it; unset → client's own key is forwarded, nothing stored |
|
|
104
|
+
| `NER` | `0` | `1` enables the spaCy pass (bare Indian names & addresses); much slower — see `loadtest/` |
|
|
105
|
+
| `MIN_CONFIDENCE` | `0.5` | detection threshold |
|
|
106
|
+
| `REDIS_URL` | *(unset)* | unset → in-process ephemeral sessions only (single replica) |
|
|
107
|
+
| `SESSION_KEY` | *(unset)* | hex, 32 bytes; **required** when `REDIS_URL` is set |
|
|
108
|
+
| `SESSION_TTL_SECONDS` / `SESSION_TTL_MAX_SECONDS` | `3600` / `86400` | |
|
|
109
|
+
| `REQUIRE_MAXMEMORY_NOEVICTION` | `true` | |
|
|
110
|
+
| `MAX_REQUEST_BYTES` | `2000000` | |
|
|
111
|
+
| `UPSTREAM_TIMEOUT_SECONDS` / `UPSTREAM_CONNECT_TIMEOUT_SECONDS` | `120` / `10` | |
|
|
112
|
+
| `RATE_LIMIT_PER_MINUTE` / `RATE_LIMIT_BURST` | `0` (off) | keyed by a hash of the client's `Authorization` |
|
|
113
|
+
| `TOOL_CALL_MAX_DEPTH` / `TOOL_CALL_MAX_ITEMS` | `32` / `10000` | bound the tool-argument JSON walk |
|
|
114
|
+
| `JSON_LOGS` | `1` | structured logs; every record also passes MaskFlow's PII scrub filter |
|
|
115
|
+
| `CORS_ALLOW_ORIGINS` | `[]` | JSON list |
|
|
116
|
+
|
|
117
|
+
The gateway's masking config comes only from these variables — it does not
|
|
118
|
+
read a `.maskflowrc`.
|
|
119
|
+
|
|
120
|
+
## Run
|
|
121
|
+
|
|
122
|
+
```bash
|
|
123
|
+
pip install "maskflow-gateway[redis]"
|
|
124
|
+
export MASKFLOW_GATEWAY_REDIS_URL=redis://localhost:6379/0
|
|
125
|
+
export MASKFLOW_GATEWAY_SESSION_KEY=$(python -c "import os;print(os.urandom(32).hex())")
|
|
126
|
+
maskflow-gateway --host 0.0.0.0 --port 8000 --workers 4
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
Deploy artifacts in [`deploy/`](deploy/): multi-arch `Dockerfile`,
|
|
130
|
+
`docker-compose.yml` (gateway + noeviction Redis), a Helm chart
|
|
131
|
+
(`deploy/helm/maskflow-gateway`, with HPA / PDB / ServiceMonitor), and
|
|
132
|
+
Fly / Render / Railway templates.
|
|
133
|
+
|
|
134
|
+
## Throughput
|
|
135
|
+
|
|
136
|
+
Published honestly, with the hardware, in [`loadtest/README.md`](loadtest/README.md).
|
|
137
|
+
Rough laptop floor: **~430 req/s** pattern-only, **~90 req/s** NER-enabled
|
|
138
|
+
(4 workers, Intel i7-9750H).
|
|
139
|
+
|
|
140
|
+
## License
|
|
141
|
+
|
|
142
|
+
MIT — like the rest of MaskFlow. No license gates, no paid flags, no
|
|
143
|
+
telemetry.
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
# maskflow-gateway
|
|
2
|
+
|
|
3
|
+
A drop-in **OpenAI / Anthropic-compatible proxy** that detects PII in a
|
|
4
|
+
request, replaces it with reversible typed placeholders before the request
|
|
5
|
+
reaches the provider, and restores the originals in the response —
|
|
6
|
+
**including mid-stream**.
|
|
7
|
+
|
|
8
|
+
Point your existing client's base URL at the gateway. No SDK change, no
|
|
9
|
+
code change.
|
|
10
|
+
|
|
11
|
+
```python
|
|
12
|
+
from openai import OpenAI
|
|
13
|
+
|
|
14
|
+
client = OpenAI(
|
|
15
|
+
base_url="http://localhost:8000/v1", # <- the gateway
|
|
16
|
+
api_key="sk-...", # <- your real OpenAI key, passed straight through
|
|
17
|
+
)
|
|
18
|
+
client.chat.completions.create(
|
|
19
|
+
model="gpt-4o",
|
|
20
|
+
messages=[
|
|
21
|
+
{"role": "user", "content": "Email Rahul at rahul.sharma@example.com about PAN ABCPE1234F"}
|
|
22
|
+
],
|
|
23
|
+
)
|
|
24
|
+
# OpenAI sees: "Email Rahul at <EMAIL_1> about PAN <PAN_1>"
|
|
25
|
+
# You get back: the model's reply with <EMAIL_1>/<PAN_1> restored to the real values
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
## Endpoints
|
|
29
|
+
|
|
30
|
+
| Route | What it does |
|
|
31
|
+
|---|---|
|
|
32
|
+
| `POST /v1/chat/completions` | OpenAI chat, streaming + non-streaming, tool calls |
|
|
33
|
+
| `POST /v1/messages` | Anthropic Messages, streaming + non-streaming, tool use |
|
|
34
|
+
| `POST /v1/embeddings` | masks each input **before** it is embedded — the RAG path |
|
|
35
|
+
| `POST /v1/mask` / `POST /v1/unmask` | direct masking, no upstream call |
|
|
36
|
+
| `GET /healthz` | liveness (never touches Redis) |
|
|
37
|
+
| `GET /readyz` | readiness — **503 fail-closed** if Redis `maxmemory-policy != noeviction` |
|
|
38
|
+
| `GET /metrics` | Prometheus |
|
|
39
|
+
| `GET /v1/entities` | every PII type the loaded packs detect |
|
|
40
|
+
|
|
41
|
+
## Streaming unmask
|
|
42
|
+
|
|
43
|
+
The model streams the reply in arbitrary chunks and a placeholder like
|
|
44
|
+
`<PERSON_NAME_1>` can be split across two SSE frames. The gateway parses
|
|
45
|
+
the provider's SSE, keeps a rolling buffer + a trie of the session's active
|
|
46
|
+
placeholders, and emits the longest prefix that is *certain* — a completed
|
|
47
|
+
placeholder (replaced) or a character that cannot begin any placeholder —
|
|
48
|
+
retaining only the tail that could still grow into one. A two-layer
|
|
49
|
+
decoder handles chunk splits mid-UTF-8. Property: for **any** chunking of a
|
|
50
|
+
masked reply, the concatenated stream equals the non-streaming
|
|
51
|
+
`unmask` result (fuzz-tested at every byte boundary).
|
|
52
|
+
|
|
53
|
+
## Sessions (multi-turn / tool loops)
|
|
54
|
+
|
|
55
|
+
Send `X-Maskflow-Session: <your opaque id>` to keep `<PHONE_1>` meaning the
|
|
56
|
+
same number across every turn and tool call of one agent run. Without the
|
|
57
|
+
header each request is masked and unmasked in isolation.
|
|
58
|
+
|
|
59
|
+
- `X-Maskflow-Session-TTL: <seconds>` overrides the default (3600), capped
|
|
60
|
+
at `MASKFLOW_GATEWAY_SESSION_TTL_MAX_SECONDS` (86400).
|
|
61
|
+
- Keyed sessions need Redis (`MASKFLOW_GATEWAY_REDIS_URL`). Mappings are
|
|
62
|
+
encrypted with **AES-256-GCM** (`MASKFLOW_GATEWAY_SESSION_KEY`, 32 random
|
|
63
|
+
bytes hex) before they touch Redis, with a mandatory TTL.
|
|
64
|
+
- Redis **must** run `maxmemory-policy noeviction` — an evicted session
|
|
65
|
+
mid-conversation means unmask finds nothing and a user sees raw
|
|
66
|
+
`<PERSON_NAME_1>` text. `/readyz` returns 503 until this is fixed.
|
|
67
|
+
|
|
68
|
+
## Configuration
|
|
69
|
+
|
|
70
|
+
Every setting is an environment variable prefixed `MASKFLOW_GATEWAY_`:
|
|
71
|
+
|
|
72
|
+
| Variable | Default | Notes |
|
|
73
|
+
|---|---|---|
|
|
74
|
+
| `OPENAI_BASE_URL` | `https://api.openai.com/v1` | must include the version segment |
|
|
75
|
+
| `ANTHROPIC_BASE_URL` | `https://api.anthropic.com/v1` | |
|
|
76
|
+
| `UPSTREAM_API_KEY` | *(unset)* | set → gateway injects it; unset → client's own key is forwarded, nothing stored |
|
|
77
|
+
| `NER` | `0` | `1` enables the spaCy pass (bare Indian names & addresses); much slower — see `loadtest/` |
|
|
78
|
+
| `MIN_CONFIDENCE` | `0.5` | detection threshold |
|
|
79
|
+
| `REDIS_URL` | *(unset)* | unset → in-process ephemeral sessions only (single replica) |
|
|
80
|
+
| `SESSION_KEY` | *(unset)* | hex, 32 bytes; **required** when `REDIS_URL` is set |
|
|
81
|
+
| `SESSION_TTL_SECONDS` / `SESSION_TTL_MAX_SECONDS` | `3600` / `86400` | |
|
|
82
|
+
| `REQUIRE_MAXMEMORY_NOEVICTION` | `true` | |
|
|
83
|
+
| `MAX_REQUEST_BYTES` | `2000000` | |
|
|
84
|
+
| `UPSTREAM_TIMEOUT_SECONDS` / `UPSTREAM_CONNECT_TIMEOUT_SECONDS` | `120` / `10` | |
|
|
85
|
+
| `RATE_LIMIT_PER_MINUTE` / `RATE_LIMIT_BURST` | `0` (off) | keyed by a hash of the client's `Authorization` |
|
|
86
|
+
| `TOOL_CALL_MAX_DEPTH` / `TOOL_CALL_MAX_ITEMS` | `32` / `10000` | bound the tool-argument JSON walk |
|
|
87
|
+
| `JSON_LOGS` | `1` | structured logs; every record also passes MaskFlow's PII scrub filter |
|
|
88
|
+
| `CORS_ALLOW_ORIGINS` | `[]` | JSON list |
|
|
89
|
+
|
|
90
|
+
The gateway's masking config comes only from these variables — it does not
|
|
91
|
+
read a `.maskflowrc`.
|
|
92
|
+
|
|
93
|
+
## Run
|
|
94
|
+
|
|
95
|
+
```bash
|
|
96
|
+
pip install "maskflow-gateway[redis]"
|
|
97
|
+
export MASKFLOW_GATEWAY_REDIS_URL=redis://localhost:6379/0
|
|
98
|
+
export MASKFLOW_GATEWAY_SESSION_KEY=$(python -c "import os;print(os.urandom(32).hex())")
|
|
99
|
+
maskflow-gateway --host 0.0.0.0 --port 8000 --workers 4
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
Deploy artifacts in [`deploy/`](deploy/): multi-arch `Dockerfile`,
|
|
103
|
+
`docker-compose.yml` (gateway + noeviction Redis), a Helm chart
|
|
104
|
+
(`deploy/helm/maskflow-gateway`, with HPA / PDB / ServiceMonitor), and
|
|
105
|
+
Fly / Render / Railway templates.
|
|
106
|
+
|
|
107
|
+
## Throughput
|
|
108
|
+
|
|
109
|
+
Published honestly, with the hardware, in [`loadtest/README.md`](loadtest/README.md).
|
|
110
|
+
Rough laptop floor: **~430 req/s** pattern-only, **~90 req/s** NER-enabled
|
|
111
|
+
(4 workers, Intel i7-9750H).
|
|
112
|
+
|
|
113
|
+
## License
|
|
114
|
+
|
|
115
|
+
MIT — like the rest of MaskFlow. No license gates, no paid flags, no
|
|
116
|
+
telemetry.
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
# MaskFlow Gateway -- OpenAI/Anthropic-compatible masking proxy.
|
|
2
|
+
#
|
|
3
|
+
# Build (multi-arch in CI via buildx):
|
|
4
|
+
# docker build -f packages/maskflow-gateway/deploy/Dockerfile \
|
|
5
|
+
# --build-arg MASKFLOW_GATEWAY_VERSION=0.1.0 \
|
|
6
|
+
# -t maskflow/gateway packages/maskflow-gateway
|
|
7
|
+
#
|
|
8
|
+
# The spaCy model is baked in so MASKFLOW_GATEWAY_NER=1 works out of the
|
|
9
|
+
# box; it is inert (never loaded) when NER is off, which is the default.
|
|
10
|
+
#
|
|
11
|
+
# Published as ghcr.io/maskflow/gateway:<version> and :latest by
|
|
12
|
+
# .github/workflows/release-gateway.yml on a `gateway-v*` tag.
|
|
13
|
+
|
|
14
|
+
FROM python:3.12-slim AS build
|
|
15
|
+
|
|
16
|
+
ARG MASKFLOW_GATEWAY_VERSION=""
|
|
17
|
+
ENV PIP_NO_CACHE_DIR=1 PIP_DISABLE_PIP_VERSION_CHECK=1
|
|
18
|
+
|
|
19
|
+
RUN python -m venv /opt/venv
|
|
20
|
+
ENV PATH="/opt/venv/bin:$PATH"
|
|
21
|
+
|
|
22
|
+
RUN set -eux; \
|
|
23
|
+
pip install "maskflow-gateway[redis]${MASKFLOW_GATEWAY_VERSION:+==${MASKFLOW_GATEWAY_VERSION}}"; \
|
|
24
|
+
python -m spacy download en_core_web_sm; \
|
|
25
|
+
python -c "import maskflow_gateway.app, maskflow_pack_india, maskflow_pack_intl"
|
|
26
|
+
|
|
27
|
+
FROM python:3.12-slim
|
|
28
|
+
|
|
29
|
+
ENV PATH="/opt/venv/bin:$PATH" \
|
|
30
|
+
PYTHONUNBUFFERED=1 \
|
|
31
|
+
MASKFLOW_GATEWAY_JSON_LOGS=1
|
|
32
|
+
COPY --from=build /opt/venv /opt/venv
|
|
33
|
+
|
|
34
|
+
RUN useradd --create-home --uid 10001 gateway
|
|
35
|
+
USER gateway
|
|
36
|
+
WORKDIR /home/gateway
|
|
37
|
+
|
|
38
|
+
EXPOSE 8000
|
|
39
|
+
# Liveness only -- /readyz (which pings Redis) is for the orchestrator.
|
|
40
|
+
HEALTHCHECK --interval=15s --timeout=3s --start-period=5s --retries=3 \
|
|
41
|
+
CMD python -c "import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8000/healthz').status==200 else 1)"
|
|
42
|
+
|
|
43
|
+
ENTRYPOINT ["maskflow-gateway"]
|
|
44
|
+
CMD ["--host", "0.0.0.0", "--port", "8000"]
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# Local / single-box deployment: gateway + Redis with a mandatory
|
|
2
|
+
# noeviction policy (an evicted session = a failed unmask = raw
|
|
3
|
+
# placeholders shown to a user).
|
|
4
|
+
#
|
|
5
|
+
# export MASKFLOW_GATEWAY_SESSION_KEY=$(python -c "import os;print(os.urandom(32).hex())")
|
|
6
|
+
# docker compose -f packages/maskflow-gateway/deploy/docker-compose.yml up
|
|
7
|
+
|
|
8
|
+
services:
|
|
9
|
+
gateway:
|
|
10
|
+
image: ghcr.io/maskflow/gateway:latest
|
|
11
|
+
build:
|
|
12
|
+
context: ..
|
|
13
|
+
dockerfile: deploy/Dockerfile
|
|
14
|
+
ports:
|
|
15
|
+
- "8000:8000"
|
|
16
|
+
environment:
|
|
17
|
+
MASKFLOW_GATEWAY_REDIS_URL: redis://redis:6379/0
|
|
18
|
+
MASKFLOW_GATEWAY_SESSION_KEY: ${MASKFLOW_GATEWAY_SESSION_KEY:?set MASKFLOW_GATEWAY_SESSION_KEY to 32 random bytes, hex-encoded}
|
|
19
|
+
MASKFLOW_GATEWAY_NER: ${MASKFLOW_GATEWAY_NER:-0}
|
|
20
|
+
MASKFLOW_GATEWAY_OPENAI_BASE_URL: ${MASKFLOW_GATEWAY_OPENAI_BASE_URL:-https://api.openai.com/v1}
|
|
21
|
+
MASKFLOW_GATEWAY_ANTHROPIC_BASE_URL: ${MASKFLOW_GATEWAY_ANTHROPIC_BASE_URL:-https://api.anthropic.com/v1}
|
|
22
|
+
MASKFLOW_GATEWAY_RATE_LIMIT_PER_MINUTE: ${MASKFLOW_GATEWAY_RATE_LIMIT_PER_MINUTE:-0}
|
|
23
|
+
depends_on:
|
|
24
|
+
redis:
|
|
25
|
+
condition: service_healthy
|
|
26
|
+
restart: unless-stopped
|
|
27
|
+
|
|
28
|
+
redis:
|
|
29
|
+
image: redis:7-alpine
|
|
30
|
+
command: ["redis-server", "--maxmemory-policy", "noeviction", "--save", "", "--appendonly", "no"]
|
|
31
|
+
healthcheck:
|
|
32
|
+
test: ["CMD", "redis-cli", "ping"]
|
|
33
|
+
interval: 5s
|
|
34
|
+
timeout: 3s
|
|
35
|
+
retries: 5
|
|
36
|
+
restart: unless-stopped
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
# fly.io -- `fly launch --copy-config --dockerfile deploy/Dockerfile` from
|
|
2
|
+
# packages/maskflow-gateway, then:
|
|
3
|
+
# fly secrets set MASKFLOW_GATEWAY_SESSION_KEY=$(python -c "import os;print(os.urandom(32).hex())")
|
|
4
|
+
# fly redis create # and set MASKFLOW_GATEWAY_REDIS_URL to its URL
|
|
5
|
+
# Ensure the Redis has maxmemory-policy=noeviction (Upstash: eviction OFF).
|
|
6
|
+
|
|
7
|
+
app = "maskflow-gateway"
|
|
8
|
+
primary_region = "bom"
|
|
9
|
+
|
|
10
|
+
[build]
|
|
11
|
+
dockerfile = "deploy/Dockerfile"
|
|
12
|
+
|
|
13
|
+
[env]
|
|
14
|
+
MASKFLOW_GATEWAY_NER = "0"
|
|
15
|
+
MASKFLOW_GATEWAY_JSON_LOGS = "1"
|
|
16
|
+
|
|
17
|
+
[http_service]
|
|
18
|
+
internal_port = 8000
|
|
19
|
+
force_https = true
|
|
20
|
+
auto_stop_machines = "suspend"
|
|
21
|
+
auto_start_machines = true
|
|
22
|
+
min_machines_running = 1
|
|
23
|
+
|
|
24
|
+
[[http_service.checks]]
|
|
25
|
+
method = "GET"
|
|
26
|
+
path = "/readyz"
|
|
27
|
+
interval = "10s"
|
|
28
|
+
timeout = "3s"
|
|
29
|
+
|
|
30
|
+
[[vm]]
|
|
31
|
+
size = "shared-cpu-2x"
|
|
32
|
+
memory = "512mb"
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
apiVersion: v2
|
|
2
|
+
name: maskflow-gateway
|
|
3
|
+
description: OpenAI/Anthropic-compatible proxy that masks PII before it reaches an LLM provider and restores it in the response, streaming included.
|
|
4
|
+
type: application
|
|
5
|
+
version: 0.1.0
|
|
6
|
+
appVersion: "0.1.0"
|
|
7
|
+
home: https://maskflow.in
|
|
8
|
+
sources:
|
|
9
|
+
- https://github.com/maskflow/maskflow
|
|
10
|
+
keywords:
|
|
11
|
+
- pii
|
|
12
|
+
- llm
|
|
13
|
+
- proxy
|
|
14
|
+
- dpdp
|
|
15
|
+
- privacy
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
MaskFlow Gateway {{ .Chart.AppVersion }} deployed as {{ include "maskflow-gateway.fullname" . }}.
|
|
2
|
+
|
|
3
|
+
Point your OpenAI / Anthropic client's base URL at the service:
|
|
4
|
+
|
|
5
|
+
http://{{ include "maskflow-gateway.fullname" . }}.{{ .Release.Namespace }}.svc:{{ .Values.service.port }}/v1
|
|
6
|
+
|
|
7
|
+
Checklist:
|
|
8
|
+
{{- if .Values.redis.url }}
|
|
9
|
+
* Redis: {{ .Values.redis.url }} -- confirm `maxmemory-policy noeviction`
|
|
10
|
+
(the gateway's /readyz returns 503 otherwise, and an evicted session
|
|
11
|
+
means a user sees raw <PERSON_NAME_1> placeholders).
|
|
12
|
+
{{- else }}
|
|
13
|
+
* No Redis configured: sessions are in-process. Keep replicaCount: 1 or
|
|
14
|
+
two replicas will disagree on placeholder identity.
|
|
15
|
+
{{- end }}
|
|
16
|
+
{{- if not (or .Values.sessionKey.existingSecret .Values.sessionKey.value) }}
|
|
17
|
+
* MASKFLOW_GATEWAY_SESSION_KEY is NOT set. Redis sessions need it
|
|
18
|
+
(AES-256-GCM at rest). Set sessionKey.existingSecret or sessionKey.value.
|
|
19
|
+
{{- end }}
|
|
20
|
+
* NER pass is {{ if eq (toString (index .Values.env "MASKFLOW_GATEWAY_NER")) "1" }}ON{{ else }}OFF{{ end }}.
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
{{- define "maskflow-gateway.name" -}}
|
|
2
|
+
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
|
|
3
|
+
{{- end -}}
|
|
4
|
+
|
|
5
|
+
{{- define "maskflow-gateway.fullname" -}}
|
|
6
|
+
{{- if .Values.fullnameOverride -}}
|
|
7
|
+
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
|
|
8
|
+
{{- else -}}
|
|
9
|
+
{{- printf "%s-%s" .Release.Name (include "maskflow-gateway.name" .) | trunc 63 | trimSuffix "-" -}}
|
|
10
|
+
{{- end -}}
|
|
11
|
+
{{- end -}}
|
|
12
|
+
|
|
13
|
+
{{- define "maskflow-gateway.labels" -}}
|
|
14
|
+
app.kubernetes.io/name: {{ include "maskflow-gateway.name" . }}
|
|
15
|
+
app.kubernetes.io/instance: {{ .Release.Name }}
|
|
16
|
+
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
|
|
17
|
+
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
|
18
|
+
helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version }}
|
|
19
|
+
{{- end -}}
|
|
20
|
+
|
|
21
|
+
{{- define "maskflow-gateway.selectorLabels" -}}
|
|
22
|
+
app.kubernetes.io/name: {{ include "maskflow-gateway.name" . }}
|
|
23
|
+
app.kubernetes.io/instance: {{ .Release.Name }}
|
|
24
|
+
{{- end -}}
|
|
25
|
+
|
|
26
|
+
{{- define "maskflow-gateway.image" -}}
|
|
27
|
+
{{- printf "%s:%s" .Values.image.repository (default .Chart.AppVersion .Values.image.tag) -}}
|
|
28
|
+
{{- end -}}
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
apiVersion: apps/v1
|
|
2
|
+
kind: Deployment
|
|
3
|
+
metadata:
|
|
4
|
+
name: {{ include "maskflow-gateway.fullname" . }}
|
|
5
|
+
labels:
|
|
6
|
+
{{- include "maskflow-gateway.labels" . | nindent 4 }}
|
|
7
|
+
spec:
|
|
8
|
+
{{- if not .Values.autoscaling.enabled }}
|
|
9
|
+
replicas: {{ .Values.replicaCount }}
|
|
10
|
+
{{- end }}
|
|
11
|
+
selector:
|
|
12
|
+
matchLabels:
|
|
13
|
+
{{- include "maskflow-gateway.selectorLabels" . | nindent 6 }}
|
|
14
|
+
template:
|
|
15
|
+
metadata:
|
|
16
|
+
labels:
|
|
17
|
+
{{- include "maskflow-gateway.selectorLabels" . | nindent 8 }}
|
|
18
|
+
{{- with .Values.podAnnotations }}
|
|
19
|
+
annotations:
|
|
20
|
+
{{- toYaml . | nindent 8 }}
|
|
21
|
+
{{- end }}
|
|
22
|
+
spec:
|
|
23
|
+
securityContext:
|
|
24
|
+
runAsNonRoot: true
|
|
25
|
+
runAsUser: 10001
|
|
26
|
+
fsGroup: 10001
|
|
27
|
+
containers:
|
|
28
|
+
- name: gateway
|
|
29
|
+
image: {{ include "maskflow-gateway.image" . }}
|
|
30
|
+
imagePullPolicy: {{ .Values.image.pullPolicy }}
|
|
31
|
+
ports:
|
|
32
|
+
- name: http
|
|
33
|
+
containerPort: 8000
|
|
34
|
+
env:
|
|
35
|
+
{{- range $k, $v := .Values.env }}
|
|
36
|
+
- name: {{ $k }}
|
|
37
|
+
value: {{ $v | quote }}
|
|
38
|
+
{{- end }}
|
|
39
|
+
{{- if .Values.redis.url }}
|
|
40
|
+
- name: MASKFLOW_GATEWAY_REDIS_URL
|
|
41
|
+
value: {{ .Values.redis.url | quote }}
|
|
42
|
+
{{- end }}
|
|
43
|
+
- name: MASKFLOW_GATEWAY_SESSION_KEY
|
|
44
|
+
valueFrom:
|
|
45
|
+
secretKeyRef:
|
|
46
|
+
name: {{ .Values.sessionKey.existingSecret | default (include "maskflow-gateway.fullname" .) }}
|
|
47
|
+
key: session-key
|
|
48
|
+
optional: {{ not (or .Values.sessionKey.existingSecret .Values.sessionKey.value) }}
|
|
49
|
+
{{- if or .Values.upstreamApiKey.existingSecret .Values.upstreamApiKey.value }}
|
|
50
|
+
- name: MASKFLOW_GATEWAY_UPSTREAM_API_KEY
|
|
51
|
+
valueFrom:
|
|
52
|
+
secretKeyRef:
|
|
53
|
+
name: {{ .Values.upstreamApiKey.existingSecret | default (include "maskflow-gateway.fullname" .) }}
|
|
54
|
+
key: upstream-api-key
|
|
55
|
+
{{- end }}
|
|
56
|
+
resources:
|
|
57
|
+
{{- toYaml .Values.resources | nindent 12 }}
|
|
58
|
+
startupProbe:
|
|
59
|
+
httpGet:
|
|
60
|
+
path: /healthz
|
|
61
|
+
port: http
|
|
62
|
+
failureThreshold: 30
|
|
63
|
+
periodSeconds: 2
|
|
64
|
+
livenessProbe:
|
|
65
|
+
httpGet:
|
|
66
|
+
path: /healthz
|
|
67
|
+
port: http
|
|
68
|
+
periodSeconds: 10
|
|
69
|
+
readinessProbe:
|
|
70
|
+
httpGet:
|
|
71
|
+
path: /readyz
|
|
72
|
+
port: http
|
|
73
|
+
periodSeconds: 10
|
|
74
|
+
securityContext:
|
|
75
|
+
allowPrivilegeEscalation: false
|
|
76
|
+
readOnlyRootFilesystem: true
|
|
77
|
+
capabilities:
|
|
78
|
+
drop: ["ALL"]
|
|
79
|
+
{{- with .Values.nodeSelector }}
|
|
80
|
+
nodeSelector:
|
|
81
|
+
{{- toYaml . | nindent 8 }}
|
|
82
|
+
{{- end }}
|
|
83
|
+
{{- with .Values.tolerations }}
|
|
84
|
+
tolerations:
|
|
85
|
+
{{- toYaml . | nindent 8 }}
|
|
86
|
+
{{- end }}
|
|
87
|
+
{{- with .Values.affinity }}
|
|
88
|
+
affinity:
|
|
89
|
+
{{- toYaml . | nindent 8 }}
|
|
90
|
+
{{- end }}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
{{- if .Values.autoscaling.enabled -}}
|
|
2
|
+
apiVersion: autoscaling/v2
|
|
3
|
+
kind: HorizontalPodAutoscaler
|
|
4
|
+
metadata:
|
|
5
|
+
name: {{ include "maskflow-gateway.fullname" . }}
|
|
6
|
+
labels:
|
|
7
|
+
{{- include "maskflow-gateway.labels" . | nindent 4 }}
|
|
8
|
+
spec:
|
|
9
|
+
scaleTargetRef:
|
|
10
|
+
apiVersion: apps/v1
|
|
11
|
+
kind: Deployment
|
|
12
|
+
name: {{ include "maskflow-gateway.fullname" . }}
|
|
13
|
+
minReplicas: {{ .Values.autoscaling.minReplicas }}
|
|
14
|
+
maxReplicas: {{ .Values.autoscaling.maxReplicas }}
|
|
15
|
+
metrics:
|
|
16
|
+
- type: Resource
|
|
17
|
+
resource:
|
|
18
|
+
name: cpu
|
|
19
|
+
target:
|
|
20
|
+
type: Utilization
|
|
21
|
+
averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }}
|
|
22
|
+
{{- end -}}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
{{- if .Values.ingress.enabled -}}
|
|
2
|
+
apiVersion: networking.k8s.io/v1
|
|
3
|
+
kind: Ingress
|
|
4
|
+
metadata:
|
|
5
|
+
name: {{ include "maskflow-gateway.fullname" . }}
|
|
6
|
+
labels:
|
|
7
|
+
{{- include "maskflow-gateway.labels" . | nindent 4 }}
|
|
8
|
+
{{- with .Values.ingress.annotations }}
|
|
9
|
+
annotations:
|
|
10
|
+
{{- toYaml . | nindent 4 }}
|
|
11
|
+
{{- end }}
|
|
12
|
+
spec:
|
|
13
|
+
{{- with .Values.ingress.className }}
|
|
14
|
+
ingressClassName: {{ . }}
|
|
15
|
+
{{- end }}
|
|
16
|
+
{{- with .Values.ingress.tls }}
|
|
17
|
+
tls:
|
|
18
|
+
{{- toYaml . | nindent 4 }}
|
|
19
|
+
{{- end }}
|
|
20
|
+
rules:
|
|
21
|
+
{{- range .Values.ingress.hosts }}
|
|
22
|
+
- host: {{ .host | quote }}
|
|
23
|
+
http:
|
|
24
|
+
paths:
|
|
25
|
+
{{- range .paths }}
|
|
26
|
+
- path: {{ .path }}
|
|
27
|
+
pathType: {{ .pathType }}
|
|
28
|
+
backend:
|
|
29
|
+
service:
|
|
30
|
+
name: {{ include "maskflow-gateway.fullname" $ }}
|
|
31
|
+
port:
|
|
32
|
+
number: {{ $.Values.service.port }}
|
|
33
|
+
{{- end }}
|
|
34
|
+
{{- end }}
|
|
35
|
+
{{- end -}}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
{{- if .Values.podDisruptionBudget.enabled -}}
|
|
2
|
+
apiVersion: policy/v1
|
|
3
|
+
kind: PodDisruptionBudget
|
|
4
|
+
metadata:
|
|
5
|
+
name: {{ include "maskflow-gateway.fullname" . }}
|
|
6
|
+
labels:
|
|
7
|
+
{{- include "maskflow-gateway.labels" . | nindent 4 }}
|
|
8
|
+
spec:
|
|
9
|
+
minAvailable: {{ .Values.podDisruptionBudget.minAvailable }}
|
|
10
|
+
selector:
|
|
11
|
+
matchLabels:
|
|
12
|
+
{{- include "maskflow-gateway.selectorLabels" . | nindent 6 }}
|
|
13
|
+
{{- end -}}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
{{- if or .Values.sessionKey.value .Values.upstreamApiKey.value -}}
|
|
2
|
+
apiVersion: v1
|
|
3
|
+
kind: Secret
|
|
4
|
+
metadata:
|
|
5
|
+
name: {{ include "maskflow-gateway.fullname" . }}
|
|
6
|
+
labels:
|
|
7
|
+
{{- include "maskflow-gateway.labels" . | nindent 4 }}
|
|
8
|
+
type: Opaque
|
|
9
|
+
stringData:
|
|
10
|
+
{{- if .Values.sessionKey.value }}
|
|
11
|
+
session-key: {{ .Values.sessionKey.value | quote }}
|
|
12
|
+
{{- end }}
|
|
13
|
+
{{- if .Values.upstreamApiKey.value }}
|
|
14
|
+
upstream-api-key: {{ .Values.upstreamApiKey.value | quote }}
|
|
15
|
+
{{- end }}
|
|
16
|
+
{{- end -}}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
apiVersion: v1
|
|
2
|
+
kind: Service
|
|
3
|
+
metadata:
|
|
4
|
+
name: {{ include "maskflow-gateway.fullname" . }}
|
|
5
|
+
labels:
|
|
6
|
+
{{- include "maskflow-gateway.labels" . | nindent 4 }}
|
|
7
|
+
spec:
|
|
8
|
+
type: {{ .Values.service.type }}
|
|
9
|
+
ports:
|
|
10
|
+
- name: http
|
|
11
|
+
port: {{ .Values.service.port }}
|
|
12
|
+
targetPort: http
|
|
13
|
+
selector:
|
|
14
|
+
{{- include "maskflow-gateway.selectorLabels" . | nindent 4 }}
|