cybat-sdk 0.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cybat_sdk-0.1.1/PKG-INFO +177 -0
- cybat_sdk-0.1.1/README.md +167 -0
- cybat_sdk-0.1.1/cybat_sdk/__init__.py +5 -0
- cybat_sdk-0.1.1/cybat_sdk/collector.py +212 -0
- cybat_sdk-0.1.1/cybat_sdk/integrations/__init__.py +0 -0
- cybat_sdk-0.1.1/cybat_sdk/integrations/asgi.py +143 -0
- cybat_sdk-0.1.1/cybat_sdk/integrations/django.py +142 -0
- cybat_sdk-0.1.1/cybat_sdk/integrations/flask.py +102 -0
- cybat_sdk-0.1.1/cybat_sdk/redact.py +309 -0
- cybat_sdk-0.1.1/cybat_sdk.egg-info/PKG-INFO +177 -0
- cybat_sdk-0.1.1/cybat_sdk.egg-info/SOURCES.txt +18 -0
- cybat_sdk-0.1.1/cybat_sdk.egg-info/dependency_links.txt +1 -0
- cybat_sdk-0.1.1/cybat_sdk.egg-info/top_level.txt +1 -0
- cybat_sdk-0.1.1/pyproject.toml +29 -0
- cybat_sdk-0.1.1/setup.cfg +4 -0
- cybat_sdk-0.1.1/tests/test_asgi_integration.py +200 -0
- cybat_sdk-0.1.1/tests/test_collector.py +238 -0
- cybat_sdk-0.1.1/tests/test_django_integration.py +101 -0
- cybat_sdk-0.1.1/tests/test_flask_integration.py +102 -0
- cybat_sdk-0.1.1/tests/test_redact.py +204 -0
cybat_sdk-0.1.1/PKG-INFO
ADDED
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: cybat-sdk
|
|
3
|
+
Version: 0.1.1
|
|
4
|
+
Summary: Cybat SDK for Python — WSGI/ASGI middleware that reports HTTP traffic directly to the Cybat Intelligence Engine
|
|
5
|
+
License-Expression: MIT
|
|
6
|
+
Project-URL: Homepage, https://cybat.co
|
|
7
|
+
Project-URL: Documentation, https://cybat.co/#/docs
|
|
8
|
+
Requires-Python: >=3.9
|
|
9
|
+
Description-Content-Type: text/markdown
|
|
10
|
+
|
|
11
|
+
# cybat-sdk (Python)
|
|
12
|
+
|
|
13
|
+
Flask, Django and ASGI (FastAPI / Starlette / any ASGI app) integration for the Cybat Intelligence Engine. Captures HTTP traffic your app already receives and reports it directly to Cybat — no GCP org-level log sink required.
|
|
14
|
+
|
|
15
|
+
## Install
|
|
16
|
+
|
|
17
|
+
```bash
|
|
18
|
+
pip install cybat-sdk
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
No required dependencies — Flask/Django/etc. are assumed already installed by your project; only the integration module for the framework you actually use imports it.
|
|
22
|
+
|
|
23
|
+
## Usage
|
|
24
|
+
|
|
25
|
+
### Flask
|
|
26
|
+
|
|
27
|
+
```python
|
|
28
|
+
from flask import Flask
|
|
29
|
+
from cybat_sdk.integrations.flask import init_app
|
|
30
|
+
|
|
31
|
+
app = Flask(__name__)
|
|
32
|
+
init_app(app,
|
|
33
|
+
api_key=os.environ["CYBAT_API_KEY"],
|
|
34
|
+
gcp_project_id="my-gcp-project", # enables automatic blocking — see below
|
|
35
|
+
service_name="checkout-api", # optional, just a label on incidents
|
|
36
|
+
)
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
### Django
|
|
40
|
+
|
|
41
|
+
Django instantiates middleware classes itself with no way to pass constructor kwargs, so configuration comes from `settings.py` instead of a function call:
|
|
42
|
+
|
|
43
|
+
```python
|
|
44
|
+
# settings.py
|
|
45
|
+
MIDDLEWARE = [
|
|
46
|
+
# ... your other middleware ...
|
|
47
|
+
"cybat_sdk.integrations.django.CybatMiddleware",
|
|
48
|
+
]
|
|
49
|
+
CYBAT_API_KEY = os.environ["CYBAT_API_KEY"]
|
|
50
|
+
CYBAT_GCP_PROJECT_ID = "my-gcp-project" # optional, enables automatic blocking
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
Every option has a `CYBAT_*` settings equivalent — see [Options](#options) below for the full list; the Django setting name is `CYBAT_` + the option name upper-cased (`retry_backoff_ms` → `CYBAT_RETRY_BACKOFF_MS`).
|
|
54
|
+
|
|
55
|
+
### FastAPI / Starlette / any ASGI app
|
|
56
|
+
|
|
57
|
+
```python
|
|
58
|
+
from cybat_sdk.integrations.asgi import CybatASGIMiddleware
|
|
59
|
+
|
|
60
|
+
app.add_middleware(CybatASGIMiddleware,
|
|
61
|
+
api_key=os.environ["CYBAT_API_KEY"],
|
|
62
|
+
gcp_project_id="my-gcp-project",
|
|
63
|
+
)
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
This speaks the raw ASGI protocol (`scope`/`receive`/`send`) directly rather than wrapping a framework-specific base class, so it works correctly with streaming responses too, and needs no FastAPI/Starlette import. `add_middleware(cls, **options)` instantiates `cls(app, **options)`, which is exactly this class's constructor — no adapter needed. For a bare ASGI app without that helper: `app = CybatASGIMiddleware(app, api_key=...)`.
|
|
67
|
+
|
|
68
|
+
Create an API key from the Intelligence Dashboard → **API Keys**. It's shown once at creation — store it as a secret (`CYBAT_API_KEY`), it can't be retrieved again.
|
|
69
|
+
|
|
70
|
+
## Multi-cloud routing
|
|
71
|
+
|
|
72
|
+
Set **one** of `gcp_project_id` or `aws_project_id` — never both on the same SDK instance. Which one you set decides where data is sent, automatically:
|
|
73
|
+
|
|
74
|
+
| You set | Data goes to | Status |
|
|
75
|
+
|---------|--------------|--------|
|
|
76
|
+
| `gcp_project_id` (or neither) | `gcp-ingest.cybat.co` | Live |
|
|
77
|
+
| `aws_project_id` | `aws-ingest.cybat.co` | **Not yet implemented** — batches will fail to send until AWS-based ingestion ships |
|
|
78
|
+
|
|
79
|
+
An app that runs in both clouds needs two SDK instances, one per cloud — not one instance with both fields set (the constructor raises if you do that, since it's ambiguous which cloud's infrastructure should be enforced against).
|
|
80
|
+
|
|
81
|
+
## ⚠️ Automatic blocking requires `gcp_project_id`
|
|
82
|
+
|
|
83
|
+
Cybat classifies and alerts on threats regardless of `gcp_project_id`. But **automatically blocking** a malicious IP (VPC Firewall / Cloud Armor) requires knowing which GCP project to act on — your app might not even be hosted on GCP, so there's no way to infer this from the traffic itself. If you omit `gcp_project_id`, threats are still detected and you still get alerted, but nothing gets blocked automatically.
|
|
84
|
+
|
|
85
|
+
To enable blocking:
|
|
86
|
+
1. Set `gcp_project_id` to the GCP project your app's infrastructure lives in.
|
|
87
|
+
2. Grant `soc-ai-agent@cybat-public.iam.gserviceaccount.com` the `roles/compute.securityAdmin` and `roles/compute.networkViewer` roles on that project (a Cloud Shell command for this is shown in the Intelligence Dashboard onboarding flow).
|
|
88
|
+
|
|
89
|
+
## ⚠️ Behind a load balancer or reverse proxy?
|
|
90
|
+
|
|
91
|
+
Each integration resolves the client IP from the framework's own notion of it (Flask's `request.remote_addr`, Django's `request.META["REMOTE_ADDR"]`, the ASGI `scope["client"]`), which reflects the *direct* TCP peer. If your app sits behind your own load balancer, CDN, or reverse proxy, configure that framework's standard mechanism for trusting `X-Forwarded-For` — otherwise every request reports the proxy's IP instead of the real client, and Cybat won't be able to tell your traffic apart:
|
|
92
|
+
|
|
93
|
+
- **Flask**: wrap your WSGI app with Werkzeug's `ProxyFix`.
|
|
94
|
+
- **Django**: set `SECURE_PROXY_SSL_HEADER` / put a proxy-aware middleware before Cybat's in `MIDDLEWARE`, per Django's deployment docs.
|
|
95
|
+
- **ASGI**: use Starlette's/Uvicorn's `--proxy-headers --forwarded-allow-ips` (or `ProxyHeadersMiddleware`).
|
|
96
|
+
|
|
97
|
+
## Options
|
|
98
|
+
|
|
99
|
+
All three integrations take the same options (Flask/ASGI as keyword arguments, Django as `CYBAT_*` settings):
|
|
100
|
+
|
|
101
|
+
```python
|
|
102
|
+
{
|
|
103
|
+
"api_key": str, # required
|
|
104
|
+
"gcp_project_id": str, # enables automatic blocking + routes to GCP — see above
|
|
105
|
+
"aws_project_id": str, # routes to AWS — NOT YET LIVE, see "Multi-cloud routing"
|
|
106
|
+
"service_name": str, # optional label shown on incidents
|
|
107
|
+
"ingest_url": str, # overrides cloud auto-detection entirely; default depends on gcp_project_id/aws_project_id
|
|
108
|
+
"flush_interval_ms": 2000,
|
|
109
|
+
"max_batch_size": 100, # flushes immediately at this size
|
|
110
|
+
"max_retries": 2, # a batch is dropped, not queued forever, after this
|
|
111
|
+
"retry_backoff_ms": 500, # base delay before the first retry; doubles each retry (exponential)
|
|
112
|
+
"status_filter": "all", # "all" | "errors" | a callable (status: int) -> bool
|
|
113
|
+
"capture_body": False, # send a redacted excerpt of the request body — see "Request bodies & privacy"
|
|
114
|
+
"body_max_bytes": 2048, # default 2048, max 8192 — cap on the body excerpt after redaction
|
|
115
|
+
"body_status_filter": "all", # capture bodies only for matching responses
|
|
116
|
+
}
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
`status_filter` controls which captured requests actually get sent:
|
|
120
|
+
- `"all"` (default) — every request, regardless of status code.
|
|
121
|
+
- `"errors"` — only 4xx/5xx. Use this to cut ingestion volume/cost on high-traffic apps where 2xx telemetry isn't needed — Cybat still classifies and alerts on threats using only error responses, the same signal the old GCP Log Sink integration filtered down to.
|
|
122
|
+
- A callable — full control, e.g. `lambda status: status != 304` to drop cache-hit noise but keep everything else, 2xx included.
|
|
123
|
+
|
|
124
|
+
## Request bodies & privacy
|
|
125
|
+
|
|
126
|
+
Off by default. Set `capture_body=True` to include the request body in each event — that's where SQL/command injection, credential stuffing and prompt-injection against LLM-backed endpoints actually show up, so it meaningfully improves detection. It also means request content leaves your infrastructure, so **redaction runs in your process, before an event is even buffered, and can't be turned off** once capture is on:
|
|
127
|
+
|
|
128
|
+
| What | Becomes |
|
|
129
|
+
|------|---------|
|
|
130
|
+
| Emails | `j***n@example.com` |
|
|
131
|
+
| Card numbers (Luhn-checked, 13–19 digits) | `XXXX-XXXX-XXXX-1234` |
|
|
132
|
+
| JWTs, `Bearer …` / `Basic …` tokens | `[REDACTED jwt]`, `Bearer [REDACTED]` |
|
|
133
|
+
| Provider keys (AWS, Google, GitHub, Slack, Stripe) and PEM private keys | `[REDACTED key]` |
|
|
134
|
+
| The value of any field named like a secret (`password`, `token`, `secret`, `authorization`, `cookie`, `session`, `api_key`, `card`, `cvv`, `ssn`, `iban`…), in JSON, form bodies and query strings | `[REDACTED len=13 sus=quote,comment]` |
|
|
135
|
+
|
|
136
|
+
A redacted secret is replaced by a **content-free shape**, not blanked: its length plus which kinds of attack characters it contained (`quote`, `comment`, `sqlkw`, `angle`, `semicolon`, `traversal`, `cmdchar`, `template`, `nullbyte`). That's deliberate — if you blanked a `password` field entirely you'd also blank the SQL injection an attacker typed into it. With the shape, Cybat can still see "a 14-character password containing quote, comment and SQL-keyword characters" without ever seeing the value.
|
|
137
|
+
|
|
138
|
+
- JSON, form-encoded and text bodies only. Multipart, binary, and empty bodies are always skipped, regardless of `capture_body`.
|
|
139
|
+
- The excerpt is capped at `body_max_bytes` after redaction, centered on the most attack-looking part rather than just the first N bytes, and flagged `bodyTruncated` (on the wire) when it was cut.
|
|
140
|
+
- If redaction raises for any reason, no body is sent — never an unredacted one.
|
|
141
|
+
- `body_status_filter="errors"` captures bodies only for 4xx/5xx responses, which keeps volume (and what you send) down while still covering most attacks.
|
|
142
|
+
|
|
143
|
+
**URLs, referers and user agents are redacted the same way whether or not `capture_body` is on** — query strings routinely carry tokens and emails (`/reset?token=…&email=…`). Secret-named query values become the same shape, so `?password=' OR 1=1--` still reads as an injection attempt.
|
|
144
|
+
|
|
145
|
+
Redaction is pattern-based. It covers the common cases above but can't know that a field called `notes` happens to contain something private — if a route handles data you don't want to leave your infrastructure at all, leave `capture_body` off, or use `body_status_filter` / a `status_filter` callable to exclude it.
|
|
146
|
+
|
|
147
|
+
## Behavior
|
|
148
|
+
|
|
149
|
+
- Never raises into your app. A network error, a rejected request, or a misconfigured key results in a dropped batch and a printed warning, not a crash.
|
|
150
|
+
- Buffers in memory (thread-safe) and flushes on a background daemon thread every `flush_interval_ms` — there's no persistent queue in the SDK itself. If the process exits between flushes, buffered events are lost (matches the durability model of application logs in general). `collector.stop()` stops the background thread on shutdown; it does not force a final flush — call `collector.flush()` first if you need one.
|
|
151
|
+
- Uses only the standard library (`urllib`) for the HTTP call — no extra dependency.
|
|
152
|
+
- One HTTP event captured per request: method, URL, status, remote IP, user agent, referer, latency, response size, timestamp (plus a redacted body excerpt if `capture_body` is on).
|
|
153
|
+
- A failed send retries `max_retries` times with exponential backoff (`retry_backoff_ms`, `retry_backoff_ms * 2`, `retry_backoff_ms * 4`, ...) before the batch is dropped for good. Retries are scheduled on a background timer thread and never block the request that triggered them.
|
|
154
|
+
|
|
155
|
+
## Wire format
|
|
156
|
+
|
|
157
|
+
See [`../README.md`](../README.md) for the shared batch/auth contract every language SDK implements. Field names on the wire (`remoteIp`, `latencyMs`, `bodyTruncated`, ...) are camelCase per that shared contract even though this package's own Python API (`gcp_project_id`, `status_filter`, ...) is snake_case — that boundary is deliberate: the wire format is a cross-language contract, the options are this language's idiomatic API.
|
|
158
|
+
|
|
159
|
+
## Publishing (maintainers)
|
|
160
|
+
|
|
161
|
+
CI is `.github/workflows/publish-python.yml`, using PyPI Trusted Publishing (OIDC) — no API token stored anywhere. One-time setup on [pypi.org](https://pypi.org) *before* the first release (PyPI lets you register a "pending publisher" for a project that doesn't exist yet):
|
|
162
|
+
|
|
163
|
+
1. **pypi.org → Your projects → Publishing → Add a pending publisher**, with:
|
|
164
|
+
- PyPI project name: `cybat-sdk`
|
|
165
|
+
- Owner: `CybatInc`, Repository name: `SDK`, Workflow name: `publish-python.yml`, Environment name: `pypi`
|
|
166
|
+
2. **GitHub repo → Settings → Environments → New environment**, name it `pypi` (must match exactly). Optionally add Required reviewers here for a manual approval gate before every publish — the OIDC equivalent of the 2FA-gated staging step `publish-node.yml` uses for npm.
|
|
167
|
+
|
|
168
|
+
To release:
|
|
169
|
+
|
|
170
|
+
```bash
|
|
171
|
+
# bump "version" in pyproject.toml, commit that change, then tag THAT commit
|
|
172
|
+
git add pyproject.toml && git commit -m "chore: bump cybat-sdk to 0.1.1"
|
|
173
|
+
git tag 0.1.1
|
|
174
|
+
git push origin main --tags
|
|
175
|
+
```
|
|
176
|
+
|
|
177
|
+
Same bare `X.Y.Z` tag format as `@cybat/sdk-node`'s releases — what keeps one tag from triggering both workflows is each one's `paths` filter (`python/**` here, `node/**` for Node's), which only matches if the tagged commit itself touched that package's directory. Tag the version-bump commit itself, not an unrelated one, or the path filter won't match and nothing will publish. The workflow runs the full test suite before building, then fails the build outright if the tag doesn't match `pyproject.toml`'s version.
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
# cybat-sdk (Python)
|
|
2
|
+
|
|
3
|
+
Flask, Django and ASGI (FastAPI / Starlette / any ASGI app) integration for the Cybat Intelligence Engine. Captures HTTP traffic your app already receives and reports it directly to Cybat — no GCP org-level log sink required.
|
|
4
|
+
|
|
5
|
+
## Install
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
pip install cybat-sdk
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
No required dependencies — Flask/Django/etc. are assumed already installed by your project; only the integration module for the framework you actually use imports it.
|
|
12
|
+
|
|
13
|
+
## Usage
|
|
14
|
+
|
|
15
|
+
### Flask
|
|
16
|
+
|
|
17
|
+
```python
|
|
18
|
+
from flask import Flask
|
|
19
|
+
from cybat_sdk.integrations.flask import init_app
|
|
20
|
+
|
|
21
|
+
app = Flask(__name__)
|
|
22
|
+
init_app(app,
|
|
23
|
+
api_key=os.environ["CYBAT_API_KEY"],
|
|
24
|
+
gcp_project_id="my-gcp-project", # enables automatic blocking — see below
|
|
25
|
+
service_name="checkout-api", # optional, just a label on incidents
|
|
26
|
+
)
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
### Django
|
|
30
|
+
|
|
31
|
+
Django instantiates middleware classes itself with no way to pass constructor kwargs, so configuration comes from `settings.py` instead of a function call:
|
|
32
|
+
|
|
33
|
+
```python
|
|
34
|
+
# settings.py
|
|
35
|
+
MIDDLEWARE = [
|
|
36
|
+
# ... your other middleware ...
|
|
37
|
+
"cybat_sdk.integrations.django.CybatMiddleware",
|
|
38
|
+
]
|
|
39
|
+
CYBAT_API_KEY = os.environ["CYBAT_API_KEY"]
|
|
40
|
+
CYBAT_GCP_PROJECT_ID = "my-gcp-project" # optional, enables automatic blocking
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
Every option has a `CYBAT_*` settings equivalent — see [Options](#options) below for the full list; the Django setting name is `CYBAT_` + the option name upper-cased (`retry_backoff_ms` → `CYBAT_RETRY_BACKOFF_MS`).
|
|
44
|
+
|
|
45
|
+
### FastAPI / Starlette / any ASGI app
|
|
46
|
+
|
|
47
|
+
```python
|
|
48
|
+
from cybat_sdk.integrations.asgi import CybatASGIMiddleware
|
|
49
|
+
|
|
50
|
+
app.add_middleware(CybatASGIMiddleware,
|
|
51
|
+
api_key=os.environ["CYBAT_API_KEY"],
|
|
52
|
+
gcp_project_id="my-gcp-project",
|
|
53
|
+
)
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
This speaks the raw ASGI protocol (`scope`/`receive`/`send`) directly rather than wrapping a framework-specific base class, so it works correctly with streaming responses too, and needs no FastAPI/Starlette import. `add_middleware(cls, **options)` instantiates `cls(app, **options)`, which is exactly this class's constructor — no adapter needed. For a bare ASGI app without that helper: `app = CybatASGIMiddleware(app, api_key=...)`.
|
|
57
|
+
|
|
58
|
+
Create an API key from the Intelligence Dashboard → **API Keys**. It's shown once at creation — store it as a secret (`CYBAT_API_KEY`), it can't be retrieved again.
|
|
59
|
+
|
|
60
|
+
## Multi-cloud routing
|
|
61
|
+
|
|
62
|
+
Set **one** of `gcp_project_id` or `aws_project_id` — never both on the same SDK instance. Which one you set decides where data is sent, automatically:
|
|
63
|
+
|
|
64
|
+
| You set | Data goes to | Status |
|
|
65
|
+
|---------|--------------|--------|
|
|
66
|
+
| `gcp_project_id` (or neither) | `gcp-ingest.cybat.co` | Live |
|
|
67
|
+
| `aws_project_id` | `aws-ingest.cybat.co` | **Not yet implemented** — batches will fail to send until AWS-based ingestion ships |
|
|
68
|
+
|
|
69
|
+
An app that runs in both clouds needs two SDK instances, one per cloud — not one instance with both fields set (the constructor raises if you do that, since it's ambiguous which cloud's infrastructure should be enforced against).
|
|
70
|
+
|
|
71
|
+
## ⚠️ Automatic blocking requires `gcp_project_id`
|
|
72
|
+
|
|
73
|
+
Cybat classifies and alerts on threats regardless of `gcp_project_id`. But **automatically blocking** a malicious IP (VPC Firewall / Cloud Armor) requires knowing which GCP project to act on — your app might not even be hosted on GCP, so there's no way to infer this from the traffic itself. If you omit `gcp_project_id`, threats are still detected and you still get alerted, but nothing gets blocked automatically.
|
|
74
|
+
|
|
75
|
+
To enable blocking:
|
|
76
|
+
1. Set `gcp_project_id` to the GCP project your app's infrastructure lives in.
|
|
77
|
+
2. Grant `soc-ai-agent@cybat-public.iam.gserviceaccount.com` the `roles/compute.securityAdmin` and `roles/compute.networkViewer` roles on that project (a Cloud Shell command for this is shown in the Intelligence Dashboard onboarding flow).
|
|
78
|
+
|
|
79
|
+
## ⚠️ Behind a load balancer or reverse proxy?
|
|
80
|
+
|
|
81
|
+
Each integration resolves the client IP from the framework's own notion of it (Flask's `request.remote_addr`, Django's `request.META["REMOTE_ADDR"]`, the ASGI `scope["client"]`), which reflects the *direct* TCP peer. If your app sits behind your own load balancer, CDN, or reverse proxy, configure that framework's standard mechanism for trusting `X-Forwarded-For` — otherwise every request reports the proxy's IP instead of the real client, and Cybat won't be able to tell your traffic apart:
|
|
82
|
+
|
|
83
|
+
- **Flask**: wrap your WSGI app with Werkzeug's `ProxyFix`.
|
|
84
|
+
- **Django**: set `SECURE_PROXY_SSL_HEADER` / put a proxy-aware middleware before Cybat's in `MIDDLEWARE`, per Django's deployment docs.
|
|
85
|
+
- **ASGI**: use Starlette's/Uvicorn's `--proxy-headers --forwarded-allow-ips` (or `ProxyHeadersMiddleware`).
|
|
86
|
+
|
|
87
|
+
## Options
|
|
88
|
+
|
|
89
|
+
All three integrations take the same options (Flask/ASGI as keyword arguments, Django as `CYBAT_*` settings):
|
|
90
|
+
|
|
91
|
+
```python
|
|
92
|
+
{
|
|
93
|
+
"api_key": str, # required
|
|
94
|
+
"gcp_project_id": str, # enables automatic blocking + routes to GCP — see above
|
|
95
|
+
"aws_project_id": str, # routes to AWS — NOT YET LIVE, see "Multi-cloud routing"
|
|
96
|
+
"service_name": str, # optional label shown on incidents
|
|
97
|
+
"ingest_url": str, # overrides cloud auto-detection entirely; default depends on gcp_project_id/aws_project_id
|
|
98
|
+
"flush_interval_ms": 2000,
|
|
99
|
+
"max_batch_size": 100, # flushes immediately at this size
|
|
100
|
+
"max_retries": 2, # a batch is dropped, not queued forever, after this
|
|
101
|
+
"retry_backoff_ms": 500, # base delay before the first retry; doubles each retry (exponential)
|
|
102
|
+
"status_filter": "all", # "all" | "errors" | a callable (status: int) -> bool
|
|
103
|
+
"capture_body": False, # send a redacted excerpt of the request body — see "Request bodies & privacy"
|
|
104
|
+
"body_max_bytes": 2048, # default 2048, max 8192 — cap on the body excerpt after redaction
|
|
105
|
+
"body_status_filter": "all", # capture bodies only for matching responses
|
|
106
|
+
}
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
`status_filter` controls which captured requests actually get sent:
|
|
110
|
+
- `"all"` (default) — every request, regardless of status code.
|
|
111
|
+
- `"errors"` — only 4xx/5xx. Use this to cut ingestion volume/cost on high-traffic apps where 2xx telemetry isn't needed — Cybat still classifies and alerts on threats using only error responses, the same signal the old GCP Log Sink integration filtered down to.
|
|
112
|
+
- A callable — full control, e.g. `lambda status: status != 304` to drop cache-hit noise but keep everything else, 2xx included.
|
|
113
|
+
|
|
114
|
+
## Request bodies & privacy
|
|
115
|
+
|
|
116
|
+
Off by default. Set `capture_body=True` to include the request body in each event — that's where SQL/command injection, credential stuffing and prompt-injection against LLM-backed endpoints actually show up, so it meaningfully improves detection. It also means request content leaves your infrastructure, so **redaction runs in your process, before an event is even buffered, and can't be turned off** once capture is on:
|
|
117
|
+
|
|
118
|
+
| What | Becomes |
|
|
119
|
+
|------|---------|
|
|
120
|
+
| Emails | `j***n@example.com` |
|
|
121
|
+
| Card numbers (Luhn-checked, 13–19 digits) | `XXXX-XXXX-XXXX-1234` |
|
|
122
|
+
| JWTs, `Bearer …` / `Basic …` tokens | `[REDACTED jwt]`, `Bearer [REDACTED]` |
|
|
123
|
+
| Provider keys (AWS, Google, GitHub, Slack, Stripe) and PEM private keys | `[REDACTED key]` |
|
|
124
|
+
| The value of any field named like a secret (`password`, `token`, `secret`, `authorization`, `cookie`, `session`, `api_key`, `card`, `cvv`, `ssn`, `iban`…), in JSON, form bodies and query strings | `[REDACTED len=13 sus=quote,comment]` |
|
|
125
|
+
|
|
126
|
+
A redacted secret is replaced by a **content-free shape**, not blanked: its length plus which kinds of attack characters it contained (`quote`, `comment`, `sqlkw`, `angle`, `semicolon`, `traversal`, `cmdchar`, `template`, `nullbyte`). That's deliberate — if you blanked a `password` field entirely you'd also blank the SQL injection an attacker typed into it. With the shape, Cybat can still see "a 14-character password containing quote, comment and SQL-keyword characters" without ever seeing the value.
|
|
127
|
+
|
|
128
|
+
- JSON, form-encoded and text bodies only. Multipart, binary, and empty bodies are always skipped, regardless of `capture_body`.
|
|
129
|
+
- The excerpt is capped at `body_max_bytes` after redaction, centered on the most attack-looking part rather than just the first N bytes, and flagged `bodyTruncated` (on the wire) when it was cut.
|
|
130
|
+
- If redaction raises for any reason, no body is sent — never an unredacted one.
|
|
131
|
+
- `body_status_filter="errors"` captures bodies only for 4xx/5xx responses, which keeps volume (and what you send) down while still covering most attacks.
|
|
132
|
+
|
|
133
|
+
**URLs, referers and user agents are redacted the same way whether or not `capture_body` is on** — query strings routinely carry tokens and emails (`/reset?token=…&email=…`). Secret-named query values become the same shape, so `?password=' OR 1=1--` still reads as an injection attempt.
|
|
134
|
+
|
|
135
|
+
Redaction is pattern-based. It covers the common cases above but can't know that a field called `notes` happens to contain something private — if a route handles data you don't want to leave your infrastructure at all, leave `capture_body` off, or use `body_status_filter` / a `status_filter` callable to exclude it.
|
|
136
|
+
|
|
137
|
+
## Behavior
|
|
138
|
+
|
|
139
|
+
- Never raises into your app. A network error, a rejected request, or a misconfigured key results in a dropped batch and a printed warning, not a crash.
|
|
140
|
+
- Buffers in memory (thread-safe) and flushes on a background daemon thread every `flush_interval_ms` — there's no persistent queue in the SDK itself. If the process exits between flushes, buffered events are lost (matches the durability model of application logs in general). `collector.stop()` stops the background thread on shutdown; it does not force a final flush — call `collector.flush()` first if you need one.
|
|
141
|
+
- Uses only the standard library (`urllib`) for the HTTP call — no extra dependency.
|
|
142
|
+
- One HTTP event captured per request: method, URL, status, remote IP, user agent, referer, latency, response size, timestamp (plus a redacted body excerpt if `capture_body` is on).
|
|
143
|
+
- A failed send retries `max_retries` times with exponential backoff (`retry_backoff_ms`, `retry_backoff_ms * 2`, `retry_backoff_ms * 4`, ...) before the batch is dropped for good. Retries are scheduled on a background timer thread and never block the request that triggered them.
|
|
144
|
+
|
|
145
|
+
## Wire format
|
|
146
|
+
|
|
147
|
+
See [`../README.md`](../README.md) for the shared batch/auth contract every language SDK implements. Field names on the wire (`remoteIp`, `latencyMs`, `bodyTruncated`, ...) are camelCase per that shared contract even though this package's own Python API (`gcp_project_id`, `status_filter`, ...) is snake_case — that boundary is deliberate: the wire format is a cross-language contract, the options are this language's idiomatic API.
|
|
148
|
+
|
|
149
|
+
## Publishing (maintainers)
|
|
150
|
+
|
|
151
|
+
CI is `.github/workflows/publish-python.yml`, using PyPI Trusted Publishing (OIDC) — no API token stored anywhere. One-time setup on [pypi.org](https://pypi.org) *before* the first release (PyPI lets you register a "pending publisher" for a project that doesn't exist yet):
|
|
152
|
+
|
|
153
|
+
1. **pypi.org → Your projects → Publishing → Add a pending publisher**, with:
|
|
154
|
+
- PyPI project name: `cybat-sdk`
|
|
155
|
+
- Owner: `CybatInc`, Repository name: `SDK`, Workflow name: `publish-python.yml`, Environment name: `pypi`
|
|
156
|
+
2. **GitHub repo → Settings → Environments → New environment**, name it `pypi` (must match exactly). Optionally add Required reviewers here for a manual approval gate before every publish — the OIDC equivalent of the 2FA-gated staging step `publish-node.yml` uses for npm.
|
|
157
|
+
|
|
158
|
+
To release:
|
|
159
|
+
|
|
160
|
+
```bash
|
|
161
|
+
# bump "version" in pyproject.toml, commit that change, then tag THAT commit
|
|
162
|
+
git add pyproject.toml && git commit -m "chore: bump cybat-sdk to 0.1.1"
|
|
163
|
+
git tag 0.1.1
|
|
164
|
+
git push origin main --tags
|
|
165
|
+
```
|
|
166
|
+
|
|
167
|
+
Same bare `X.Y.Z` tag format as `@cybat/sdk-node`'s releases — what keeps one tag from triggering both workflows is each one's `paths` filter (`python/**` here, `node/**` for Node's), which only matches if the tagged commit itself touched that package's directory. Tag the version-bump commit itself, not an unrelated one, or the path filter won't match and nothing will publish. The workflow runs the full test suite before building, then fails the build outright if the tag doesn't match `pyproject.toml`'s version.
|
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Buffers captured HTTP events and flushes them to ingest-api (behind API Gateway).
|
|
3
|
+
Never raises into the host app -- a broken or unreachable ingest endpoint must
|
|
4
|
+
never take down the app being instrumented. On repeated failure the batch is
|
|
5
|
+
dropped rather than retried indefinitely (Pub/Sub is the durability layer on
|
|
6
|
+
our side; this SDK only needs to survive transient network blips).
|
|
7
|
+
|
|
8
|
+
Mirrors SDK/node/src/index.ts's CybatCollector -- same options, same defaults,
|
|
9
|
+
same wire format. See SDK/README.md for the wire contract every language SDK
|
|
10
|
+
implements identically.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import json
|
|
16
|
+
import re
|
|
17
|
+
import threading
|
|
18
|
+
import urllib.error
|
|
19
|
+
import urllib.request
|
|
20
|
+
from dataclasses import dataclass, field
|
|
21
|
+
from typing import Any, Callable, Dict, List, Optional, Union
|
|
22
|
+
|
|
23
|
+
from .redact import mask_body, snippet_around
|
|
24
|
+
|
|
25
|
+
StatusFilterFn = Callable[[int], bool]
|
|
26
|
+
# 'all' | 'errors' | a predicate -- see resolve_status_filter.
|
|
27
|
+
StatusFilter = Union[str, StatusFilterFn]
|
|
28
|
+
|
|
29
|
+
_DEFAULT_GCP_INGEST_URL = "https://gcp-ingest.cybat.co"
|
|
30
|
+
_DEFAULT_AWS_INGEST_URL = "https://aws-ingest.cybat.co"
|
|
31
|
+
_BODY_DEFAULT_BYTES = 2048
|
|
32
|
+
_BODY_HARD_MAX_BYTES = 8192
|
|
33
|
+
_CAPTURABLE_CONTENT_TYPE = re.compile(r"^(application/(json|[\w.+-]+\+json|x-www-form-urlencoded)|text/)", re.I)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def resolve_status_filter(filt: Optional[StatusFilter]) -> StatusFilterFn:
|
|
37
|
+
"""Resolves a StatusFilter option into a plain predicate, once, at construction."""
|
|
38
|
+
if callable(filt):
|
|
39
|
+
return filt
|
|
40
|
+
if filt == "errors":
|
|
41
|
+
return lambda status: status >= 400
|
|
42
|
+
return lambda status: True
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
@dataclass
|
|
46
|
+
class CybatOptions:
|
|
47
|
+
"""
|
|
48
|
+
Python-facing options use snake_case (idiomatic here); the wire format each
|
|
49
|
+
event is serialized to is still the shared camelCase contract in
|
|
50
|
+
SDK/README.md -- that boundary is deliberate, not an inconsistency.
|
|
51
|
+
"""
|
|
52
|
+
|
|
53
|
+
api_key: str
|
|
54
|
+
# The GCP project this app's traffic should be enforced against. Required for
|
|
55
|
+
# automatic blocking (VPC Firewall / Cloud Armor) -- without it, Cybat still
|
|
56
|
+
# detects and alerts on threats, but never touches infrastructure, since it has
|
|
57
|
+
# no project to act on. Also determines routing (mutually exclusive with
|
|
58
|
+
# aws_project_id -- one SDK instance targets one cloud).
|
|
59
|
+
gcp_project_id: Optional[str] = None
|
|
60
|
+
# NOT YET LIVE -- aws-ingest.cybat.co has no backend behind it yet. Setting this
|
|
61
|
+
# today means every batch fails to send until AWS-based ingestion ships.
|
|
62
|
+
aws_project_id: Optional[str] = None
|
|
63
|
+
# Optional label identifying which of the customer's services this is, shown on incidents.
|
|
64
|
+
service_name: Optional[str] = None
|
|
65
|
+
# Overrides cloud auto-detection entirely -- set this explicitly for local/staging testing.
|
|
66
|
+
ingest_url: Optional[str] = None
|
|
67
|
+
flush_interval_ms: int = 2000
|
|
68
|
+
max_batch_size: int = 100
|
|
69
|
+
# Retries on a failed send before the batch is dropped. Default: 2 (3 attempts total).
|
|
70
|
+
max_retries: int = 2
|
|
71
|
+
# Base delay in ms before the first retry; doubles on each subsequent retry
|
|
72
|
+
# (exponential backoff): 500ms then 1000ms by default.
|
|
73
|
+
retry_backoff_ms: int = 500
|
|
74
|
+
status_filter: StatusFilter = "all"
|
|
75
|
+
# Include the request body in each event. Default: off. See README's
|
|
76
|
+
# "Request bodies & privacy" -- redaction is not optional once this is on.
|
|
77
|
+
capture_body: bool = False
|
|
78
|
+
body_max_bytes: int = _BODY_DEFAULT_BYTES
|
|
79
|
+
body_status_filter: StatusFilter = "all"
|
|
80
|
+
|
|
81
|
+
def __post_init__(self) -> None:
|
|
82
|
+
if not self.api_key:
|
|
83
|
+
raise ValueError("[cybat-sdk] api_key is required")
|
|
84
|
+
self.body_max_bytes = min(_BODY_HARD_MAX_BYTES, max(256, self.body_max_bytes))
|
|
85
|
+
self.ingest_url = _resolve_ingest_url(self)
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _resolve_ingest_url(opts: CybatOptions) -> str:
|
|
89
|
+
"""
|
|
90
|
+
Picks which cloud's ingestion endpoint to send to. An explicit ingest_url
|
|
91
|
+
always wins (escape hatch for local/staging testing). Otherwise the choice
|
|
92
|
+
follows whichever project ID was configured -- gcp_project_id and
|
|
93
|
+
aws_project_id are mutually exclusive because one SDK instance represents one
|
|
94
|
+
app running in one cloud; setting both is almost certainly a copy-paste
|
|
95
|
+
mistake, not a deliberate multi-cloud single app.
|
|
96
|
+
"""
|
|
97
|
+
if opts.ingest_url:
|
|
98
|
+
return opts.ingest_url
|
|
99
|
+
if opts.gcp_project_id and opts.aws_project_id:
|
|
100
|
+
raise ValueError(
|
|
101
|
+
"[cybat-sdk] Set only one of gcp_project_id or aws_project_id -- a single SDK "
|
|
102
|
+
"instance targets a single cloud. Use two instances for an app split across both."
|
|
103
|
+
)
|
|
104
|
+
if opts.aws_project_id:
|
|
105
|
+
return _DEFAULT_AWS_INGEST_URL
|
|
106
|
+
return _DEFAULT_GCP_INGEST_URL
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
class CybatCollector:
|
|
110
|
+
def __init__(self, opts: CybatOptions) -> None:
|
|
111
|
+
self.opts = opts
|
|
112
|
+
self._buffer: List[Dict[str, Any]] = []
|
|
113
|
+
self._lock = threading.Lock()
|
|
114
|
+
self._should_capture = resolve_status_filter(opts.status_filter)
|
|
115
|
+
self._should_capture_body = resolve_status_filter(opts.body_status_filter)
|
|
116
|
+
|
|
117
|
+
self._stop_event = threading.Event()
|
|
118
|
+
self._timer_thread = threading.Thread(target=self._run_timer, daemon=True)
|
|
119
|
+
self._timer_thread.start()
|
|
120
|
+
|
|
121
|
+
def _run_timer(self) -> None:
|
|
122
|
+
interval_s = self.opts.flush_interval_ms / 1000
|
|
123
|
+
while not self._stop_event.wait(interval_s):
|
|
124
|
+
self.flush()
|
|
125
|
+
|
|
126
|
+
def wants(self, status: int) -> bool:
|
|
127
|
+
"""Lets an adapter skip building an event that status_filter would discard anyway."""
|
|
128
|
+
return self._should_capture(status)
|
|
129
|
+
|
|
130
|
+
def capture_body_for(self, body: Any, content_type: Optional[str], status: int) -> Dict[str, Any]:
|
|
131
|
+
"""
|
|
132
|
+
Redacts and caps a request body for inclusion in an event. Returns {} (never
|
|
133
|
+
the raw body) when capture is off, the response doesn't match
|
|
134
|
+
body_status_filter, the body is empty / binary / multipart, or redaction
|
|
135
|
+
fails for any reason.
|
|
136
|
+
|
|
137
|
+
Returned keys are the WIRE names (camelCase) -- this dict is merged
|
|
138
|
+
directly into the outgoing event.
|
|
139
|
+
"""
|
|
140
|
+
if not self.opts.capture_body or not self._should_capture_body(status):
|
|
141
|
+
return {}
|
|
142
|
+
try:
|
|
143
|
+
if body is None or body == "" or body == {}:
|
|
144
|
+
return {}
|
|
145
|
+
if isinstance(body, (bytes, bytearray)):
|
|
146
|
+
return {}
|
|
147
|
+
if content_type and not _CAPTURABLE_CONTENT_TYPE.search(content_type):
|
|
148
|
+
return {}
|
|
149
|
+
|
|
150
|
+
masked = mask_body(body)
|
|
151
|
+
text = masked if isinstance(masked, str) else json.dumps(masked)
|
|
152
|
+
excerpt = snippet_around(text, self.opts.body_max_bytes)
|
|
153
|
+
result: Dict[str, Any] = {"body": excerpt}
|
|
154
|
+
if excerpt != text:
|
|
155
|
+
result["bodyTruncated"] = True
|
|
156
|
+
return result
|
|
157
|
+
except Exception:
|
|
158
|
+
return {} # when in doubt, send nothing rather than something unredacted
|
|
159
|
+
|
|
160
|
+
def record(self, event: Dict[str, Any]) -> None:
|
|
161
|
+
"""
|
|
162
|
+
Framework integrations call this unconditionally with every request's
|
|
163
|
+
captured event -- the status_filter check happens once, here, so every
|
|
164
|
+
integration gets identical filtering behavior for free.
|
|
165
|
+
"""
|
|
166
|
+
if not self._should_capture(event.get("status", 0)):
|
|
167
|
+
return
|
|
168
|
+
should_flush = False
|
|
169
|
+
with self._lock:
|
|
170
|
+
self._buffer.append(event)
|
|
171
|
+
if len(self._buffer) >= self.opts.max_batch_size:
|
|
172
|
+
should_flush = True
|
|
173
|
+
if should_flush:
|
|
174
|
+
self.flush()
|
|
175
|
+
|
|
176
|
+
def flush(self) -> None:
|
|
177
|
+
with self._lock:
|
|
178
|
+
if not self._buffer:
|
|
179
|
+
return
|
|
180
|
+
batch, self._buffer = self._buffer, []
|
|
181
|
+
self._send(batch, 0)
|
|
182
|
+
|
|
183
|
+
def _send(self, batch: List[Dict[str, Any]], attempt: int) -> None:
|
|
184
|
+
try:
|
|
185
|
+
data = json.dumps(batch).encode("utf-8")
|
|
186
|
+
req = urllib.request.Request(
|
|
187
|
+
self.opts.ingest_url,
|
|
188
|
+
data=data,
|
|
189
|
+
method="POST",
|
|
190
|
+
headers={"Content-Type": "application/json", "x-api-key": self.opts.api_key},
|
|
191
|
+
)
|
|
192
|
+
with urllib.request.urlopen(req, timeout=10):
|
|
193
|
+
pass # 2xx/3xx -- response body is never needed
|
|
194
|
+
except urllib.error.HTTPError as e:
|
|
195
|
+
if attempt < self.opts.max_retries:
|
|
196
|
+
return self._retry(batch, attempt)
|
|
197
|
+
print(f"[cybat-sdk] ingest rejected batch ({e.code}), dropping {len(batch)} event(s)")
|
|
198
|
+
except Exception as e: # network error, timeout, DNS failure, etc.
|
|
199
|
+
if attempt < self.opts.max_retries:
|
|
200
|
+
return self._retry(batch, attempt)
|
|
201
|
+
print(f"[cybat-sdk] failed to reach ingest endpoint, dropping {len(batch)} event(s): {e}")
|
|
202
|
+
|
|
203
|
+
def _retry(self, batch: List[Dict[str, Any]], attempt: int) -> None:
|
|
204
|
+
"""Exponential backoff: attempt 0 waits retry_backoff_ms, attempt 1 waits 2x that, etc."""
|
|
205
|
+
delay_s = (self.opts.retry_backoff_ms * (2**attempt)) / 1000
|
|
206
|
+
timer = threading.Timer(delay_s, self._send, args=(batch, attempt + 1))
|
|
207
|
+
timer.daemon = True
|
|
208
|
+
timer.start()
|
|
209
|
+
|
|
210
|
+
def stop(self) -> None:
|
|
211
|
+
"""Stops the flush timer. Call on graceful shutdown if you need a clean exit."""
|
|
212
|
+
self._stop_event.set()
|
|
File without changes
|