hypdmz 0.1.0a1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- hypdmz-0.1.0a1/PKG-INFO +134 -0
- hypdmz-0.1.0a1/README.md +109 -0
- hypdmz-0.1.0a1/hypdmz/__init__.py +3 -0
- hypdmz-0.1.0a1/hypdmz/__main__.py +5 -0
- hypdmz-0.1.0a1/hypdmz/cli.py +165 -0
- hypdmz-0.1.0a1/hypdmz/doctor.py +76 -0
- hypdmz-0.1.0a1/hypdmz/egress.py +263 -0
- hypdmz-0.1.0a1/hypdmz/env.example +37 -0
- hypdmz-0.1.0a1/hypdmz/env.py +254 -0
- hypdmz-0.1.0a1/hypdmz/examples/first-swarm/scores.csv +51 -0
- hypdmz-0.1.0a1/hypdmz/examples/first-swarm/seats.toml +11 -0
- hypdmz-0.1.0a1/hypdmz/examples/first-swarm/task.toml +14 -0
- hypdmz-0.1.0a1/hypdmz/images/controller/Dockerfile +11 -0
- hypdmz-0.1.0a1/hypdmz/images/controller/apply-lockdown.sh +96 -0
- hypdmz-0.1.0a1/hypdmz/images/controller/entrypoint.sh +75 -0
- hypdmz-0.1.0a1/hypdmz/images/controller/inspect-network.sh +27 -0
- hypdmz-0.1.0a1/hypdmz/images/proxy/Dockerfile +26 -0
- hypdmz-0.1.0a1/hypdmz/images/proxy/app/__init__.py +0 -0
- hypdmz-0.1.0a1/hypdmz/images/proxy/app/audit.py +206 -0
- hypdmz-0.1.0a1/hypdmz/images/proxy/app/dlp.py +66 -0
- hypdmz-0.1.0a1/hypdmz/images/proxy/app/firewall.py +218 -0
- hypdmz-0.1.0a1/hypdmz/images/proxy/app/firewall_policy.py +89 -0
- hypdmz-0.1.0a1/hypdmz/images/proxy/app/injection.py +99 -0
- hypdmz-0.1.0a1/hypdmz/images/proxy/app/limits.py +63 -0
- hypdmz-0.1.0a1/hypdmz/images/proxy/app/main.py +493 -0
- hypdmz-0.1.0a1/hypdmz/images/proxy/app/request_gates.py +111 -0
- hypdmz-0.1.0a1/hypdmz/images/proxy/app/tls_mitm.py +242 -0
- hypdmz-0.1.0a1/hypdmz/images/proxy/app/vault.py +442 -0
- hypdmz-0.1.0a1/hypdmz/images/proxy/config/firewall.json +40 -0
- hypdmz-0.1.0a1/hypdmz/images/proxy/entrypoint.sh +29 -0
- hypdmz-0.1.0a1/hypdmz/images/proxy/requirements.txt +4 -0
- hypdmz-0.1.0a1/hypdmz/images/swarm/Dockerfile +26 -0
- hypdmz-0.1.0a1/hypdmz/images/swarm/entrypoint.sh +78 -0
- hypdmz-0.1.0a1/hypdmz/images/swarm/probe_dns.py +14 -0
- hypdmz-0.1.0a1/hypdmz/platforms/__init__.py +16 -0
- hypdmz-0.1.0a1/hypdmz/platforms/base.py +108 -0
- hypdmz-0.1.0a1/hypdmz/platforms/linux.py +146 -0
- hypdmz-0.1.0a1/hypdmz/platforms/macos.py +101 -0
- hypdmz-0.1.0a1/hypdmz/platforms/windows.py +135 -0
- hypdmz-0.1.0a1/hypdmz/redact.py +78 -0
- hypdmz-0.1.0a1/hypdmz/runner.py +388 -0
- hypdmz-0.1.0a1/hypdmz/status.py +75 -0
- hypdmz-0.1.0a1/hypdmz.egg-info/PKG-INFO +134 -0
- hypdmz-0.1.0a1/hypdmz.egg-info/SOURCES.txt +49 -0
- hypdmz-0.1.0a1/hypdmz.egg-info/dependency_links.txt +1 -0
- hypdmz-0.1.0a1/hypdmz.egg-info/entry_points.txt +2 -0
- hypdmz-0.1.0a1/hypdmz.egg-info/requires.txt +4 -0
- hypdmz-0.1.0a1/hypdmz.egg-info/top_level.txt +1 -0
- hypdmz-0.1.0a1/pyproject.toml +46 -0
- hypdmz-0.1.0a1/setup.cfg +4 -0
- hypdmz-0.1.0a1/tests/test_launcher.py +140 -0
hypdmz-0.1.0a1/PKG-INFO
ADDED
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: hypdmz
|
|
3
|
+
Version: 0.1.0a1
|
|
4
|
+
Summary: Run a HypSwarm research swarm inside a governed-egress pod: a UID-scoped egress floor, an allowlisting TLS-intercepting proxy, vaulted credentials, hourly caps, a hash-chained audit — and an egress check from the swarm's own seat on every start.
|
|
5
|
+
Project-URL: Homepage, https://hypdmz.com
|
|
6
|
+
Project-URL: HypSwarm, https://hypswarm.com
|
|
7
|
+
Project-URL: HypGraph, https://hypgraph.com
|
|
8
|
+
Keywords: agents,sandbox,egress,podman,hypswarm,hypgraph,security
|
|
9
|
+
Classifier: Development Status :: 3 - Alpha
|
|
10
|
+
Classifier: Environment :: Console
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: Intended Audience :: Science/Research
|
|
13
|
+
Classifier: Operating System :: Microsoft :: Windows
|
|
14
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
15
|
+
Classifier: Operating System :: MacOS
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
18
|
+
Classifier: Topic :: Security
|
|
19
|
+
Classifier: Topic :: System :: Systems Administration
|
|
20
|
+
Requires-Python: >=3.9
|
|
21
|
+
Description-Content-Type: text/markdown
|
|
22
|
+
Provides-Extra: dev
|
|
23
|
+
Requires-Dist: pytest>=8; extra == "dev"
|
|
24
|
+
Requires-Dist: build>=1.2; extra == "dev"
|
|
25
|
+
|
|
26
|
+
# HypDMZ
|
|
27
|
+
|
|
28
|
+
Run a [HypSwarm](https://hypswarm.com) research swarm inside a governed-egress pod. The swarm's models, tools
|
|
29
|
+
and code run in a container that can reach exactly one thing: a proxy that allows the model API and nothing
|
|
30
|
+
else, holds the API key the swarm never sees, decrypts every request to check it, caps what leaves per hour,
|
|
31
|
+
and writes a tamper-evident log of all of it. The record the swarm builds is served live in the
|
|
32
|
+
[HypGraph](https://hypgraph.com) UI. Every start ends with the swarm trying to escape, from its own seat, and the
|
|
33
|
+
pod refusing to run if it can. [hypdmz.com](https://hypdmz.com) shows the measures and the proof.
|
|
34
|
+
|
|
35
|
+
```
|
|
36
|
+
hypdmz-pod (one Podman pod, one network namespace)
|
|
37
|
+
├── hypdmz-controller NET_ADMIN · applies the UID-scoped iptables/ip6tables egress floor · lockdown gate on :8009
|
|
38
|
+
├── hypdmz-proxy :8080 allowlisting TLS-intercepting forward proxy · :8002 control API · vault · audit chain · hourly caps
|
|
39
|
+
└── hypdmz-swarm :8000 HypGraph UI on the record · `hypswarm` runs here as UID 1000: loopback only, read-only root, capped
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
**The egress floor is keyed on socket owner UID, not port.** Only the proxy's UID (1001) may dial out. Every
|
|
43
|
+
other UID — the swarm, anything it spawns — gets loopback only, over IPv4 and IPv6, and cannot reach the
|
|
44
|
+
proxy's control port. The controller applies the rules before anything else starts, publishes `LOCKED` on
|
|
45
|
+
`127.0.0.1:8009`, and re-verifies the rule fingerprint every 30 s. The swarm container refuses to start until
|
|
46
|
+
the gate says `LOCKED`. `hypdmz up` ends by running `egress-check` from the swarm's own seat and stops the
|
|
47
|
+
swarm if any escape probe succeeds.
|
|
48
|
+
|
|
49
|
+
## Quick start
|
|
50
|
+
|
|
51
|
+
Needs Podman on the host (Podman Desktop on Windows and macOS, `podman` on Linux) and Python 3.9+. The launcher
|
|
52
|
+
has no dependencies; the three image build contexts, the default policy and an example ride inside the package.
|
|
53
|
+
|
|
54
|
+
```bash
|
|
55
|
+
pip install hypdmz
|
|
56
|
+
hypdmz init # the state dir and its .env; prints what comes next
|
|
57
|
+
podman secret create hypdmz_llm_api_key - # paste the model API key; it reaches the proxy only, as a file
|
|
58
|
+
hypdmz doctor
|
|
59
|
+
hypdmz build # three images, from the package's own contexts
|
|
60
|
+
hypdmz up # creates the CA, starts the pod, vaults the key, verifies the egress floor
|
|
61
|
+
hypdmz run first-swarm --python --budget-usd 0.20
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
Then open http://127.0.0.1:8000 — the record, live, with the swarm's lanes. `hypdmz status`, `hypdmz logs`,
|
|
65
|
+
`hypdmz egress-check`, `hypdmz down`.
|
|
66
|
+
|
|
67
|
+
`hypdmz run <folder>` copies a folder (a `task.toml`, optional `seats.toml`, data files) into the pod and runs
|
|
68
|
+
`hypswarm run` there as the swarm user; a bundled example's name works too (`hypdmz example` lists them,
|
|
69
|
+
`hypdmz example first-swarm ./mine` copies one out to edit). `--python` gives the seats a Python tool inside the
|
|
70
|
+
locked container. `--profile scientific|software|trading|knowledge|optimization` uses HypSwarm's rosters instead
|
|
71
|
+
of `seats.toml`. Every run is an investigation in the served record (`records/swarm.hypgraph` in the state dir);
|
|
72
|
+
`--record NAME` writes a separate file instead.
|
|
73
|
+
|
|
74
|
+
## What the swarm can and cannot do
|
|
75
|
+
|
|
76
|
+
| | |
|
|
77
|
+
|---|---|
|
|
78
|
+
| Reach the model API | yes — through the proxy, `MODEL_BASE_URL`'s host only, inference paths only (`hypdmz policy`) |
|
|
79
|
+
| See the API key | no — it sends a placeholder; the proxy's vault (AES-256-GCM, master key in `proxy-data/`) adds the real one, scoped to the model host. The key arrives as a podman secret mounted into the proxy alone (mode 0400, proxy UID), never in an environment |
|
|
80
|
+
| Reach any other host, by name, by IP, over IPv6, via DNS | no — iptables owner match, verified by `egress-check` |
|
|
81
|
+
| Reach the proxy's control API | no — `:8002` is open to root and the proxy UID only; from the host it needs a bearer token |
|
|
82
|
+
| Read the TLS interception CA's private key | no — the swarm mounts the certificate alone |
|
|
83
|
+
| Rewrite its own code | no — root-owned, `go-w`, checked with `access(2)` as the swarm user at start |
|
|
84
|
+
| Send a secret out in a request | no — DLP over bodies and auth headers blocks it (`DLP_MODE=block`) |
|
|
85
|
+
| Call the model without limit | no — the proxy caps forwarded calls and request megabytes per sliding hour (`limits` in the policy; 429 past the cap, audited) |
|
|
86
|
+
| Exhaust the machine | no — read-only root filesystem, size-capped `/tmp` and home, memory, CPU and process-count limits on the container; only `/records` and `/work` persist |
|
|
87
|
+
| Do anything unlogged | no — every request, allowed or denied, is a line in a SHA-256 hash chain (`proxy-data/audit/`); optional remote sink |
|
|
88
|
+
|
|
89
|
+
## The policy is a file
|
|
90
|
+
|
|
91
|
+
`hypdmz up` seeds `<state dir>/policy/firewall.json` from the package's default on the first run; after that it is
|
|
92
|
+
yours. The proxy hot-reloads it (size + mtime, checked once a second). A missing or malformed file denies
|
|
93
|
+
everything. Rules carry host, ports, methods, path prefixes, an optional `guard: "public_web"` request-shape gate,
|
|
94
|
+
`expires_at`, and the vault credential names to inject; `limits` holds the hourly caps. `hypdmz policy` shows it,
|
|
95
|
+
`hypdmz policy path` says where it is, `hypdmz policy reset` restores the default (keeping a `.bak`).
|
|
96
|
+
|
|
97
|
+
## Layout
|
|
98
|
+
|
|
99
|
+
```
|
|
100
|
+
hypdmz/ the launcher (stdlib only): init · doctor · build · up · down · status · logs · debug · egress-check · run · policy · example
|
|
101
|
+
hypdmz/images/ the three build contexts, shipped in the wheel: controller (egress floor), proxy, swarm
|
|
102
|
+
hypdmz/images/proxy/config/firewall.json the default policy
|
|
103
|
+
hypdmz/examples/ first-swarm — a two-seat t-test with the python instrument
|
|
104
|
+
hypdmz/env.example the .env template `hypdmz init` writes
|
|
105
|
+
tests/ the launcher without podman (configuration, payload, policy seed, the command lines it would run)
|
|
106
|
+
docs/DESIGN.md what was taken from PegasusDMZ, what was left out, the blast radius, the limits
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
State (secrets, policy, records) lives outside the package and outside any repository: `%USERPROFILE%\.hypdmz` on
|
|
110
|
+
Windows, `~/.local/state/hypdmz` on Linux, `~/Library/Application Support/HypDMZ` on macOS (`HYPDMZ_STATE_DIR`
|
|
111
|
+
overrides). `.env` keys: `MODEL_BASE_URL`, `MODEL`, `HYPDMZ_UI_PORT` / `HYPDMZ_CONSOLE_PORT` / `HYPDMZ_PROXY_PORT`,
|
|
112
|
+
`HYPDMZ_SWARM_MEMORY` / `_CPUS` / `_PIDS` / `_TMP`, `HYPDMZ_KEY_SECRET`, `HYPDMZ_RECORDS_DIR`, `DLP_MODE`,
|
|
113
|
+
`AUDIT_SINK_URL` / `AUDIT_SINK_TOKEN`.
|
|
114
|
+
|
|
115
|
+
## Control API (`:8002`, loopback only, bearer token)
|
|
116
|
+
|
|
117
|
+
`GET /healthz` · `GET /api/status` · `GET /api/policy` · `GET /api/audit?n=` · `GET /api/audit/verify` ·
|
|
118
|
+
`GET /api/vault` · `POST /api/vault` `{name, value, type, injection_spec?, scope_host?}` · `DELETE /api/vault/{name}`.
|
|
119
|
+
Everything but `/healthz` needs `Authorization: Bearer <token>`; the token is generated once into
|
|
120
|
+
`<state dir>/tokens/console.token` (0600) and mounted read-only into the proxy. `hypdmz status` prints the
|
|
121
|
+
`curl` line. A `Host` header that is not loopback, or a cross-origin state change, is refused as well.
|
|
122
|
+
|
|
123
|
+
## Using the pod's proxy from the host
|
|
124
|
+
|
|
125
|
+
The proxy is published on `127.0.0.1:8080` (`HYPDMZ_PROXY_PORT`). Host tools get the same governance:
|
|
126
|
+
|
|
127
|
+
```
|
|
128
|
+
HTTPS_PROXY=http://127.0.0.1:8080 SSL_CERT_FILE=<state dir>/certs/public/hypdmz-ca-cert.pem OPENAI_API_KEY=__hypdmz_inject__
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
## Status
|
|
132
|
+
|
|
133
|
+
Alpha, private beta. Verified on Windows 11 (Podman Desktop, WSL2); the Linux and macOS paths are written and
|
|
134
|
+
untested. Known limits are on [hypdmz.com](https://hypdmz.com#limits) and in `docs/DESIGN.md`.
|
hypdmz-0.1.0a1/README.md
ADDED
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
# HypDMZ
|
|
2
|
+
|
|
3
|
+
Run a [HypSwarm](https://hypswarm.com) research swarm inside a governed-egress pod. The swarm's models, tools
|
|
4
|
+
and code run in a container that can reach exactly one thing: a proxy that allows the model API and nothing
|
|
5
|
+
else, holds the API key the swarm never sees, decrypts every request to check it, caps what leaves per hour,
|
|
6
|
+
and writes a tamper-evident log of all of it. The record the swarm builds is served live in the
|
|
7
|
+
[HypGraph](https://hypgraph.com) UI. Every start ends with the swarm trying to escape, from its own seat, and the
|
|
8
|
+
pod refusing to run if it can. [hypdmz.com](https://hypdmz.com) shows the measures and the proof.
|
|
9
|
+
|
|
10
|
+
```
|
|
11
|
+
hypdmz-pod (one Podman pod, one network namespace)
|
|
12
|
+
├── hypdmz-controller NET_ADMIN · applies the UID-scoped iptables/ip6tables egress floor · lockdown gate on :8009
|
|
13
|
+
├── hypdmz-proxy :8080 allowlisting TLS-intercepting forward proxy · :8002 control API · vault · audit chain · hourly caps
|
|
14
|
+
└── hypdmz-swarm :8000 HypGraph UI on the record · `hypswarm` runs here as UID 1000: loopback only, read-only root, capped
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
**The egress floor is keyed on socket owner UID, not port.** Only the proxy's UID (1001) may dial out. Every
|
|
18
|
+
other UID — the swarm, anything it spawns — gets loopback only, over IPv4 and IPv6, and cannot reach the
|
|
19
|
+
proxy's control port. The controller applies the rules before anything else starts, publishes `LOCKED` on
|
|
20
|
+
`127.0.0.1:8009`, and re-verifies the rule fingerprint every 30 s. The swarm container refuses to start until
|
|
21
|
+
the gate says `LOCKED`. `hypdmz up` ends by running `egress-check` from the swarm's own seat and stops the
|
|
22
|
+
swarm if any escape probe succeeds.
|
|
23
|
+
|
|
24
|
+
## Quick start
|
|
25
|
+
|
|
26
|
+
Needs Podman on the host (Podman Desktop on Windows and macOS, `podman` on Linux) and Python 3.9+. The launcher
|
|
27
|
+
has no dependencies; the three image build contexts, the default policy and an example ride inside the package.
|
|
28
|
+
|
|
29
|
+
```bash
|
|
30
|
+
pip install hypdmz
|
|
31
|
+
hypdmz init # the state dir and its .env; prints what comes next
|
|
32
|
+
podman secret create hypdmz_llm_api_key - # paste the model API key; it reaches the proxy only, as a file
|
|
33
|
+
hypdmz doctor
|
|
34
|
+
hypdmz build # three images, from the package's own contexts
|
|
35
|
+
hypdmz up # creates the CA, starts the pod, vaults the key, verifies the egress floor
|
|
36
|
+
hypdmz run first-swarm --python --budget-usd 0.20
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
Then open http://127.0.0.1:8000 — the record, live, with the swarm's lanes. `hypdmz status`, `hypdmz logs`,
|
|
40
|
+
`hypdmz egress-check`, `hypdmz down`.
|
|
41
|
+
|
|
42
|
+
`hypdmz run <folder>` copies a folder (a `task.toml`, optional `seats.toml`, data files) into the pod and runs
|
|
43
|
+
`hypswarm run` there as the swarm user; a bundled example's name works too (`hypdmz example` lists them,
|
|
44
|
+
`hypdmz example first-swarm ./mine` copies one out to edit). `--python` gives the seats a Python tool inside the
|
|
45
|
+
locked container. `--profile scientific|software|trading|knowledge|optimization` uses HypSwarm's rosters instead
|
|
46
|
+
of `seats.toml`. Every run is an investigation in the served record (`records/swarm.hypgraph` in the state dir);
|
|
47
|
+
`--record NAME` writes a separate file instead.
|
|
48
|
+
|
|
49
|
+
## What the swarm can and cannot do
|
|
50
|
+
|
|
51
|
+
| | |
|
|
52
|
+
|---|---|
|
|
53
|
+
| Reach the model API | yes — through the proxy, `MODEL_BASE_URL`'s host only, inference paths only (`hypdmz policy`) |
|
|
54
|
+
| See the API key | no — it sends a placeholder; the proxy's vault (AES-256-GCM, master key in `proxy-data/`) adds the real one, scoped to the model host. The key arrives as a podman secret mounted into the proxy alone (mode 0400, proxy UID), never in an environment |
|
|
55
|
+
| Reach any other host, by name, by IP, over IPv6, via DNS | no — iptables owner match, verified by `egress-check` |
|
|
56
|
+
| Reach the proxy's control API | no — `:8002` is open to root and the proxy UID only; from the host it needs a bearer token |
|
|
57
|
+
| Read the TLS interception CA's private key | no — the swarm mounts the certificate alone |
|
|
58
|
+
| Rewrite its own code | no — root-owned, `go-w`, checked with `access(2)` as the swarm user at start |
|
|
59
|
+
| Send a secret out in a request | no — DLP over bodies and auth headers blocks it (`DLP_MODE=block`) |
|
|
60
|
+
| Call the model without limit | no — the proxy caps forwarded calls and request megabytes per sliding hour (`limits` in the policy; 429 past the cap, audited) |
|
|
61
|
+
| Exhaust the machine | no — read-only root filesystem, size-capped `/tmp` and home, memory, CPU and process-count limits on the container; only `/records` and `/work` persist |
|
|
62
|
+
| Do anything unlogged | no — every request, allowed or denied, is a line in a SHA-256 hash chain (`proxy-data/audit/`); optional remote sink |
|
|
63
|
+
|
|
64
|
+
## The policy is a file
|
|
65
|
+
|
|
66
|
+
`hypdmz up` seeds `<state dir>/policy/firewall.json` from the package's default on the first run; after that it is
|
|
67
|
+
yours. The proxy hot-reloads it (size + mtime, checked once a second). A missing or malformed file denies
|
|
68
|
+
everything. Rules carry host, ports, methods, path prefixes, an optional `guard: "public_web"` request-shape gate,
|
|
69
|
+
`expires_at`, and the vault credential names to inject; `limits` holds the hourly caps. `hypdmz policy` shows it,
|
|
70
|
+
`hypdmz policy path` says where it is, `hypdmz policy reset` restores the default (keeping a `.bak`).
|
|
71
|
+
|
|
72
|
+
## Layout
|
|
73
|
+
|
|
74
|
+
```
|
|
75
|
+
hypdmz/ the launcher (stdlib only): init · doctor · build · up · down · status · logs · debug · egress-check · run · policy · example
|
|
76
|
+
hypdmz/images/ the three build contexts, shipped in the wheel: controller (egress floor), proxy, swarm
|
|
77
|
+
hypdmz/images/proxy/config/firewall.json the default policy
|
|
78
|
+
hypdmz/examples/ first-swarm — a two-seat t-test with the python instrument
|
|
79
|
+
hypdmz/env.example the .env template `hypdmz init` writes
|
|
80
|
+
tests/ the launcher without podman (configuration, payload, policy seed, the command lines it would run)
|
|
81
|
+
docs/DESIGN.md what was taken from PegasusDMZ, what was left out, the blast radius, the limits
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
State (secrets, policy, records) lives outside the package and outside any repository: `%USERPROFILE%\.hypdmz` on
|
|
85
|
+
Windows, `~/.local/state/hypdmz` on Linux, `~/Library/Application Support/HypDMZ` on macOS (`HYPDMZ_STATE_DIR`
|
|
86
|
+
overrides). `.env` keys: `MODEL_BASE_URL`, `MODEL`, `HYPDMZ_UI_PORT` / `HYPDMZ_CONSOLE_PORT` / `HYPDMZ_PROXY_PORT`,
|
|
87
|
+
`HYPDMZ_SWARM_MEMORY` / `_CPUS` / `_PIDS` / `_TMP`, `HYPDMZ_KEY_SECRET`, `HYPDMZ_RECORDS_DIR`, `DLP_MODE`,
|
|
88
|
+
`AUDIT_SINK_URL` / `AUDIT_SINK_TOKEN`.
|
|
89
|
+
|
|
90
|
+
## Control API (`:8002`, loopback only, bearer token)
|
|
91
|
+
|
|
92
|
+
`GET /healthz` · `GET /api/status` · `GET /api/policy` · `GET /api/audit?n=` · `GET /api/audit/verify` ·
|
|
93
|
+
`GET /api/vault` · `POST /api/vault` `{name, value, type, injection_spec?, scope_host?}` · `DELETE /api/vault/{name}`.
|
|
94
|
+
Everything but `/healthz` needs `Authorization: Bearer <token>`; the token is generated once into
|
|
95
|
+
`<state dir>/tokens/console.token` (0600) and mounted read-only into the proxy. `hypdmz status` prints the
|
|
96
|
+
`curl` line. A `Host` header that is not loopback, or a cross-origin state change, is refused as well.
|
|
97
|
+
|
|
98
|
+
## Using the pod's proxy from the host
|
|
99
|
+
|
|
100
|
+
The proxy is published on `127.0.0.1:8080` (`HYPDMZ_PROXY_PORT`). Host tools get the same governance:
|
|
101
|
+
|
|
102
|
+
```
|
|
103
|
+
HTTPS_PROXY=http://127.0.0.1:8080 SSL_CERT_FILE=<state dir>/certs/public/hypdmz-ca-cert.pem OPENAI_API_KEY=__hypdmz_inject__
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
## Status
|
|
107
|
+
|
|
108
|
+
Alpha, private beta. Verified on Windows 11 (Podman Desktop, WSL2); the Linux and macOS paths are written and
|
|
109
|
+
untested. Known limits are on [hypdmz.com](https://hypdmz.com#limits) and in `docs/DESIGN.md`.
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import json
|
|
5
|
+
import shutil
|
|
6
|
+
import sys
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
from hypdmz import __version__
|
|
10
|
+
from hypdmz.doctor import cmd_doctor
|
|
11
|
+
from hypdmz.egress import cmd_egress_check
|
|
12
|
+
from hypdmz.env import Config, load
|
|
13
|
+
from hypdmz.platforms import detect
|
|
14
|
+
from hypdmz.runner import bundled_examples, cmd_build, cmd_debug, cmd_down, cmd_logs, cmd_run, cmd_up, ensure_policy
|
|
15
|
+
from hypdmz.status import cmd_status
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def cmd_init(cfg: Config) -> int:
|
|
19
|
+
"""Create the state directory and its .env from the package's template; say what comes next."""
|
|
20
|
+
cfg.state_dir.mkdir(parents=True, exist_ok=True)
|
|
21
|
+
if cfg.env_file.exists():
|
|
22
|
+
print(f"[hypdmz] .env already there: {cfg.env_file}")
|
|
23
|
+
else:
|
|
24
|
+
shutil.copyfile(cfg.env_example, cfg.env_file)
|
|
25
|
+
print(f"[hypdmz] wrote {cfg.env_file}")
|
|
26
|
+
created = ensure_policy(cfg)
|
|
27
|
+
if not created:
|
|
28
|
+
print(f"[hypdmz] policy already there: {cfg.policy_path}")
|
|
29
|
+
print(f"""
|
|
30
|
+
[hypdmz] next:
|
|
31
|
+
1. edit {cfg.env_file}: MODEL_BASE_URL and MODEL (DeepInfra is the default)
|
|
32
|
+
2. put the model API key in a podman secret: podman secret create {cfg.key_secret} -
|
|
33
|
+
(paste the key, Enter, then Ctrl-D on Linux/macOS or Ctrl-Z Enter on Windows)
|
|
34
|
+
3. hypdmz doctor · hypdmz build · hypdmz up · hypdmz run first-swarm --python --budget-usd 0.20
|
|
35
|
+
the allowlist is {cfg.policy_path} (hot-reloaded; `hypdmz policy reset` restores the default)""")
|
|
36
|
+
return 0
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def cmd_policy(cfg: Config, action: str) -> int:
|
|
40
|
+
if action == "path":
|
|
41
|
+
print(cfg.policy_path)
|
|
42
|
+
return 0
|
|
43
|
+
if action == "reset":
|
|
44
|
+
cfg.policy_dir.mkdir(parents=True, exist_ok=True)
|
|
45
|
+
if cfg.policy_path.exists():
|
|
46
|
+
backup = cfg.policy_path.with_suffix(".json.bak")
|
|
47
|
+
shutil.copyfile(cfg.policy_path, backup)
|
|
48
|
+
print(f"[hypdmz] previous policy kept at {backup}")
|
|
49
|
+
shutil.copyfile(cfg.default_policy, cfg.policy_path)
|
|
50
|
+
print(f"[hypdmz] policy reset to the package default → {cfg.policy_path} (the proxy picks it up within a second)")
|
|
51
|
+
return 0
|
|
52
|
+
# show
|
|
53
|
+
src = cfg.policy_path if cfg.policy_path.exists() else cfg.default_policy
|
|
54
|
+
try:
|
|
55
|
+
pol = json.loads(src.read_text(encoding="utf-8"))
|
|
56
|
+
except Exception as e:
|
|
57
|
+
print(f"[hypdmz] {src} does not parse ({e}) — the proxy denies everything until it does")
|
|
58
|
+
return 1
|
|
59
|
+
print(f"[hypdmz] {src}{'' if src == cfg.policy_path else ' (package default; no operator policy yet)'}")
|
|
60
|
+
lim = pol.get("limits") or {}
|
|
61
|
+
print(f" limits: {lim.get('calls_per_hour') or '∞'} calls/h · {lim.get('megabytes_per_hour') or '∞'} MB/h")
|
|
62
|
+
for c in pol.get("cidr_blocklist") or []:
|
|
63
|
+
print(f" block {c.get('name')}: {', '.join(c.get('cidrs') or [])}")
|
|
64
|
+
for r in pol.get("rules") or []:
|
|
65
|
+
hosts = r.get("host") or ", ".join(r.get("hosts") or r.get("host_suffixes") or []) or r.get("host_regex") or "*"
|
|
66
|
+
flag = "" if r.get("enabled", True) else " (disabled)"
|
|
67
|
+
guard = f" guard={r['guard']}" if r.get("guard") else ""
|
|
68
|
+
exp = f" until {r['expires_at']}" if r.get("expires_at") else ""
|
|
69
|
+
print(f" rule {r.get('name')}: {r.get('scheme', 'https')}://{hosts} {r.get('ports') or ''} "
|
|
70
|
+
f"{'/'.join(r.get('methods') or ['*'])} {' '.join(r.get('path_prefixes') or ['/'])}"
|
|
71
|
+
f"{' injects ' + ','.join(r['injects']) if r.get('injects') else ''}{guard}{exp}{flag}")
|
|
72
|
+
return 0
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def cmd_examples(cfg: Config, name: str, dest: Path | None) -> int:
|
|
76
|
+
names = bundled_examples(cfg)
|
|
77
|
+
if not name:
|
|
78
|
+
for n in names:
|
|
79
|
+
print(n)
|
|
80
|
+
return 0
|
|
81
|
+
if name not in names:
|
|
82
|
+
print(f"[hypdmz] no bundled example '{name}' (have: {', '.join(names)})")
|
|
83
|
+
return 2
|
|
84
|
+
target = (dest or Path(name)).resolve()
|
|
85
|
+
if target.exists():
|
|
86
|
+
print(f"[hypdmz] {target} exists")
|
|
87
|
+
return 2
|
|
88
|
+
shutil.copytree(cfg.examples_dir / name, target)
|
|
89
|
+
print(f"[hypdmz] copied {name} → {target} (edit task.toml and seats.toml, then: hypdmz run {target})")
|
|
90
|
+
return 0
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def main(argv: list[str] | None = None) -> int:
|
|
94
|
+
if hasattr(sys.stdout, "reconfigure"):
|
|
95
|
+
sys.stdout.reconfigure(encoding="utf-8", errors="replace") # Windows consoles default to cp1252
|
|
96
|
+
p = argparse.ArgumentParser(prog="hypdmz", description="Run a HypSwarm research swarm inside a governed-egress pod.")
|
|
97
|
+
p.add_argument("--version", action="version", version=f"hypdmz {__version__}")
|
|
98
|
+
sub = p.add_subparsers(dest="cmd", required=True)
|
|
99
|
+
|
|
100
|
+
sub.add_parser("init", help="create the state dir and .env; print what comes next")
|
|
101
|
+
s = sub.add_parser("doctor", help="preflight checks"); s.add_argument("--json", action="store_true")
|
|
102
|
+
s = sub.add_parser("build", help="build the three images from the package's contexts"); s.add_argument("--dry-run", action="store_true")
|
|
103
|
+
s = sub.add_parser("up", help="create the pod, start the containers, verify the egress floor")
|
|
104
|
+
s.add_argument("--dry-run", action="store_true"); s.add_argument("--skip-egress-check", action="store_true")
|
|
105
|
+
sub.add_parser("down", help="stop and remove the pod")
|
|
106
|
+
s = sub.add_parser("status", help="pod, lockdown and endpoint snapshot"); s.add_argument("--json", action="store_true")
|
|
107
|
+
s = sub.add_parser("logs", help="tail the containers"); s.add_argument("--no-follow", action="store_true")
|
|
108
|
+
sub.add_parser("debug", help="pod, lockdown state and iptables view")
|
|
109
|
+
s = sub.add_parser("egress-check", help="escape attempts as the swarm user; exit 1 on any bypass")
|
|
110
|
+
s.add_argument("--json", action="store_true")
|
|
111
|
+
s = sub.add_parser("run", help="run a swarm: a folder with task.toml (and seats.toml or --profile), or a bundled example's name")
|
|
112
|
+
s.add_argument("folder", type=Path)
|
|
113
|
+
s.add_argument("--task", default="task.toml"); s.add_argument("--seats", default="")
|
|
114
|
+
s.add_argument("--profile", default="", help="scientific | software | trading | knowledge | optimization")
|
|
115
|
+
s.add_argument("--model", default="", help="default model id (else MODEL from .env)")
|
|
116
|
+
s.add_argument("--budget-usd", type=float, default=0.0); s.add_argument("--budget-calls", type=int, default=150)
|
|
117
|
+
s.add_argument("--pricing", default="", help="input,cached,output $ per million tokens")
|
|
118
|
+
s.add_argument("--python", action="store_true", help="give the seats a python tool in the work folder (in the pod)")
|
|
119
|
+
s.add_argument("--record", default="", help="record file name under records/ (default: the served swarm.hypgraph)")
|
|
120
|
+
s.add_argument("--name", default="", help="name for this run's work folder")
|
|
121
|
+
s = sub.add_parser("policy", help="the egress allowlist: show, path, reset")
|
|
122
|
+
s.add_argument("action", nargs="?", default="show", choices=["show", "path", "reset"])
|
|
123
|
+
s = sub.add_parser("example", help="list the bundled examples, or copy one out to edit")
|
|
124
|
+
s.add_argument("name", nargs="?", default=""); s.add_argument("dest", nargs="?", type=Path, default=None)
|
|
125
|
+
|
|
126
|
+
a = p.parse_args(argv)
|
|
127
|
+
cfg = load()
|
|
128
|
+
try:
|
|
129
|
+
if a.cmd == "init":
|
|
130
|
+
return cmd_init(cfg)
|
|
131
|
+
if a.cmd == "policy":
|
|
132
|
+
return cmd_policy(cfg, a.action)
|
|
133
|
+
if a.cmd == "example":
|
|
134
|
+
return cmd_examples(cfg, a.name, a.dest)
|
|
135
|
+
platform = detect()
|
|
136
|
+
if a.cmd == "doctor":
|
|
137
|
+
return cmd_doctor(cfg, platform, json_output=a.json)
|
|
138
|
+
if a.cmd == "build":
|
|
139
|
+
return cmd_build(cfg, platform, dry_run=a.dry_run)
|
|
140
|
+
if a.cmd == "up":
|
|
141
|
+
return cmd_up(cfg, platform, dry_run=a.dry_run, skip_egress_check=a.skip_egress_check)
|
|
142
|
+
if a.cmd == "down":
|
|
143
|
+
return cmd_down(cfg, platform)
|
|
144
|
+
if a.cmd == "status":
|
|
145
|
+
return cmd_status(cfg, platform, json_output=a.json)
|
|
146
|
+
if a.cmd == "logs":
|
|
147
|
+
return cmd_logs(cfg, platform, follow=not a.no_follow)
|
|
148
|
+
if a.cmd == "debug":
|
|
149
|
+
return cmd_debug(cfg, platform)
|
|
150
|
+
if a.cmd == "egress-check":
|
|
151
|
+
return cmd_egress_check(cfg, platform, json_output=a.json)
|
|
152
|
+
if a.cmd == "run":
|
|
153
|
+
return cmd_run(cfg, platform, a.folder, task=a.task, seats=a.seats, profile=a.profile, model=a.model,
|
|
154
|
+
budget_usd=a.budget_usd, budget_calls=a.budget_calls, pricing=a.pricing,
|
|
155
|
+
python=a.python, record=a.record, name=a.name)
|
|
156
|
+
except KeyboardInterrupt:
|
|
157
|
+
return 130
|
|
158
|
+
except RuntimeError as e:
|
|
159
|
+
print(f"[hypdmz] {e}", file=sys.stderr)
|
|
160
|
+
return 1
|
|
161
|
+
return 2
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
if __name__ == "__main__":
|
|
165
|
+
sys.exit(main())
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
"""`hypdmz doctor` — preflight checks before the first `up`."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import json
|
|
5
|
+
from dataclasses import asdict
|
|
6
|
+
|
|
7
|
+
from hypdmz.env import Config, sync_folder_marker
|
|
8
|
+
from hypdmz.platforms.base import Platform, PreflightResult
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def cmd_doctor(cfg: Config, platform: Platform, *, json_output: bool = False) -> int:
|
|
12
|
+
app_results = [_check_env_file(cfg), _check_state_dir(cfg), _check_model_key(cfg), _check_package(cfg), _check_policy(cfg)]
|
|
13
|
+
plat_results = platform.preflight()
|
|
14
|
+
fail = sum(1 for r in app_results + plat_results if not r.ok)
|
|
15
|
+
if json_output:
|
|
16
|
+
print(json.dumps({"platform": platform.name, "ok": fail == 0, "fail_count": fail,
|
|
17
|
+
"application": [asdict(r) for r in app_results],
|
|
18
|
+
"platform_checks": [asdict(r) for r in plat_results]}, indent=2))
|
|
19
|
+
return 0 if fail == 0 else 1
|
|
20
|
+
print(f"[hypdmz] doctor — platform={platform.name}\n")
|
|
21
|
+
for r in app_results:
|
|
22
|
+
print(r.render())
|
|
23
|
+
print("\n── platform checks ──")
|
|
24
|
+
for r in plat_results:
|
|
25
|
+
print(r.render())
|
|
26
|
+
print()
|
|
27
|
+
print("[hypdmz] all checks passed." if fail == 0 else f"[hypdmz] {fail} check(s) failed.")
|
|
28
|
+
return 0 if fail == 0 else 1
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _check_env_file(cfg: Config) -> PreflightResult:
|
|
32
|
+
if cfg.env_file.exists():
|
|
33
|
+
return PreflightResult(".env present", True, str(cfg.env_file))
|
|
34
|
+
return PreflightResult(".env present", False, f"run `hypdmz init` (writes {cfg.env_file} from the package template)")
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _check_state_dir(cfg: Config) -> PreflightResult:
|
|
38
|
+
marker = sync_folder_marker(cfg.state_dir)
|
|
39
|
+
if marker:
|
|
40
|
+
return PreflightResult("state dir is local", False,
|
|
41
|
+
f"{cfg.state_dir} is under a cloud-synced folder ({marker}); set HYPDMZ_STATE_DIR")
|
|
42
|
+
return PreflightResult("state dir is local", True, str(cfg.state_dir))
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _check_model_key(cfg: Config) -> PreflightResult:
|
|
46
|
+
from hypdmz.runner import secret_exists
|
|
47
|
+
|
|
48
|
+
vault = cfg.proxy_data_dir / "vault.db"
|
|
49
|
+
if secret_exists(cfg.key_secret):
|
|
50
|
+
return PreflightResult("model API key", True, f"podman secret '{cfg.key_secret}' (mounted into the proxy only)")
|
|
51
|
+
if cfg.model_api_key:
|
|
52
|
+
return PreflightResult("model API key", True, "MODEL_API_KEY in .env; vaulted on the next `up` — then delete the "
|
|
53
|
+
f"line (better: podman secret create {cfg.key_secret} -)")
|
|
54
|
+
if vault.exists():
|
|
55
|
+
return PreflightResult("model API key", True, "no secret, not in .env; the vault exists (hypdmz status shows the count)")
|
|
56
|
+
return PreflightResult("model API key", False,
|
|
57
|
+
f"no key: `podman secret create {cfg.key_secret} -` (paste it), or MODEL_API_KEY in .env")
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _check_package(cfg: Config) -> PreflightResult:
|
|
61
|
+
missing = [p for p in ("controller/Dockerfile", "proxy/Dockerfile", "swarm/Dockerfile", "proxy/config/firewall.json")
|
|
62
|
+
if not (cfg.images_dir / p).exists()]
|
|
63
|
+
if missing:
|
|
64
|
+
return PreflightResult("bundled images", False, f"missing under {cfg.images_dir}: {', '.join(missing)} (broken install?)")
|
|
65
|
+
return PreflightResult("bundled images", True, f"{cfg.images_dir} (controller, proxy, swarm)")
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _check_policy(cfg: Config) -> PreflightResult:
|
|
69
|
+
src = cfg.policy_path if cfg.policy_path.exists() else cfg.default_policy
|
|
70
|
+
try:
|
|
71
|
+
pol = json.loads(src.read_text(encoding="utf-8"))
|
|
72
|
+
n = len(pol.get("rules") or [])
|
|
73
|
+
except Exception as e:
|
|
74
|
+
return PreflightResult("egress policy", False, f"{src} does not parse ({e}); the proxy denies everything until it does")
|
|
75
|
+
where = "operator policy" if src == cfg.policy_path else "package default (seeded on the first `up`)"
|
|
76
|
+
return PreflightResult("egress policy", True, f"{src}: {n} rule(s), {where}")
|