vault-env-loader 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- vault_env_loader-0.1.0.dist-info/METADATA +208 -0
- vault_env_loader-0.1.0.dist-info/RECORD +7 -0
- vault_env_loader-0.1.0.dist-info/WHEEL +5 -0
- vault_env_loader-0.1.0.dist-info/entry_points.txt +2 -0
- vault_env_loader-0.1.0.dist-info/licenses/LICENSE +21 -0
- vault_env_loader-0.1.0.dist-info/top_level.txt +1 -0
- vault_env_loader.py +844 -0
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: vault-env-loader
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Load HashiCorp Vault KV secrets into environment variables before your app starts. Zero dependencies.
|
|
5
|
+
Author: Kaylov H
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/kaylovh/vault-env-loader/tree/main/python#readme
|
|
8
|
+
Project-URL: Source, https://github.com/kaylovh/vault-env-loader
|
|
9
|
+
Project-URL: Issues, https://github.com/kaylovh/vault-env-loader/issues
|
|
10
|
+
Keywords: vault,hashicorp,secrets,env,environment,dotenv,kv,config
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Environment :: Console
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Intended Audience :: System Administrators
|
|
15
|
+
Classifier: Operating System :: OS Independent
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
18
|
+
Classifier: Topic :: Security
|
|
19
|
+
Classifier: Topic :: System :: Systems Administration
|
|
20
|
+
Requires-Python: >=3.6
|
|
21
|
+
Description-Content-Type: text/markdown
|
|
22
|
+
License-File: LICENSE
|
|
23
|
+
Dynamic: license-file
|
|
24
|
+
|
|
25
|
+
# vault-env-loader (Python)
|
|
26
|
+
|
|
27
|
+
Load secrets from HashiCorp Vault's KV engine into environment variables **before your application starts**. Your code keeps reading `os.environ` / `os.getenv` — Django settings, Flask config, pydantic `BaseSettings`, `django-environ`, `python-decouple` all keep working — and the `.env` file full of plaintext passwords can go.
|
|
28
|
+
|
|
29
|
+
- **Zero dependencies.** One file, standard library only, Python ≥ 3.6. Old projects with pinned `requests` versions are not affected.
|
|
30
|
+
- **Fails fast.** If a secret cannot be loaded, your application does not start (exit code 78), instead of failing later in business code.
|
|
31
|
+
- **Imports like Spring Cloud Vault.** `vault://secret/my-app/prod/database`.
|
|
32
|
+
- **The token stays out of your app.** `VAULT_TOKEN` is removed from the environment your application sees.
|
|
33
|
+
- Works with HashiCorp Vault (KV v1 and v2, auto-detected) and with Vault-compatible servers.
|
|
34
|
+
- Same configuration and behaviour as the [Node.js package](https://github.com/kaylovh/vault-env-loader/tree/main/node).
|
|
35
|
+
|
|
36
|
+
## Quick start
|
|
37
|
+
|
|
38
|
+
```bash
|
|
39
|
+
pip install vault-env-loader
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
Configure it where you deploy (instead of putting secrets in `.env`):
|
|
43
|
+
|
|
44
|
+
```bash
|
|
45
|
+
VAULT_ADDR=https://vault.example.com
|
|
46
|
+
VAULT_TOKEN=hvs.xxxxx # or VAULT_TOKEN_FILE=/run/secrets/vault-token
|
|
47
|
+
VAULT_ENV_IMPORT="vault://secret/order-service/prod/database, vault://secret/order-service/prod/redis?prefix=REDIS_"
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
Then pick **one** way to hook it in.
|
|
51
|
+
|
|
52
|
+
### A. Wrap the start command (recommended — no code change)
|
|
53
|
+
|
|
54
|
+
```bash
|
|
55
|
+
vault-env-loader -- gunicorn app.wsgi
|
|
56
|
+
vault-env-loader -- uvicorn app.main:app --host 0.0.0.0
|
|
57
|
+
vault-env-loader -- celery -A proj worker
|
|
58
|
+
vault-env-loader -- python manage.py migrate
|
|
59
|
+
python -m vault_env_loader -- python app.py # when the scripts directory is not on PATH
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
```dockerfile
|
|
63
|
+
ENTRYPOINT ["vault-env-loader", "--"]
|
|
64
|
+
CMD ["gunicorn", "app.wsgi", "--bind", "0.0.0.0:8000"]
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
Works for anything: systemd, supervisor, cron, shell scripts — the wrapped program does not even have to be Python. On Linux and macOS the wrapper **replaces itself** with your command (`exec`): same PID, signals and exit code need no forwarding, and PID 1 in a container is your application.
|
|
68
|
+
|
|
69
|
+
### B. One line at the top of your entry file
|
|
70
|
+
|
|
71
|
+
```python
|
|
72
|
+
import vault_env_loader; vault_env_loader.load() # must run before anything reads the configuration
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
Where "the top" is:
|
|
76
|
+
|
|
77
|
+
| Framework | Put it at the top of |
|
|
78
|
+
|---|---|
|
|
79
|
+
| Django | `manage.py`, `wsgi.py` and `asgi.py` (before `django` is imported), or the first line of `settings.py` |
|
|
80
|
+
| Flask / FastAPI | the module that creates the app, before the config is read |
|
|
81
|
+
| Celery | the module that creates the `Celery()` app |
|
|
82
|
+
| python-dotenv | **before** `load_dotenv()` |
|
|
83
|
+
|
|
84
|
+
`load()` blocks until every import is loaded, writes `os.environ`, removes `VAULT_TOKEN`, and returns the names it set. It does nothing if the environment was already loaded (for example by the wrapper, or in a reloader's child process).
|
|
85
|
+
|
|
86
|
+
### C. Just get the values
|
|
87
|
+
|
|
88
|
+
```python
|
|
89
|
+
from vault_env_loader import fetch
|
|
90
|
+
values = fetch() # {'DB_PASSWORD': '...', ...}; os.environ is not touched
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
## Configuration
|
|
94
|
+
|
|
95
|
+
Connection settings use Vault's standard variable names, so the `vault` CLI, `hvac` and other clients share them. Options of this tool start with `VAULT_ENV_`. Command-line flags of the same name take precedence.
|
|
96
|
+
|
|
97
|
+
| Variable | Flag | Meaning |
|
|
98
|
+
|---|---|---|
|
|
99
|
+
| `VAULT_ADDR` | `--addr` | Vault address. Required. |
|
|
100
|
+
| `VAULT_TOKEN` | — | Token. There is deliberately no `--token` flag: arguments are visible in `ps`. |
|
|
101
|
+
| `VAULT_TOKEN_FILE` | `--token-file` | Read the token from a file instead. |
|
|
102
|
+
| `VAULT_NAMESPACE` | — | Vault Enterprise namespace. |
|
|
103
|
+
| `VAULT_CACERT` / `VAULT_CAPATH` | — | Extra CA certificate file / directory (every `*.pem` and `*.crt` in it). The system trust store is always used too. |
|
|
104
|
+
| `VAULT_ENV_IMPORT` | `--import` | What to load, comma or newline separated (flag repeatable). Required. |
|
|
105
|
+
| `VAULT_ENV_BASE` | `--base` | Base path for relative imports. |
|
|
106
|
+
| `VAULT_ENV_OVERRIDE` | `--override` | `true`: replace variables that are already set. Default: keep them. |
|
|
107
|
+
| `VAULT_ENV_KEEP_TOKEN` | `--keep-token` | `true`: leave `VAULT_TOKEN` in the application's environment. |
|
|
108
|
+
| `VAULT_ENV_TIMEOUT` | `--timeout` | Seconds per request. Default 10. |
|
|
109
|
+
| `VAULT_ENV_RETRIES` | `--retries` | Retries for transient failures. Default 3 (1 s, 2 s, 4 s). |
|
|
110
|
+
| `VAULT_ENV_ALLOW_HTTP` | — | `true` to allow plain `http://` to a host other than localhost. Prints a warning on every start. |
|
|
111
|
+
|
|
112
|
+
**TLS verification cannot be turned off.** If `VAULT_SKIP_VERIFY` is set, vault-env-loader stops with an error rather than silently ignoring it. Point `VAULT_CACERT` at your CA instead.
|
|
113
|
+
|
|
114
|
+
**Proxies:** like every Python HTTP client, requests honour `HTTPS_PROXY` / `HTTP_PROXY` and `NO_PROXY`. Servers often set a global proxy for `pip`; if Vault is reachable directly, add its host to `NO_PROXY`. (The Node.js package does not use proxies.)
|
|
115
|
+
|
|
116
|
+
If neither `VAULT_ADDR` nor `VAULT_ENV_IMPORT` is set, nothing is loaded and the command runs anyway — the same start command works on a laptop without Vault. Setting only some of them is an error.
|
|
117
|
+
|
|
118
|
+
## Imports
|
|
119
|
+
|
|
120
|
+
```
|
|
121
|
+
[optional:]<location>[?prefix=<PREFIX>&name=auto|verbatim|upper&kv=1|2]
|
|
122
|
+
|
|
123
|
+
location: vault://<mount>/<path> absolute, as in Spring Cloud Vault
|
|
124
|
+
<path> relative to the base path
|
|
125
|
+
vault:// the base path itself
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
- **`optional:`** — skip the import if the path does not exist. Permission errors and outages still fail.
|
|
129
|
+
- **`prefix=`** — prepend to every variable name from this import.
|
|
130
|
+
- **`kv=`** — the KV engine version. Without it, vault-env-loader asks Vault (`sys/internal/ui/mounts`, like `vault kv get` does); the token's policy needs `read` on `sys/internal/ui/mounts/*` for that, otherwise version 2 is assumed.
|
|
131
|
+
- When several imports set the same variable, **the later one wins** (with a warning).
|
|
132
|
+
|
|
133
|
+
### Relative imports and the base path
|
|
134
|
+
|
|
135
|
+
The base path is taken from, in order:
|
|
136
|
+
|
|
137
|
+
1. `VAULT_ENV_BASE` (e.g. `secret/order-service/prod`);
|
|
138
|
+
2. the token's metadata: if it has both `application` and `environment`, the base is `secret/<application>/<environment>`. Create such a token with `vault token create -policy=... -metadata=application=order-service -metadata=environment=prod`.
|
|
139
|
+
|
|
140
|
+
With a token-derived base, every environment uses **the same** `VAULT_ENV_IMPORT="database, redis"` and only the token differs.
|
|
141
|
+
|
|
142
|
+
## Variable names
|
|
143
|
+
|
|
144
|
+
Vault keys may contain characters that are not valid in environment variable names (`db.password`, `spring.datasource.url`). The `name=` option decides:
|
|
145
|
+
|
|
146
|
+
| `name=` | Rule | `DB_PASSWORD` | `db.password` | `spring.datasource.url` | `database_url` |
|
|
147
|
+
|---|---|---|---|---|---|
|
|
148
|
+
| `auto` (default) | Valid names unchanged; others upper-cased with invalid characters → `_` | `DB_PASSWORD` | `DB_PASSWORD` | `SPRING_DATASOURCE_URL` | `database_url` |
|
|
149
|
+
| `verbatim` | Unchanged (`os.environ['db.password']` works; shells cannot use it) | `DB_PASSWORD` | `db.password` | `spring.datasource.url` | `database_url` |
|
|
150
|
+
| `upper` | Always upper-cased, invalid characters → `_` | `DB_PASSWORD` | `DB_PASSWORD` | `SPRING_DATASOURCE_URL` | `DATABASE_URL` |
|
|
151
|
+
|
|
152
|
+
If two keys of one import map to the same name (`db.password` and `db-password`), loading **fails** and names both keys — order never silently decides. On Windows, names are compared case-insensitively. Non-string values (Vault allows JSON) are injected as compact JSON text.
|
|
153
|
+
|
|
154
|
+
## Precedence
|
|
155
|
+
|
|
156
|
+
Variables that already exist in the environment **win** by default — the same rule as python-dotenv and Spring. That lets you override a value locally, and it means `load_dotenv()` called *after* vault-env-loader cannot overwrite Vault's values (unless you pass `override=True`), so a stale `.env` can stay in place until you have verified the migration. vault-env-loader prints the names (never the values) of variables it left alone. `VAULT_ENV_OVERRIDE=true` reverses this.
|
|
157
|
+
|
|
158
|
+
## Checking a configuration
|
|
159
|
+
|
|
160
|
+
```bash
|
|
161
|
+
vault-env-loader check
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
prints the token's expiry and metadata, the path and version each import resolved to, and the variables that would be set — **names and lengths only, never values**.
|
|
165
|
+
|
|
166
|
+
## Failures
|
|
167
|
+
|
|
168
|
+
Nothing is started unless every non-optional import loads. The process exits with **78** (`EX_CONFIG`) and one line on stderr; `load()` and `fetch()` raise `VaultEnvError`.
|
|
169
|
+
|
|
170
|
+
| Situation | Retried? |
|
|
171
|
+
|---|---|
|
|
172
|
+
| 401 / 403 — invalid, expired or revoked token, or the policy does not allow the path | no |
|
|
173
|
+
| 404 — path does not exist (skipped if `optional:`) | no |
|
|
174
|
+
| 429, 500, 502, 503, 504, connection errors, timeouts | yes, `VAULT_ENV_RETRIES` times — except a 502 that names keys with a permanent reason (e.g. `alias 'x' could not be resolved: not_found`), which fails at once |
|
|
175
|
+
| 3xx redirects | not followed (the token would go with them); fails |
|
|
176
|
+
|
|
177
|
+
A token that expires within 14 days produces a warning.
|
|
178
|
+
|
|
179
|
+
## Things to know
|
|
180
|
+
|
|
181
|
+
- **Rotation needs a restart.** Values are read once, at start — just like editing `.env`. `check` shows each path's version so you can tell whether a restart is due.
|
|
182
|
+
- **gunicorn / uwsgi / celery** — with the wrapper, the master loads once and workers inherit the environment. With `load()`, put it where the master imports it (e.g. `gunicorn.conf.py`, or use `--preload`) so that workers do not each read Vault.
|
|
183
|
+
- **Reloaders** (`runserver`, `uvicorn --reload`) start children with the loaded environment and `VAULT_ENV_LOADED`, so they do not read again. Restart the outer command to pick up new values.
|
|
184
|
+
- **One value is limited to 128 KB** (the Linux limit for one environment string); larger values are refused with the key name.
|
|
185
|
+
- **No `.pth` auto-loading.** Loading on every interpreter start (including `pip`) would be invisible and fragile; the wrapper already needs no code change.
|
|
186
|
+
- **Windows** has no `exec`: the wrapper runs the command as a child and passes its exit code through.
|
|
187
|
+
- Only token authentication for now; AppRole and Kubernetes auth may follow.
|
|
188
|
+
|
|
189
|
+
## Library API
|
|
190
|
+
|
|
191
|
+
```python
|
|
192
|
+
from vault_env_loader import load, fetch, VaultEnvError
|
|
193
|
+
|
|
194
|
+
load(env=None, **options) # -> names set in os.environ (or in the mapping `env`)
|
|
195
|
+
fetch(env=None, **options) # -> {NAME: value}; does not touch the environment
|
|
196
|
+
```
|
|
197
|
+
|
|
198
|
+
`options` mirror the variables above: `addr`, `token`, `token_file`, `imports`, `base`, `namespace`, `cacert`, `capath`, `override`, `keep_token`, `timeout`, `retries`, `allow_http`. `env` is the mapping configuration is read from (and, for `load()`, written to); default `os.environ`. `VaultEnvError` has a `code`: `config`, `auth`, `not_found`, `resolve`, `collision`, `value`, `unavailable`. Messages contain paths and key names, never values or tokens.
|
|
199
|
+
|
|
200
|
+
## Security notes
|
|
201
|
+
|
|
202
|
+
- Secret values are never written to logs, error messages, argv or disk. Unexpected errors print only the exception type, never a traceback.
|
|
203
|
+
- Redirects are not followed, so the token is only ever sent to `VAULT_ADDR`.
|
|
204
|
+
- Requests carry `User-Agent: vault-env-loader/<version> python/<version>`, so server audit logs can tell which client read what.
|
|
205
|
+
|
|
206
|
+
## License
|
|
207
|
+
|
|
208
|
+
MIT
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
vault_env_loader.py,sha256=cFXWgQWOcGYIDgdcuAM3I2uWYg-QKlGlMVnDeTnwrYI,34206
|
|
2
|
+
vault_env_loader-0.1.0.dist-info/licenses/LICENSE,sha256=zfIqtPxuRVI2pA6arKl_-G3qQU_8EorFucaFN1nKWK4,1065
|
|
3
|
+
vault_env_loader-0.1.0.dist-info/METADATA,sha256=6ndazM4FmrpNnwjJ5Uq-DgD8wevkuVFEWeZH5RyDosU,12244
|
|
4
|
+
vault_env_loader-0.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
|
|
5
|
+
vault_env_loader-0.1.0.dist-info/entry_points.txt,sha256=GZMjgaIVruI2SKxDmQe0aNDsd1-fmuBqUJIJEmITlG8,59
|
|
6
|
+
vault_env_loader-0.1.0.dist-info/top_level.txt,sha256=30RFT_XrDSyywd5Ms1UuL1G1fZ4CiuKmPqpBw22dGAw,17
|
|
7
|
+
vault_env_loader-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Kaylov H
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
vault_env_loader
|
vault_env_loader.py
ADDED
|
@@ -0,0 +1,844 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""vault-env-loader: load HashiCorp Vault KV secrets into environment variables
|
|
3
|
+
before your application starts.
|
|
4
|
+
|
|
5
|
+
One file, zero dependencies, Python >= 3.6. It is at the same time:
|
|
6
|
+
- the CLI: vault-env-loader [options] -- <command...>
|
|
7
|
+
python -m vault_env_loader [options] -- <command...>
|
|
8
|
+
- the library: load() / fetch()
|
|
9
|
+
|
|
10
|
+
Behaviour follows the Node.js package of the same name; both run the shared
|
|
11
|
+
cases in spec/cases.json. Secret values never go to logs, error messages, argv
|
|
12
|
+
or disk.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
import http.client
|
|
16
|
+
import json
|
|
17
|
+
import os
|
|
18
|
+
import re
|
|
19
|
+
import socket
|
|
20
|
+
import ssl
|
|
21
|
+
import subprocess
|
|
22
|
+
import sys
|
|
23
|
+
import time
|
|
24
|
+
import urllib.error
|
|
25
|
+
import urllib.parse
|
|
26
|
+
import urllib.request
|
|
27
|
+
|
|
28
|
+
__version__ = '0.1.0'
|
|
29
|
+
__all__ = ['load', 'fetch', 'VaultEnvError', '__version__']
|
|
30
|
+
|
|
31
|
+
TAG = '[vault-env-loader]'
|
|
32
|
+
EX_CONFIG = 78
|
|
33
|
+
LOADED_MARK = 'VAULT_ENV_LOADED'
|
|
34
|
+
TTL_WARN_SECONDS = 14 * 24 * 3600
|
|
35
|
+
# Linux MAX_ARG_STRLEN: one "NAME=value\0" string handed to execve.
|
|
36
|
+
MAX_ENV_STRING = 128 * 1024
|
|
37
|
+
MAX_RESPONSE_BYTES = 32 * 1024 * 1024
|
|
38
|
+
VALID_ENV_NAME = re.compile(r'^[A-Za-z_][A-Za-z0-9_]*$')
|
|
39
|
+
NAME_MODES = ('auto', 'verbatim', 'upper')
|
|
40
|
+
# Some Vault-compatible servers that resolve keys from other stores answer 502
|
|
41
|
+
# with "alias 'x' could not be resolved: <reason>" per failed key.
|
|
42
|
+
TRANSIENT_ALIAS_REASONS = ('timeout', 'unavailable')
|
|
43
|
+
ALIAS_FAILURE = re.compile(r"^alias '(.+)' could not be resolved: ([a-z_]+)$")
|
|
44
|
+
TRUTHY = ('1', 'true', 'yes', 'on')
|
|
45
|
+
FALSY = ('0', 'false', 'no', 'off')
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class VaultEnvError(Exception):
|
|
49
|
+
"""A configuration or Vault error. Messages name paths and keys, never values or tokens."""
|
|
50
|
+
|
|
51
|
+
def __init__(self, message, code='error'):
|
|
52
|
+
super(VaultEnvError, self).__init__(message)
|
|
53
|
+
self.message = message
|
|
54
|
+
self.code = code
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def warn(message):
|
|
58
|
+
sys.stderr.write('%s %s\n' % (TAG, message))
|
|
59
|
+
sys.stderr.flush()
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def truthy(v):
|
|
63
|
+
return isinstance(v, str) and v.strip().lower() in TRUTHY
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def non_empty(v):
|
|
67
|
+
return v.strip() if isinstance(v, str) and v.strip() != '' else None
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
# ---------------------------------------------------------------------------
|
|
71
|
+
# Configuration
|
|
72
|
+
|
|
73
|
+
def read_config(options=None, env=None):
|
|
74
|
+
options = options or {}
|
|
75
|
+
env = os.environ if env is None else env
|
|
76
|
+
|
|
77
|
+
def pick(opt, *names):
|
|
78
|
+
if options.get(opt) is not None:
|
|
79
|
+
return options[opt]
|
|
80
|
+
for n in names:
|
|
81
|
+
if non_empty(env.get(n)) is not None:
|
|
82
|
+
return env[n]
|
|
83
|
+
return None
|
|
84
|
+
|
|
85
|
+
skip = non_empty(env.get('VAULT_SKIP_VERIFY'))
|
|
86
|
+
if skip is not None and skip.lower() not in FALSY:
|
|
87
|
+
raise VaultEnvError('VAULT_SKIP_VERIFY is set, but vault-env-loader never skips TLS verification. '
|
|
88
|
+
'Unset it and point VAULT_CACERT at your CA instead.', 'config')
|
|
89
|
+
|
|
90
|
+
addr = non_empty(pick('addr', 'VAULT_ADDR'))
|
|
91
|
+
imports = pick('imports', 'VAULT_ENV_IMPORT')
|
|
92
|
+
if isinstance(imports, str):
|
|
93
|
+
imports = split_list(imports)
|
|
94
|
+
imports = [str(s).strip() for s in (imports or []) if str(s).strip()]
|
|
95
|
+
|
|
96
|
+
token_file = non_empty(pick('token_file', 'VAULT_TOKEN_FILE'))
|
|
97
|
+
token = options['token'] if options.get('token') is not None else non_empty(env.get('VAULT_TOKEN'))
|
|
98
|
+
|
|
99
|
+
if not addr and not imports:
|
|
100
|
+
return {'configured': False}
|
|
101
|
+
|
|
102
|
+
if not addr:
|
|
103
|
+
raise VaultEnvError('VAULT_ADDR is not set (VAULT_ENV_IMPORT is).', 'config')
|
|
104
|
+
if not imports:
|
|
105
|
+
raise VaultEnvError('VAULT_ENV_IMPORT is empty: name at least one path to load.', 'config')
|
|
106
|
+
if not token and token_file:
|
|
107
|
+
try:
|
|
108
|
+
with open(token_file, 'r', encoding='utf-8') as f:
|
|
109
|
+
token = f.read().strip()
|
|
110
|
+
except OSError as e:
|
|
111
|
+
raise VaultEnvError('Cannot read VAULT_TOKEN_FILE (%s): %s.' % (token_file, _errno_name(e)), 'config')
|
|
112
|
+
if not token:
|
|
113
|
+
raise VaultEnvError('VAULT_TOKEN_FILE (%s) is empty.' % token_file, 'config')
|
|
114
|
+
if not token:
|
|
115
|
+
raise VaultEnvError('No token: set VAULT_TOKEN or VAULT_TOKEN_FILE.', 'config')
|
|
116
|
+
|
|
117
|
+
url = urllib.parse.urlsplit(addr)
|
|
118
|
+
if url.scheme not in ('http', 'https') or not url.hostname:
|
|
119
|
+
if url.scheme and url.scheme not in ('http', 'https'):
|
|
120
|
+
raise VaultEnvError('VAULT_ADDR must start with https://', 'config')
|
|
121
|
+
raise VaultEnvError('VAULT_ADDR is not a valid URL.', 'config')
|
|
122
|
+
allow_http = bool(options['allow_http']) if options.get('allow_http') is not None else truthy(env.get('VAULT_ENV_ALLOW_HTTP'))
|
|
123
|
+
if url.scheme == 'http' and not is_loopback(url.hostname) and not allow_http:
|
|
124
|
+
raise VaultEnvError('VAULT_ADDR uses plain http to %s; use https, or set VAULT_ENV_ALLOW_HTTP=true '
|
|
125
|
+
'if this network really has no TLS.' % url.hostname, 'config')
|
|
126
|
+
|
|
127
|
+
def num(v, default, name):
|
|
128
|
+
if v is None or v == '':
|
|
129
|
+
return default
|
|
130
|
+
try:
|
|
131
|
+
n = float(v)
|
|
132
|
+
except (TypeError, ValueError):
|
|
133
|
+
n = -1
|
|
134
|
+
if n != n or n < 0 or n == float('inf'):
|
|
135
|
+
raise VaultEnvError('%s must be a non-negative number.' % name, 'config')
|
|
136
|
+
return n
|
|
137
|
+
|
|
138
|
+
def flag(opt, var):
|
|
139
|
+
return bool(options[opt]) if options.get(opt) is not None else truthy(env.get(var))
|
|
140
|
+
|
|
141
|
+
return {
|
|
142
|
+
'configured': True,
|
|
143
|
+
'addr': addr.rstrip('/'),
|
|
144
|
+
'insecure_http': url.scheme == 'http' and not is_loopback(url.hostname),
|
|
145
|
+
'token': token,
|
|
146
|
+
'token_from_file': non_empty(env.get('VAULT_TOKEN')) is None and options.get('token') is None and bool(token_file),
|
|
147
|
+
'namespace': non_empty(pick('namespace', 'VAULT_NAMESPACE')),
|
|
148
|
+
'ssl_context': ssl_context(non_empty(pick('cacert', 'VAULT_CACERT')), non_empty(pick('capath', 'VAULT_CAPATH'))),
|
|
149
|
+
'imports': [parse_import(i) for i in imports],
|
|
150
|
+
'base': non_empty(pick('base', 'VAULT_ENV_BASE')),
|
|
151
|
+
'override': flag('override', 'VAULT_ENV_OVERRIDE'),
|
|
152
|
+
'keep_token': flag('keep_token', 'VAULT_ENV_KEEP_TOKEN'),
|
|
153
|
+
'timeout': num(pick('timeout', 'VAULT_ENV_TIMEOUT'), 10, 'VAULT_ENV_TIMEOUT'),
|
|
154
|
+
'retries': int(num(pick('retries', 'VAULT_ENV_RETRIES'), 3, 'VAULT_ENV_RETRIES')),
|
|
155
|
+
'retry_delay': options['retry_delay'] if options.get('retry_delay') is not None else 1.0,
|
|
156
|
+
'case_insensitive': bool(options['case_insensitive']) if options.get('case_insensitive') is not None else os.name == 'nt',
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def is_loopback(host):
|
|
161
|
+
return host in ('localhost', '127.0.0.1', '::1')
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def split_list(s):
|
|
165
|
+
return re.split(r'[,\n]', s)
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def _errno_name(e):
|
|
169
|
+
import errno
|
|
170
|
+
return errno.errorcode.get(getattr(e, 'errno', None) or 0, 'error')
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def ssl_context(cafile, capath):
|
|
174
|
+
"""The system trust store plus VAULT_CACERT / every *.pem and *.crt in VAULT_CAPATH."""
|
|
175
|
+
ctx = ssl.create_default_context()
|
|
176
|
+
if cafile:
|
|
177
|
+
try:
|
|
178
|
+
ctx.load_verify_locations(cafile=cafile)
|
|
179
|
+
except (OSError, ssl.SSLError) as e:
|
|
180
|
+
raise VaultEnvError('Cannot read VAULT_CACERT (%s): %s.' % (cafile, _errno_name(e)), 'config')
|
|
181
|
+
if capath:
|
|
182
|
+
# OpenSSL's capath wants hashed file names; read the files ourselves, like the Node package.
|
|
183
|
+
try:
|
|
184
|
+
names = sorted(os.listdir(capath))
|
|
185
|
+
except OSError as e:
|
|
186
|
+
raise VaultEnvError('Cannot read VAULT_CAPATH (%s): %s.' % (capath, _errno_name(e)), 'config')
|
|
187
|
+
for n in names:
|
|
188
|
+
if re.search(r'\.(pem|crt)$', n, re.I):
|
|
189
|
+
try:
|
|
190
|
+
with open(os.path.join(capath, n), 'r', encoding='utf-8') as f:
|
|
191
|
+
ctx.load_verify_locations(cadata=f.read())
|
|
192
|
+
except (OSError, ssl.SSLError, ValueError):
|
|
193
|
+
raise VaultEnvError('Cannot load certificate %s from VAULT_CAPATH.' % n, 'config')
|
|
194
|
+
return ctx
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
# ---------------------------------------------------------------------------
|
|
198
|
+
# Imports: [optional:]location[?prefix=..&name=auto|verbatim|upper&kv=1|2]
|
|
199
|
+
# vault://<mount>/<path> absolute (same as Spring Cloud Vault)
|
|
200
|
+
# vault:// the base path itself
|
|
201
|
+
# <path> relative to the base path
|
|
202
|
+
|
|
203
|
+
def parse_import(raw):
|
|
204
|
+
spec = raw.strip()
|
|
205
|
+
optional = False
|
|
206
|
+
if spec.startswith('optional:'):
|
|
207
|
+
optional = True
|
|
208
|
+
spec = spec[len('optional:'):].strip()
|
|
209
|
+
query = ''
|
|
210
|
+
q = spec.find('?')
|
|
211
|
+
if q >= 0:
|
|
212
|
+
query = spec[q + 1:]
|
|
213
|
+
spec = spec[:q]
|
|
214
|
+
prefix, name, kv = '', 'auto', None
|
|
215
|
+
for part in query.split('&') if query else []:
|
|
216
|
+
if not part:
|
|
217
|
+
continue
|
|
218
|
+
k, _, v = part.partition('=')
|
|
219
|
+
k, v = urllib.parse.unquote(k), urllib.parse.unquote(v)
|
|
220
|
+
if k == 'prefix':
|
|
221
|
+
prefix = v
|
|
222
|
+
elif k == 'name':
|
|
223
|
+
if v not in NAME_MODES:
|
|
224
|
+
raise VaultEnvError('Import "%s": name must be one of %s.' % (raw, ', '.join(NAME_MODES)), 'config')
|
|
225
|
+
name = v
|
|
226
|
+
elif k == 'kv':
|
|
227
|
+
if v not in ('1', '2'):
|
|
228
|
+
raise VaultEnvError('Import "%s": kv must be 1 or 2.' % raw, 'config')
|
|
229
|
+
kv = int(v)
|
|
230
|
+
else:
|
|
231
|
+
raise VaultEnvError('Import "%s": unknown option "%s" (known: prefix, name, kv).' % (raw, k), 'config')
|
|
232
|
+
if name != 'verbatim' and not re.match(r'^[A-Za-z0-9_]*$', prefix):
|
|
233
|
+
raise VaultEnvError('Import "%s": prefix may only contain letters, digits and _.' % raw, 'config')
|
|
234
|
+
|
|
235
|
+
if spec in ('vault://', 'vault:'):
|
|
236
|
+
kind, p = 'base', ''
|
|
237
|
+
elif spec.startswith('vault://'):
|
|
238
|
+
kind, p = 'absolute', spec[len('vault://'):]
|
|
239
|
+
elif '://' in spec:
|
|
240
|
+
raise VaultEnvError('Import "%s": only vault:// locations are supported.' % raw, 'config')
|
|
241
|
+
else:
|
|
242
|
+
kind, p = 'relative', spec
|
|
243
|
+
p = p.strip('/')
|
|
244
|
+
if kind != 'base' and not p:
|
|
245
|
+
raise VaultEnvError('Import "%s": empty path.' % raw, 'config')
|
|
246
|
+
if p and any(s in ('', '.', '..') for s in p.split('/')):
|
|
247
|
+
raise VaultEnvError('Import "%s": path segments may not be empty, "." or "..".' % raw, 'config')
|
|
248
|
+
if kind == 'absolute' and '/' not in p:
|
|
249
|
+
raise VaultEnvError('Import "%s": expected vault://<mount>/<path>.' % raw, 'config')
|
|
250
|
+
return {'raw': raw, 'optional': optional, 'kind': kind, 'path': p, 'prefix': prefix, 'name': name, 'kv': kv}
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
# ---------------------------------------------------------------------------
|
|
254
|
+
# HTTP
|
|
255
|
+
|
|
256
|
+
class _NoRedirect(urllib.request.HTTPRedirectHandler):
|
|
257
|
+
# urllib would follow a 3xx and send X-Vault-Token to wherever it points.
|
|
258
|
+
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
|
259
|
+
return None
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
class _NetworkError(Exception):
|
|
263
|
+
def __init__(self, what):
|
|
264
|
+
super(_NetworkError, self).__init__(what)
|
|
265
|
+
self.what = what
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def encode_path(p):
|
|
269
|
+
return '/'.join(urllib.parse.quote(s, safe='') for s in p.split('/'))
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
def _opener(cfg):
|
|
273
|
+
if '_opener' not in cfg:
|
|
274
|
+
# Proxies come from HTTPS_PROXY / HTTP_PROXY / NO_PROXY, as everywhere in Python.
|
|
275
|
+
cfg['_opener'] = urllib.request.build_opener(
|
|
276
|
+
_NoRedirect(), urllib.request.HTTPSHandler(context=cfg['ssl_context']))
|
|
277
|
+
return cfg['_opener']
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
def http_get(cfg, api_path):
|
|
281
|
+
"""GET <addr>/v1/<api_path> -> (status, parsed JSON body or None). Raises _NetworkError."""
|
|
282
|
+
headers = {
|
|
283
|
+
'X-Vault-Token': cfg['token'],
|
|
284
|
+
'X-Vault-Request': 'true',
|
|
285
|
+
'User-Agent': 'vault-env-loader/%s python/%s' % (__version__, '.'.join(map(str, sys.version_info[:3]))),
|
|
286
|
+
'Accept': 'application/json',
|
|
287
|
+
}
|
|
288
|
+
if cfg.get('namespace'):
|
|
289
|
+
headers['X-Vault-Namespace'] = cfg['namespace']
|
|
290
|
+
req = urllib.request.Request('%s/v1/%s' % (cfg['addr'], api_path), headers=headers, method='GET')
|
|
291
|
+
try:
|
|
292
|
+
resp = _opener(cfg).open(req, timeout=cfg['timeout'])
|
|
293
|
+
except urllib.error.HTTPError as e:
|
|
294
|
+
resp = e
|
|
295
|
+
except socket.timeout:
|
|
296
|
+
raise _NetworkError('timed out')
|
|
297
|
+
except urllib.error.URLError as e:
|
|
298
|
+
reason = e.reason
|
|
299
|
+
if isinstance(reason, socket.timeout):
|
|
300
|
+
raise _NetworkError('timed out')
|
|
301
|
+
if isinstance(reason, ssl.SSLError):
|
|
302
|
+
raise _NetworkError('failed (TLS: %s)' % (getattr(reason, 'reason', None) or 'certificate verification failed'))
|
|
303
|
+
raise _NetworkError('failed (%s)' % (_errno_name(reason) if isinstance(reason, OSError) else 'network error'))
|
|
304
|
+
except (OSError, ValueError, http.client.HTTPException) as e:
|
|
305
|
+
raise _NetworkError('failed (%s)' % (_errno_name(e) if isinstance(e, OSError) else type(e).__name__))
|
|
306
|
+
try:
|
|
307
|
+
status = resp.getcode()
|
|
308
|
+
try:
|
|
309
|
+
data = resp.read(MAX_RESPONSE_BYTES + 1)
|
|
310
|
+
except socket.timeout:
|
|
311
|
+
raise _NetworkError('timed out')
|
|
312
|
+
except OSError as e:
|
|
313
|
+
raise _NetworkError('failed (%s)' % _errno_name(e))
|
|
314
|
+
except http.client.HTTPException as e:
|
|
315
|
+
raise _NetworkError('failed (%s)' % type(e).__name__)
|
|
316
|
+
if len(data) > MAX_RESPONSE_BYTES:
|
|
317
|
+
raise _NetworkError('failed (response too large)')
|
|
318
|
+
finally:
|
|
319
|
+
resp.close()
|
|
320
|
+
try:
|
|
321
|
+
body = json.loads(data.decode('utf-8')) if data else None
|
|
322
|
+
except ValueError:
|
|
323
|
+
body = None
|
|
324
|
+
return status, body
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
def alias_failures(body):
|
|
328
|
+
errors = body.get('errors') if isinstance(body, dict) else None
|
|
329
|
+
out = []
|
|
330
|
+
for e in errors if isinstance(errors, list) else []:
|
|
331
|
+
m = ALIAS_FAILURE.match(str(e))
|
|
332
|
+
if m:
|
|
333
|
+
out.append({'alias': m.group(1), 'reason': m.group(2)})
|
|
334
|
+
return out
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
def retryable(status, body):
|
|
338
|
+
if status in (429, 500, 503, 504):
|
|
339
|
+
return True
|
|
340
|
+
if status == 502:
|
|
341
|
+
failures = alias_failures(body)
|
|
342
|
+
return not failures or all(f['reason'] in TRANSIENT_ALIAS_REASONS for f in failures)
|
|
343
|
+
return False
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
def get_with_retry(cfg, api_path):
|
|
347
|
+
"""GET with retries on transient failures; returns the final (status, body)."""
|
|
348
|
+
attempt = 0
|
|
349
|
+
while True:
|
|
350
|
+
res, net_err = None, None
|
|
351
|
+
try:
|
|
352
|
+
res = http_get(cfg, api_path)
|
|
353
|
+
except _NetworkError as e:
|
|
354
|
+
net_err = e
|
|
355
|
+
if res is not None and not retryable(*res):
|
|
356
|
+
return res
|
|
357
|
+
if attempt >= cfg['retries']:
|
|
358
|
+
if res is not None:
|
|
359
|
+
return res
|
|
360
|
+
hint = '' if net_err.what == 'timed out' else _proxy_hint(cfg)
|
|
361
|
+
raise VaultEnvError('Request to Vault %s.%s' % (net_err.what, hint), 'unavailable')
|
|
362
|
+
time.sleep(cfg['retry_delay'] * (2 ** attempt))
|
|
363
|
+
attempt += 1
|
|
364
|
+
|
|
365
|
+
|
|
366
|
+
def _proxy_hint(cfg):
|
|
367
|
+
host = urllib.parse.urlsplit(cfg['addr']).hostname
|
|
368
|
+
proxies = urllib.request.getproxies()
|
|
369
|
+
scheme = urllib.parse.urlsplit(cfg['addr']).scheme
|
|
370
|
+
if proxies.get(scheme) and not urllib.request.proxy_bypass(host):
|
|
371
|
+
return ' The request went through the proxy in %s_PROXY; add %s to NO_PROXY if Vault is reachable directly.' % (scheme.upper(), host)
|
|
372
|
+
return ''
|
|
373
|
+
|
|
374
|
+
|
|
375
|
+
# ---------------------------------------------------------------------------
|
|
376
|
+
# Vault reads
|
|
377
|
+
|
|
378
|
+
def lookup_self(cfg):
|
|
379
|
+
status, body = get_with_retry(cfg, 'auth/token/lookup-self')
|
|
380
|
+
if status in (401, 403):
|
|
381
|
+
return {'denied': True, 'ttl': 0, 'meta': {}}
|
|
382
|
+
if status != 200 or not isinstance(body, dict) or not isinstance(body.get('data'), dict):
|
|
383
|
+
return None
|
|
384
|
+
d = body['data']
|
|
385
|
+
ttl = d.get('ttl')
|
|
386
|
+
meta = d.get('meta')
|
|
387
|
+
return {
|
|
388
|
+
'denied': False,
|
|
389
|
+
'ttl': ttl if isinstance(ttl, (int, float)) and not isinstance(ttl, bool) else 0,
|
|
390
|
+
'meta': meta if isinstance(meta, dict) else {},
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
|
|
394
|
+
def resolve_base(cfg, me):
|
|
395
|
+
if cfg.get('base'):
|
|
396
|
+
return {'base': cfg['base'].strip('/'), 'source': 'VAULT_ENV_BASE'}
|
|
397
|
+
m = me and me.get('meta') or {}
|
|
398
|
+
app, env = non_empty(m.get('application')), non_empty(m.get('environment'))
|
|
399
|
+
if app and env:
|
|
400
|
+
return {'base': 'secret/%s/%s' % (m['application'], m['environment']), 'source': 'token metadata',
|
|
401
|
+
'app': m['application'], 'env': m['environment']}
|
|
402
|
+
return {'base': None, 'source': None, 'denied': bool(me and me.get('denied'))}
|
|
403
|
+
|
|
404
|
+
|
|
405
|
+
def full_path(imp, base_info):
|
|
406
|
+
if imp['kind'] == 'absolute':
|
|
407
|
+
if base_info.get('source') == 'token metadata':
|
|
408
|
+
seg = imp['path'].split('/') + ['', '']
|
|
409
|
+
if seg[0] == 'secret' and (seg[1] != base_info['app'] or seg[2] != base_info['env']):
|
|
410
|
+
raise VaultEnvError(
|
|
411
|
+
'Import "%s" points at %s/%s, but this token belongs to %s/%s. A token can only read its own '
|
|
412
|
+
'application and environment.' % (imp['raw'], seg[1] or '?', seg[2] or '?',
|
|
413
|
+
base_info['app'], base_info['env']), 'config')
|
|
414
|
+
return imp['path']
|
|
415
|
+
if not base_info.get('base'):
|
|
416
|
+
if base_info.get('denied'):
|
|
417
|
+
raise VaultEnvError(
|
|
418
|
+
'Import "%s" is relative, and Vault refused to describe the token (permission denied on '
|
|
419
|
+
'auth/token/lookup-self): the token is invalid, expired or revoked, its IP is not allowed, or its '
|
|
420
|
+
'policy lacks lookup-self. Set VAULT_ENV_BASE or write vault://<mount>/<path> if the policy is '
|
|
421
|
+
'intended.' % imp['raw'], 'auth')
|
|
422
|
+
raise VaultEnvError('Import "%s" is relative, but there is no base path: set VAULT_ENV_BASE or write '
|
|
423
|
+
'vault://<mount>/<path>.' % imp['raw'], 'config')
|
|
424
|
+
return base_info['base'] if imp['kind'] == 'base' else '%s/%s' % (base_info['base'], imp['path'])
|
|
425
|
+
|
|
426
|
+
|
|
427
|
+
def locate(cfg, imp, p):
|
|
428
|
+
"""Learn the mount and KV version the way `vault kv get` does."""
|
|
429
|
+
first = p.split('/')[0]
|
|
430
|
+
fallback = {'mount': first, 'rest': p[len(first) + 1:], 'kv': imp['kv'] or 2, 'probed': False}
|
|
431
|
+
if imp['kv']:
|
|
432
|
+
return fallback
|
|
433
|
+
status, body = get_with_retry(cfg, 'sys/internal/ui/mounts/%s' % encode_path(p))
|
|
434
|
+
d = body.get('data') if status == 200 and isinstance(body, dict) else None
|
|
435
|
+
if not isinstance(d, dict):
|
|
436
|
+
return fallback
|
|
437
|
+
mount = d['path'].rstrip('/') if isinstance(d.get('path'), str) and d['path'] else first
|
|
438
|
+
if p != mount and not p.startswith(mount + '/'):
|
|
439
|
+
return fallback
|
|
440
|
+
options = d.get('options') if isinstance(d.get('options'), dict) else {}
|
|
441
|
+
if str(options.get('version')) == '2':
|
|
442
|
+
version = 2
|
|
443
|
+
elif d.get('type') in ('kv', 'generic'):
|
|
444
|
+
version = 1
|
|
445
|
+
else:
|
|
446
|
+
version = 2
|
|
447
|
+
return {'mount': mount, 'rest': p[len(mount) + 1:], 'kv': version, 'probed': True}
|
|
448
|
+
|
|
449
|
+
|
|
450
|
+
def describe_status(status, body, imp, p, loc):
|
|
451
|
+
where = '"%s" (%s)' % (imp['raw'], p)
|
|
452
|
+
if status in (401, 403):
|
|
453
|
+
return VaultEnvError('Permission denied reading %s: the token is invalid, expired or revoked, its IP is not '
|
|
454
|
+
'allowed, or its policy does not allow reading this path.' % where, 'auth')
|
|
455
|
+
if status == 404:
|
|
456
|
+
other = 1 if loc['kv'] == 2 else 2
|
|
457
|
+
hint = '' if loc['probed'] else ' If this is a KV version %d mount, add ?kv=%d.' % (other, other)
|
|
458
|
+
return VaultEnvError('Not found: %s.%s' % (where, hint), 'not_found')
|
|
459
|
+
failures = alias_failures(body) if status == 502 else []
|
|
460
|
+
if failures:
|
|
461
|
+
listed = ', '.join('%s (%s)' % (f['alias'], f['reason']) for f in failures)
|
|
462
|
+
return VaultEnvError('Vault could not resolve %s: %s.' % (where, listed), 'resolve')
|
|
463
|
+
return VaultEnvError('Vault answered HTTP %s for %s.' % (status, where), 'unavailable')
|
|
464
|
+
|
|
465
|
+
|
|
466
|
+
def read_import(cfg, imp, base_info):
|
|
467
|
+
p = full_path(imp, base_info)
|
|
468
|
+
loc = locate(cfg, imp, p)
|
|
469
|
+
if loc['kv'] == 2:
|
|
470
|
+
api_path = '%s/data/%s' % (encode_path(loc['mount']), encode_path(loc['rest']))
|
|
471
|
+
else:
|
|
472
|
+
api_path = encode_path(p)
|
|
473
|
+
status, body = get_with_retry(cfg, api_path)
|
|
474
|
+
if status == 200 and isinstance(body, dict) and isinstance(body.get('data'), dict):
|
|
475
|
+
outer = body['data']
|
|
476
|
+
data = outer.get('data') if loc['kv'] == 2 else outer
|
|
477
|
+
meta = outer.get('metadata') if loc['kv'] == 2 else None
|
|
478
|
+
version = meta.get('version') if isinstance(meta, dict) else None
|
|
479
|
+
if isinstance(data, dict):
|
|
480
|
+
return {'path': p, 'kv': loc['kv'], 'version': version, 'data': data}
|
|
481
|
+
# KV v2 answers 200 with data: null for a deleted latest version.
|
|
482
|
+
if loc['kv'] == 2:
|
|
483
|
+
return {'path': p, 'kv': loc['kv'], 'missing': True}
|
|
484
|
+
if status == 404 and imp['optional']:
|
|
485
|
+
return {'path': p, 'kv': loc['kv'], 'missing': True}
|
|
486
|
+
raise describe_status(status, body, imp, p, loc)
|
|
487
|
+
|
|
488
|
+
|
|
489
|
+
# ---------------------------------------------------------------------------
|
|
490
|
+
# Key -> environment variable names
|
|
491
|
+
|
|
492
|
+
def env_name(key, mode):
|
|
493
|
+
if mode == 'verbatim':
|
|
494
|
+
return key
|
|
495
|
+
if mode == 'auto' and VALID_ENV_NAME.match(key):
|
|
496
|
+
return key
|
|
497
|
+
n = re.sub(r'[^A-Za-z0-9_]', '_', key).upper()
|
|
498
|
+
if re.match(r'^[0-9]', n):
|
|
499
|
+
n = '_' + n
|
|
500
|
+
return n
|
|
501
|
+
|
|
502
|
+
|
|
503
|
+
def string_value(v):
|
|
504
|
+
if isinstance(v, str):
|
|
505
|
+
return v
|
|
506
|
+
if v is None:
|
|
507
|
+
return ''
|
|
508
|
+
if isinstance(v, float) and v.is_integer() and abs(v) < 1e21:
|
|
509
|
+
v = int(v) # JSON has one number type: 1.0 is written 1, as in JavaScript
|
|
510
|
+
return json.dumps(v, separators=(',', ':'), ensure_ascii=False)
|
|
511
|
+
|
|
512
|
+
|
|
513
|
+
def map_entries(data, imp, case_insensitive):
|
|
514
|
+
"""One import's data -> [{key, name, value}]. Fails on collisions and on values an
|
|
515
|
+
environment cannot carry. Errors name keys, never values."""
|
|
516
|
+
entries = []
|
|
517
|
+
seen = {}
|
|
518
|
+
for key in data:
|
|
519
|
+
name = imp['prefix'] + env_name(key, imp['name'])
|
|
520
|
+
if not name or '=' in name or '\0' in name:
|
|
521
|
+
raise VaultEnvError('Key "%s" in "%s" cannot be used as an environment variable name; use ?name=auto '
|
|
522
|
+
'or ?name=upper.' % (key, imp['raw']), 'value')
|
|
523
|
+
cmp = name.upper() if case_insensitive else name
|
|
524
|
+
if cmp in seen:
|
|
525
|
+
raise VaultEnvError('Keys "%s" and "%s" in "%s" both become %s. Rename one of them in Vault, or choose '
|
|
526
|
+
'another ?name= mode.' % (seen[cmp], key, imp['raw'], name), 'collision')
|
|
527
|
+
seen[cmp] = key
|
|
528
|
+
value = string_value(data[key])
|
|
529
|
+
if '\0' in value:
|
|
530
|
+
raise VaultEnvError('The value of "%s" in "%s" contains a NUL byte, which an environment variable cannot '
|
|
531
|
+
'hold.' % (key, imp['raw']), 'value')
|
|
532
|
+
if len(name.encode('utf-8')) + 1 + len(value.encode('utf-8', 'surrogatepass')) + 1 > MAX_ENV_STRING:
|
|
533
|
+
raise VaultEnvError('The value of "%s" in "%s" is larger than 128 KB, the limit for one environment '
|
|
534
|
+
'variable.' % (key, imp['raw']), 'value')
|
|
535
|
+
entries.append({'key': key, 'name': name, 'value': value})
|
|
536
|
+
return entries
|
|
537
|
+
|
|
538
|
+
|
|
539
|
+
def merge_imports(results, case_insensitive):
|
|
540
|
+
"""Later imports win over earlier ones. Returns the merged values and the overridden names."""
|
|
541
|
+
values = {}
|
|
542
|
+
canonical = {}
|
|
543
|
+
overridden = []
|
|
544
|
+
for r in results:
|
|
545
|
+
for e in r.get('entries') or []:
|
|
546
|
+
cmp = e['name'].upper() if case_insensitive else e['name']
|
|
547
|
+
if cmp in canonical:
|
|
548
|
+
del values[canonical[cmp]]
|
|
549
|
+
if e['name'] not in overridden:
|
|
550
|
+
overridden.append(e['name'])
|
|
551
|
+
canonical[cmp] = e['name']
|
|
552
|
+
values[e['name']] = e['value']
|
|
553
|
+
return {'values': values, 'overridden': overridden}
|
|
554
|
+
|
|
555
|
+
|
|
556
|
+
def plan(values, env, override):
|
|
557
|
+
"""Decide what to write into env. Existing variables win unless override."""
|
|
558
|
+
to_set = {}
|
|
559
|
+
kept = []
|
|
560
|
+
for name in values:
|
|
561
|
+
if not override and env.get(name) is not None:
|
|
562
|
+
kept.append(name)
|
|
563
|
+
else:
|
|
564
|
+
to_set[name] = values[name]
|
|
565
|
+
return {'set': to_set, 'kept': kept}
|
|
566
|
+
|
|
567
|
+
|
|
568
|
+
# ---------------------------------------------------------------------------
|
|
569
|
+
# Pipeline
|
|
570
|
+
|
|
571
|
+
def collect(cfg):
|
|
572
|
+
warnings = []
|
|
573
|
+
if cfg['insecure_http']:
|
|
574
|
+
warnings.append('VAULT_ADDR uses plain http: the token and every secret cross the network unencrypted.')
|
|
575
|
+
me = lookup_self(cfg)
|
|
576
|
+
if me and not me['denied'] and 0 < me['ttl'] < TTL_WARN_SECONDS:
|
|
577
|
+
warnings.append('The token expires in %d day(s). Issue a new one before then.' % max(1, round(me['ttl'] / 86400.0)))
|
|
578
|
+
base_info = resolve_base(cfg, me)
|
|
579
|
+
results = []
|
|
580
|
+
for imp in cfg['imports']:
|
|
581
|
+
r = read_import(cfg, imp, base_info)
|
|
582
|
+
if r.get('missing'):
|
|
583
|
+
if not imp['optional']:
|
|
584
|
+
raise VaultEnvError('Not found: "%s" (%s).' % (imp['raw'], r['path']), 'not_found')
|
|
585
|
+
results.append({'raw': imp['raw'], 'path': r['path'], 'kv': r['kv'], 'skipped': True, 'entries': []})
|
|
586
|
+
continue
|
|
587
|
+
results.append({'raw': imp['raw'], 'path': r['path'], 'kv': r['kv'], 'version': r['version'],
|
|
588
|
+
'entries': map_entries(r['data'], imp, cfg['case_insensitive'])})
|
|
589
|
+
merged = merge_imports(results, cfg['case_insensitive'])
|
|
590
|
+
if merged['overridden']:
|
|
591
|
+
warnings.append('Set by more than one import (the later one wins): %s.' % ', '.join(merged['overridden']))
|
|
592
|
+
return {'self': me, 'base_info': base_info, 'results': results, 'values': merged['values'], 'warnings': warnings}
|
|
593
|
+
|
|
594
|
+
|
|
595
|
+
def apply(values, env, cfg):
|
|
596
|
+
p = plan(values, env, cfg['override'])
|
|
597
|
+
for name, value in p['set'].items():
|
|
598
|
+
env[name] = value
|
|
599
|
+
if p['kept']:
|
|
600
|
+
warn('Already set in the environment, kept as is (set VAULT_ENV_OVERRIDE=true to replace): %s.' % ', '.join(p['kept']))
|
|
601
|
+
env[LOADED_MARK] = '1'
|
|
602
|
+
if not cfg['keep_token']:
|
|
603
|
+
env.pop('VAULT_TOKEN', None)
|
|
604
|
+
if cfg['token_from_file']:
|
|
605
|
+
env.pop('VAULT_TOKEN_FILE', None)
|
|
606
|
+
return list(p['set'])
|
|
607
|
+
|
|
608
|
+
|
|
609
|
+
NOT_CONFIGURED = 'VAULT_ADDR and VAULT_ENV_IMPORT are not set; nothing loaded from Vault.'
|
|
610
|
+
|
|
611
|
+
|
|
612
|
+
# ---------------------------------------------------------------------------
|
|
613
|
+
# Library API
|
|
614
|
+
|
|
615
|
+
def fetch(env=None, **options):
|
|
616
|
+
"""Read the configured imports and return {NAME: value} without touching the environment.
|
|
617
|
+
|
|
618
|
+
Keyword options mirror the environment variables: addr, token, token_file, imports, base,
|
|
619
|
+
namespace, cacert, capath, timeout, retries, allow_http.
|
|
620
|
+
"""
|
|
621
|
+
cfg = read_config(options, os.environ if env is None else env)
|
|
622
|
+
if not cfg['configured']:
|
|
623
|
+
raise VaultEnvError(NOT_CONFIGURED, 'config')
|
|
624
|
+
result = collect(cfg)
|
|
625
|
+
for w in result['warnings']:
|
|
626
|
+
warn(w)
|
|
627
|
+
return result['values']
|
|
628
|
+
|
|
629
|
+
|
|
630
|
+
def load(env=None, **options):
|
|
631
|
+
"""Load into os.environ (or `env`). Returns the names that were set.
|
|
632
|
+
|
|
633
|
+
Call it before anything reads the configuration. Does nothing when VAULT_ENV_LOADED is
|
|
634
|
+
already set (for example by the vault-env-loader wrapper). Keyword options as for fetch(),
|
|
635
|
+
plus override and keep_token.
|
|
636
|
+
"""
|
|
637
|
+
env = os.environ if env is None else env
|
|
638
|
+
if env.get(LOADED_MARK):
|
|
639
|
+
return []
|
|
640
|
+
cfg = read_config(options, env)
|
|
641
|
+
if not cfg['configured']:
|
|
642
|
+
warn(NOT_CONFIGURED)
|
|
643
|
+
return []
|
|
644
|
+
result = collect(cfg)
|
|
645
|
+
for w in result['warnings']:
|
|
646
|
+
warn(w)
|
|
647
|
+
return apply(result['values'], env, cfg)
|
|
648
|
+
|
|
649
|
+
|
|
650
|
+
# ---------------------------------------------------------------------------
|
|
651
|
+
# CLI
|
|
652
|
+
|
|
653
|
+
HELP = """vault-env-loader %s - load Vault KV secrets into environment variables, then run a command.
|
|
654
|
+
|
|
655
|
+
Usage:
|
|
656
|
+
vault-env-loader [options] -- <command> [args...]
|
|
657
|
+
vault-env-loader check [options] show what would be loaded (names only, never values)
|
|
658
|
+
vault-env-loader --version
|
|
659
|
+
|
|
660
|
+
Options (each also has an environment variable):
|
|
661
|
+
--addr <url> VAULT_ADDR
|
|
662
|
+
--token-file <path> VAULT_TOKEN_FILE (the token itself only via VAULT_TOKEN)
|
|
663
|
+
--import <spec> VAULT_ENV_IMPORT repeatable; comma-separated in the variable
|
|
664
|
+
--base <path> VAULT_ENV_BASE base for relative imports
|
|
665
|
+
--override VAULT_ENV_OVERRIDE replace variables that are already set
|
|
666
|
+
--keep-token VAULT_ENV_KEEP_TOKEN pass VAULT_TOKEN on to the command
|
|
667
|
+
--timeout <seconds> VAULT_ENV_TIMEOUT per request, default 10
|
|
668
|
+
--retries <n> VAULT_ENV_RETRIES default 3
|
|
669
|
+
|
|
670
|
+
Import spec: [optional:]vault://<mount>/<path> | <relative path> | vault:// [?prefix=X_&name=auto|verbatim|upper&kv=1|2]
|
|
671
|
+
Docs: https://github.com/kaylovh/vault-env-loader
|
|
672
|
+
""" % __version__
|
|
673
|
+
|
|
674
|
+
|
|
675
|
+
def parse_args(argv):
|
|
676
|
+
opts = {}
|
|
677
|
+
imports = []
|
|
678
|
+
command = None
|
|
679
|
+
sub = None
|
|
680
|
+
i = 0
|
|
681
|
+
while i < len(argv):
|
|
682
|
+
a = argv[i]
|
|
683
|
+
|
|
684
|
+
def val():
|
|
685
|
+
if i + 1 >= len(argv):
|
|
686
|
+
raise VaultEnvError('%s needs a value.' % a, 'config')
|
|
687
|
+
return argv[i + 1]
|
|
688
|
+
|
|
689
|
+
if a == '--':
|
|
690
|
+
command = argv[i + 1:]
|
|
691
|
+
break
|
|
692
|
+
elif a in ('--addr', '--token-file', '--import', '--base', '--timeout', '--retries'):
|
|
693
|
+
v = val()
|
|
694
|
+
i += 1
|
|
695
|
+
if a == '--import':
|
|
696
|
+
imports.extend(split_list(v))
|
|
697
|
+
else:
|
|
698
|
+
opts[a[2:].replace('-', '_')] = v
|
|
699
|
+
elif a == '--override':
|
|
700
|
+
opts['override'] = True
|
|
701
|
+
elif a == '--keep-token':
|
|
702
|
+
opts['keep_token'] = True
|
|
703
|
+
elif a == '--token':
|
|
704
|
+
raise VaultEnvError('There is no --token option: command-line arguments are visible to every user (ps). '
|
|
705
|
+
'Use VAULT_TOKEN or --token-file.', 'config')
|
|
706
|
+
elif a == 'check' and sub is None:
|
|
707
|
+
sub = 'check'
|
|
708
|
+
elif a in ('--version', '-v'):
|
|
709
|
+
sub = 'version'
|
|
710
|
+
elif a in ('--help', '-h'):
|
|
711
|
+
sub = 'help'
|
|
712
|
+
elif not a.startswith('-') and sub is None:
|
|
713
|
+
# Allow "vault-env-loader gunicorn app:app" without "--".
|
|
714
|
+
command = argv[i:]
|
|
715
|
+
break
|
|
716
|
+
else:
|
|
717
|
+
raise VaultEnvError('Unknown option %s. See --help.' % a, 'config')
|
|
718
|
+
i += 1
|
|
719
|
+
if imports:
|
|
720
|
+
opts['imports'] = imports
|
|
721
|
+
return {'opts': opts, 'command': command, 'sub': sub}
|
|
722
|
+
|
|
723
|
+
|
|
724
|
+
def check(opts):
|
|
725
|
+
cfg = read_config(opts, os.environ)
|
|
726
|
+
if not cfg['configured']:
|
|
727
|
+
sys.stdout.write(NOT_CONFIGURED + '\n')
|
|
728
|
+
return 0
|
|
729
|
+
result = collect(cfg)
|
|
730
|
+
me = result['self']
|
|
731
|
+
base_info = result['base_info']
|
|
732
|
+
lines = ['vault-env-loader %s' % __version__, 'addr %s' % cfg['addr']]
|
|
733
|
+
if me and not me['denied']:
|
|
734
|
+
meta = ' '.join('%s=%s' % (k, v) for k, v in me['meta'].items())
|
|
735
|
+
expiry = 'expires in %dh' % round(me['ttl'] / 3600.0) if me['ttl'] else 'no expiry'
|
|
736
|
+
lines.append('token %s%s' % (expiry, ', ' + meta if meta else ''))
|
|
737
|
+
else:
|
|
738
|
+
lines.append('token (lookup-self not permitted)')
|
|
739
|
+
lines.append('base %s' % ('%s (from %s)' % (base_info['base'], base_info['source']) if base_info.get('base') else '(none)'))
|
|
740
|
+
kept = plan(result['values'], os.environ, cfg['override'])['kept']
|
|
741
|
+
|
|
742
|
+
def canon(n):
|
|
743
|
+
return n.upper() if cfg['case_insensitive'] else n
|
|
744
|
+
|
|
745
|
+
last_import = {}
|
|
746
|
+
for i, r in enumerate(result['results']):
|
|
747
|
+
for e in r['entries']:
|
|
748
|
+
last_import[canon(e['name'])] = i
|
|
749
|
+
for i, r in enumerate(result['results']):
|
|
750
|
+
lines.append('')
|
|
751
|
+
version = ', version %s' % r['version'] if r.get('version') is not None else ''
|
|
752
|
+
skipped = ' [optional, not found: skipped]' if r.get('skipped') else ''
|
|
753
|
+
lines.append('import %s -> %s (kv v%s%s)%s' % (r['raw'], r['path'], r['kv'], version, skipped))
|
|
754
|
+
width = max([len(e['name']) for e in r['entries']] or [0])
|
|
755
|
+
for e in r['entries']:
|
|
756
|
+
notes = []
|
|
757
|
+
if e['name'] != e['key']:
|
|
758
|
+
notes.append('from %s' % e['key'])
|
|
759
|
+
if last_import.get(canon(e['name'])) != i:
|
|
760
|
+
notes.append('overridden by a later import')
|
|
761
|
+
elif e['name'] in kept:
|
|
762
|
+
notes.append('already set: kept')
|
|
763
|
+
lines.append(' %s %5d chars%s' % (e['name'].ljust(width), len(e['value']), ' (%s)' % '; '.join(notes) if notes else ''))
|
|
764
|
+
for w in result['warnings']:
|
|
765
|
+
warn(w)
|
|
766
|
+
sys.stdout.write('\n'.join(lines) + '\n')
|
|
767
|
+
return 0
|
|
768
|
+
|
|
769
|
+
|
|
770
|
+
def run(opts, command):
|
|
771
|
+
env = dict(os.environ)
|
|
772
|
+
if not env.get(LOADED_MARK):
|
|
773
|
+
cfg = read_config(opts, env)
|
|
774
|
+
if not cfg['configured']:
|
|
775
|
+
warn(NOT_CONFIGURED)
|
|
776
|
+
else:
|
|
777
|
+
result = collect(cfg)
|
|
778
|
+
for w in result['warnings']:
|
|
779
|
+
warn(w)
|
|
780
|
+
apply(result['values'], env, cfg)
|
|
781
|
+
return execute(command, env)
|
|
782
|
+
|
|
783
|
+
|
|
784
|
+
def execute(command, env):
|
|
785
|
+
sys.stdout.flush()
|
|
786
|
+
sys.stderr.flush()
|
|
787
|
+
if os.name != 'nt':
|
|
788
|
+
# Replace this process: same PID, signals and exit status need no forwarding.
|
|
789
|
+
try:
|
|
790
|
+
os.execvpe(command[0], command, env)
|
|
791
|
+
except FileNotFoundError:
|
|
792
|
+
warn('Command not found: %s' % command[0])
|
|
793
|
+
return 127
|
|
794
|
+
except OSError as e:
|
|
795
|
+
warn('Could not start %s: %s' % (command[0], _errno_name(e)))
|
|
796
|
+
return 126
|
|
797
|
+
# Windows has no exec: run the command and pass its exit code through. Ctrl+C reaches
|
|
798
|
+
# the whole console group, so the child gets it too; we only wait for it.
|
|
799
|
+
try:
|
|
800
|
+
child = subprocess.Popen(command, env=env)
|
|
801
|
+
except FileNotFoundError:
|
|
802
|
+
warn('Command not found: %s' % command[0])
|
|
803
|
+
return 127
|
|
804
|
+
except OSError as e:
|
|
805
|
+
warn('Could not start %s: %s' % (command[0], _errno_name(e)))
|
|
806
|
+
return 126
|
|
807
|
+
while True:
|
|
808
|
+
try:
|
|
809
|
+
return child.wait()
|
|
810
|
+
except KeyboardInterrupt:
|
|
811
|
+
continue
|
|
812
|
+
|
|
813
|
+
|
|
814
|
+
def main(argv=None):
|
|
815
|
+
argv = sys.argv[1:] if argv is None else argv
|
|
816
|
+
try:
|
|
817
|
+
parsed = parse_args(argv)
|
|
818
|
+
if parsed['sub'] == 'version':
|
|
819
|
+
sys.stdout.write(__version__ + '\n')
|
|
820
|
+
return 0
|
|
821
|
+
if parsed['sub'] == 'help' or (not parsed['sub'] and not parsed['command']):
|
|
822
|
+
sys.stdout.write(HELP)
|
|
823
|
+
return 0 if parsed['sub'] == 'help' else 2
|
|
824
|
+
if parsed['sub'] == 'check':
|
|
825
|
+
return check(parsed['opts'])
|
|
826
|
+
if not parsed['command']:
|
|
827
|
+
raise VaultEnvError('No command given after --.', 'config')
|
|
828
|
+
return run(parsed['opts'], parsed['command'])
|
|
829
|
+
except VaultEnvError as e:
|
|
830
|
+
warn(e.message)
|
|
831
|
+
return EX_CONFIG
|
|
832
|
+
except KeyboardInterrupt:
|
|
833
|
+
return 130
|
|
834
|
+
except Exception as e: # never print a traceback that could carry request data
|
|
835
|
+
warn('Unexpected error: %s' % type(e).__name__)
|
|
836
|
+
return EX_CONFIG
|
|
837
|
+
|
|
838
|
+
|
|
839
|
+
def _cli():
|
|
840
|
+
sys.exit(main())
|
|
841
|
+
|
|
842
|
+
|
|
843
|
+
if __name__ == '__main__':
|
|
844
|
+
_cli()
|