flowbyte-secrets 1.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flowbyte_secrets/__init__.py +40 -0
- flowbyte_secrets/_version.py +1 -0
- flowbyte_secrets/client.py +363 -0
- flowbyte_secrets/dagster.py +73 -0
- flowbyte_secrets/env.py +58 -0
- flowbyte_secrets/errors.py +45 -0
- flowbyte_secrets-1.0.1.dist-info/METADATA +55 -0
- flowbyte_secrets-1.0.1.dist-info/RECORD +9 -0
- flowbyte_secrets-1.0.1.dist-info/WHEEL +4 -0
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
"""Flowbyte Secrets Python client.
|
|
2
|
+
|
|
3
|
+
>>> from flowbyte_secrets import SecretsClient
|
|
4
|
+
>>> c = SecretsClient(base_url="https://secrets.gmrlapp.com", token=os.environ["FBS_TOKEN"], project="dagster-prod")
|
|
5
|
+
>>> c.get("DB_PASSWORD")
|
|
6
|
+
|
|
7
|
+
The Dagster resource lives in :mod:`flowbyte_secrets.dagster` (``pip install flowbyte-secrets[dagster]``).
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from ._version import __version__
|
|
11
|
+
from .client import Project, Secret, SecretsClient, TokenScope, WhoAmI
|
|
12
|
+
from .env import load_into_env, to_env, to_env_name
|
|
13
|
+
from .errors import (
|
|
14
|
+
SecretNotFound,
|
|
15
|
+
SecretsAuthError,
|
|
16
|
+
SecretsConfigError,
|
|
17
|
+
SecretsConnectionError,
|
|
18
|
+
SecretsError,
|
|
19
|
+
SecretsForbiddenError,
|
|
20
|
+
SecretsRateLimitError,
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
__all__ = [
|
|
24
|
+
"__version__",
|
|
25
|
+
"SecretsClient",
|
|
26
|
+
"Secret",
|
|
27
|
+
"Project",
|
|
28
|
+
"TokenScope",
|
|
29
|
+
"WhoAmI",
|
|
30
|
+
"load_into_env",
|
|
31
|
+
"to_env",
|
|
32
|
+
"to_env_name",
|
|
33
|
+
"SecretsError",
|
|
34
|
+
"SecretsConfigError",
|
|
35
|
+
"SecretsConnectionError",
|
|
36
|
+
"SecretsAuthError",
|
|
37
|
+
"SecretsForbiddenError",
|
|
38
|
+
"SecretNotFound",
|
|
39
|
+
"SecretsRateLimitError",
|
|
40
|
+
]
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "1.0.1"
|
|
@@ -0,0 +1,363 @@
|
|
|
1
|
+
"""HTTP client for the Flowbyte Secrets machine API (``/api/v1/*``)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
import random
|
|
7
|
+
import time
|
|
8
|
+
from dataclasses import dataclass, field
|
|
9
|
+
from datetime import datetime
|
|
10
|
+
from email.utils import parsedate_to_datetime
|
|
11
|
+
from typing import Any, Callable, Dict, List, MutableMapping, Optional
|
|
12
|
+
from urllib.parse import quote, urlsplit
|
|
13
|
+
|
|
14
|
+
import httpx
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _is_loopback(url: str) -> bool:
|
|
18
|
+
host = (urlsplit(url).hostname or "").lower()
|
|
19
|
+
return host in ("localhost", "::1") or host.startswith("127.")
|
|
20
|
+
|
|
21
|
+
from ._version import __version__
|
|
22
|
+
from .errors import (
|
|
23
|
+
SecretNotFound,
|
|
24
|
+
SecretsAuthError,
|
|
25
|
+
SecretsConfigError,
|
|
26
|
+
SecretsConnectionError,
|
|
27
|
+
SecretsError,
|
|
28
|
+
SecretsForbiddenError,
|
|
29
|
+
SecretsRateLimitError,
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
USER_AGENT = f"flowbyte-secrets-python/{__version__}"
|
|
33
|
+
ENV_URL = "FBS_URL"
|
|
34
|
+
ENV_TOKEN = "FBS_TOKEN"
|
|
35
|
+
ENV_PROJECT = "FBS_PROJECT"
|
|
36
|
+
|
|
37
|
+
_LEVELS = {1: "Read", 2: "ReadWrite", 3: "Manage"}
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _level(value: Any) -> str:
|
|
41
|
+
if isinstance(value, int):
|
|
42
|
+
return _LEVELS.get(value, str(value))
|
|
43
|
+
return "" if value is None else str(value)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _parse_dt(value: Any) -> Optional[datetime]:
|
|
47
|
+
if not value:
|
|
48
|
+
return None
|
|
49
|
+
try:
|
|
50
|
+
return datetime.fromisoformat(str(value).replace("Z", "+00:00"))
|
|
51
|
+
except ValueError:
|
|
52
|
+
return None
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _json(response: httpx.Response) -> Any:
|
|
56
|
+
try:
|
|
57
|
+
return response.json()
|
|
58
|
+
except ValueError:
|
|
59
|
+
# Never include the payload (it may contain secret values).
|
|
60
|
+
raise SecretsError("The server returned a response that could not be parsed.", response.status_code) from None
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
@dataclass(frozen=True)
|
|
64
|
+
class Secret:
|
|
65
|
+
key: str
|
|
66
|
+
value: str = field(repr=False) # never shown in repr/logs
|
|
67
|
+
version: int = 0
|
|
68
|
+
modified_on: Optional[datetime] = None
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
@dataclass(frozen=True)
|
|
72
|
+
class Project:
|
|
73
|
+
id: str
|
|
74
|
+
slug: str
|
|
75
|
+
name: str
|
|
76
|
+
level: str
|
|
77
|
+
|
|
78
|
+
@property
|
|
79
|
+
def can_write(self) -> bool:
|
|
80
|
+
return self.level in ("ReadWrite", "Manage")
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
@dataclass(frozen=True)
|
|
84
|
+
class TokenScope:
|
|
85
|
+
project_id: str
|
|
86
|
+
level: str
|
|
87
|
+
slug: Optional[str] = None
|
|
88
|
+
name: Optional[str] = None
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
@dataclass(frozen=True)
|
|
92
|
+
class WhoAmI:
|
|
93
|
+
machine_account: str
|
|
94
|
+
token_name: str
|
|
95
|
+
company_id: str
|
|
96
|
+
scopes: List[TokenScope]
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
class SecretsClient:
|
|
100
|
+
"""Client for the Flowbyte Secrets machine API.
|
|
101
|
+
|
|
102
|
+
``base_url``, ``token`` and ``project`` fall back to ``FBS_URL``, ``FBS_TOKEN`` and ``FBS_PROJECT``.
|
|
103
|
+
Requests are retried (``max_retries``, exponential backoff) on connection errors, 5xx and 429
|
|
104
|
+
(honouring ``Retry-After``). Writes are only retried when the request cannot have been applied
|
|
105
|
+
(connection refused, 429, 503).
|
|
106
|
+
"""
|
|
107
|
+
|
|
108
|
+
def __init__(
|
|
109
|
+
self,
|
|
110
|
+
base_url: Optional[str] = None,
|
|
111
|
+
token: Optional[str] = None,
|
|
112
|
+
project: Optional[str] = None,
|
|
113
|
+
*,
|
|
114
|
+
timeout: float = 10.0,
|
|
115
|
+
max_retries: int = 3,
|
|
116
|
+
backoff_factor: float = 0.5,
|
|
117
|
+
max_retry_delay: float = 30.0,
|
|
118
|
+
transport: Optional[httpx.BaseTransport] = None,
|
|
119
|
+
http_client: Optional[httpx.Client] = None,
|
|
120
|
+
) -> None:
|
|
121
|
+
base_url = (base_url or os.environ.get(ENV_URL) or "").strip()
|
|
122
|
+
token = (token or os.environ.get(ENV_TOKEN) or "").strip()
|
|
123
|
+
project = (project or os.environ.get(ENV_PROJECT) or "").strip() or None
|
|
124
|
+
|
|
125
|
+
if not base_url:
|
|
126
|
+
raise SecretsConfigError(f"base_url is required (or set {ENV_URL}).")
|
|
127
|
+
if not base_url.startswith(("https://", "http://")):
|
|
128
|
+
raise SecretsConfigError("base_url must be an absolute http(s) URL.")
|
|
129
|
+
if base_url.startswith("http://") and not _is_loopback(base_url):
|
|
130
|
+
# The token is a bearer credential: plain http is only acceptable on this machine.
|
|
131
|
+
raise SecretsConfigError("base_url must use https (http is only allowed for localhost).")
|
|
132
|
+
if not token:
|
|
133
|
+
raise SecretsConfigError(f"token is required (or set {ENV_TOKEN}).")
|
|
134
|
+
if not token.startswith("fbs_") or any(c.isspace() for c in token):
|
|
135
|
+
raise SecretsConfigError("token is not a Flowbyte Secrets access token (expected 'fbs_...').")
|
|
136
|
+
|
|
137
|
+
self.base_url = base_url.rstrip("/") + "/"
|
|
138
|
+
self.project = project
|
|
139
|
+
self.timeout = timeout
|
|
140
|
+
self.max_retries = max(0, int(max_retries))
|
|
141
|
+
self.backoff_factor = backoff_factor
|
|
142
|
+
self.max_retry_delay = max_retry_delay
|
|
143
|
+
self._token = token
|
|
144
|
+
self._sleep: Callable[[float], None] = time.sleep
|
|
145
|
+
self._owns_client = http_client is None
|
|
146
|
+
self._http = http_client or httpx.Client(timeout=timeout, transport=transport)
|
|
147
|
+
|
|
148
|
+
# ------------------------------------------------------------------ public API
|
|
149
|
+
|
|
150
|
+
def projects(self) -> List[Project]:
|
|
151
|
+
"""Projects the token is scoped to."""
|
|
152
|
+
data = _json(self._request("GET", "api/v1/projects", project=None, idempotent=True))
|
|
153
|
+
return [
|
|
154
|
+
Project(id=str(p.get("id", "")), slug=p.get("slug", ""), name=p.get("name", ""), level=_level(p.get("level")))
|
|
155
|
+
for p in data
|
|
156
|
+
]
|
|
157
|
+
|
|
158
|
+
def list(self, project: Optional[str] = None) -> List[Secret]:
|
|
159
|
+
"""All secrets (with values) of ``project`` / the default project / the token's only project."""
|
|
160
|
+
project = self._project(project)
|
|
161
|
+
data = _json(self._request("GET", "api/v1/secrets", project=project, idempotent=True))
|
|
162
|
+
return [
|
|
163
|
+
Secret(
|
|
164
|
+
key=s.get("key", ""),
|
|
165
|
+
value=s.get("value", ""),
|
|
166
|
+
version=int(s.get("version") or 0),
|
|
167
|
+
modified_on=_parse_dt(s.get("modifiedOn")),
|
|
168
|
+
)
|
|
169
|
+
for s in data
|
|
170
|
+
]
|
|
171
|
+
|
|
172
|
+
def as_dict(self, project: Optional[str] = None) -> Dict[str, str]:
|
|
173
|
+
"""``{key: value}`` for all secrets of the project."""
|
|
174
|
+
return {s.key: s.value for s in self.list(project)}
|
|
175
|
+
|
|
176
|
+
def get_secret(self, key: str, project: Optional[str] = None) -> Secret:
|
|
177
|
+
"""One secret with its version. Raises :class:`SecretNotFound`."""
|
|
178
|
+
self._check_key(key)
|
|
179
|
+
project = self._project(project)
|
|
180
|
+
s = _json(self._request("GET", self._secret_path(key), project=project, idempotent=True, key=key))
|
|
181
|
+
return Secret(key=s.get("key", key), value=s.get("value", ""), version=int(s.get("version") or 0))
|
|
182
|
+
|
|
183
|
+
def get(self, key: str, project: Optional[str] = None) -> str:
|
|
184
|
+
"""One secret's value. Raises :class:`SecretNotFound`."""
|
|
185
|
+
return self.get_secret(key, project).value
|
|
186
|
+
|
|
187
|
+
def get_or_default(self, key: str, default: Optional[str] = None, project: Optional[str] = None) -> Optional[str]:
|
|
188
|
+
"""One secret's value, or ``default`` when it does not exist."""
|
|
189
|
+
try:
|
|
190
|
+
return self.get(key, project)
|
|
191
|
+
except SecretNotFound:
|
|
192
|
+
return default
|
|
193
|
+
|
|
194
|
+
def set(self, key: str, value: str, note: Optional[str] = None, project: Optional[str] = None) -> None:
|
|
195
|
+
"""Create or update a secret (ReadWrite token)."""
|
|
196
|
+
self._check_key(key)
|
|
197
|
+
if value is None:
|
|
198
|
+
raise ValueError("value is required")
|
|
199
|
+
body: Dict[str, Any] = {"value": value}
|
|
200
|
+
if note is not None:
|
|
201
|
+
body["note"] = note
|
|
202
|
+
self._request("PUT", self._secret_path(key), project=self._project(project), idempotent=False, key=key, json=body)
|
|
203
|
+
|
|
204
|
+
def delete(self, key: str, project: Optional[str] = None) -> None:
|
|
205
|
+
"""Delete a secret (ReadWrite token). Raises :class:`SecretNotFound`."""
|
|
206
|
+
self._check_key(key)
|
|
207
|
+
self._request("DELETE", self._secret_path(key), project=self._project(project), idempotent=False, key=key)
|
|
208
|
+
|
|
209
|
+
def whoami(self) -> WhoAmI:
|
|
210
|
+
"""Machine account, token name, company and scopes of the token."""
|
|
211
|
+
d = _json(self._request("GET", "api/v1/whoami", project=None, idempotent=True))
|
|
212
|
+
scopes: List[TokenScope] = []
|
|
213
|
+
for s in d.get("scopes") or []:
|
|
214
|
+
if isinstance(s, str):
|
|
215
|
+
pid, _, lvl = s.rpartition(":")
|
|
216
|
+
scopes.append(TokenScope(project_id=pid or s, level=lvl if pid else ""))
|
|
217
|
+
elif isinstance(s, dict):
|
|
218
|
+
scopes.append(
|
|
219
|
+
TokenScope(
|
|
220
|
+
project_id=str(s.get("projectId") or s.get("id") or ""),
|
|
221
|
+
level=_level(s.get("level")),
|
|
222
|
+
slug=s.get("slug"),
|
|
223
|
+
name=s.get("name"),
|
|
224
|
+
)
|
|
225
|
+
)
|
|
226
|
+
return WhoAmI(
|
|
227
|
+
machine_account=d.get("machineAccount", ""),
|
|
228
|
+
token_name=d.get("tokenName", ""),
|
|
229
|
+
company_id=d.get("companyId", ""),
|
|
230
|
+
scopes=scopes,
|
|
231
|
+
)
|
|
232
|
+
|
|
233
|
+
def load_into_env(
|
|
234
|
+
self,
|
|
235
|
+
prefix: str = "",
|
|
236
|
+
project: Optional[str] = None,
|
|
237
|
+
*,
|
|
238
|
+
override: bool = True,
|
|
239
|
+
map_dots: bool = False,
|
|
240
|
+
environ: Optional[MutableMapping[str, str]] = None,
|
|
241
|
+
) -> List[str]:
|
|
242
|
+
"""Inject all secrets into ``os.environ`` (or ``environ``). Returns the variable names set."""
|
|
243
|
+
from .env import load_into_env
|
|
244
|
+
|
|
245
|
+
return load_into_env(self.list(project), prefix=prefix, override=override, map_dots=map_dots, environ=environ)
|
|
246
|
+
|
|
247
|
+
def close(self) -> None:
|
|
248
|
+
if self._owns_client:
|
|
249
|
+
self._http.close()
|
|
250
|
+
|
|
251
|
+
def __enter__(self) -> "SecretsClient":
|
|
252
|
+
return self
|
|
253
|
+
|
|
254
|
+
def __exit__(self, *exc: Any) -> None:
|
|
255
|
+
self.close()
|
|
256
|
+
|
|
257
|
+
def __repr__(self) -> str:
|
|
258
|
+
return f"SecretsClient(base_url={self.base_url!r}, project={self.project!r}, token=<redacted>)"
|
|
259
|
+
|
|
260
|
+
# ------------------------------------------------------------------ internals
|
|
261
|
+
|
|
262
|
+
def _project(self, project: Optional[str]) -> Optional[str]:
|
|
263
|
+
return (project or "").strip() or self.project
|
|
264
|
+
|
|
265
|
+
@staticmethod
|
|
266
|
+
def _check_key(key: str) -> None:
|
|
267
|
+
if not key or not key.strip():
|
|
268
|
+
raise ValueError("key is required")
|
|
269
|
+
|
|
270
|
+
@staticmethod
|
|
271
|
+
def _secret_path(key: str) -> str:
|
|
272
|
+
return "api/v1/secrets/" + quote(key, safe="")
|
|
273
|
+
|
|
274
|
+
def _backoff(self, attempt: int) -> float:
|
|
275
|
+
return min(self.max_retry_delay, self.backoff_factor * (2**attempt) + random.uniform(0, 0.1))
|
|
276
|
+
|
|
277
|
+
@staticmethod
|
|
278
|
+
def _retry_after(response: httpx.Response) -> Optional[float]:
|
|
279
|
+
value = response.headers.get("Retry-After")
|
|
280
|
+
if not value:
|
|
281
|
+
return None
|
|
282
|
+
try:
|
|
283
|
+
return max(0.0, float(value))
|
|
284
|
+
except ValueError:
|
|
285
|
+
pass
|
|
286
|
+
try:
|
|
287
|
+
when = parsedate_to_datetime(value)
|
|
288
|
+
return max(0.0, when.timestamp() - time.time())
|
|
289
|
+
except (TypeError, ValueError):
|
|
290
|
+
return None
|
|
291
|
+
|
|
292
|
+
def _request(
|
|
293
|
+
self,
|
|
294
|
+
method: str,
|
|
295
|
+
path: str,
|
|
296
|
+
*,
|
|
297
|
+
project: Optional[str],
|
|
298
|
+
idempotent: bool,
|
|
299
|
+
key: Optional[str] = None,
|
|
300
|
+
json: Optional[Dict[str, Any]] = None,
|
|
301
|
+
) -> httpx.Response:
|
|
302
|
+
url = self.base_url + path
|
|
303
|
+
params = {"project": project} if project else None
|
|
304
|
+
headers = {
|
|
305
|
+
"Authorization": f"Bearer {self._token}",
|
|
306
|
+
"Accept": "application/json",
|
|
307
|
+
"User-Agent": USER_AGENT,
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
attempt = 0
|
|
311
|
+
while True:
|
|
312
|
+
try:
|
|
313
|
+
response = self._http.request(method, url, params=params, headers=headers, json=json, timeout=self.timeout)
|
|
314
|
+
except (httpx.ConnectError, httpx.ConnectTimeout) as exc:
|
|
315
|
+
# Request never reached the server: safe to retry for any method.
|
|
316
|
+
if attempt < self.max_retries:
|
|
317
|
+
self._sleep(self._backoff(attempt))
|
|
318
|
+
attempt += 1
|
|
319
|
+
continue
|
|
320
|
+
raise SecretsConnectionError(f"Could not reach {self.base_url}: {type(exc).__name__}") from None
|
|
321
|
+
except httpx.TransportError as exc: # read timeouts, protocol errors, ...
|
|
322
|
+
if idempotent and attempt < self.max_retries:
|
|
323
|
+
self._sleep(self._backoff(attempt))
|
|
324
|
+
attempt += 1
|
|
325
|
+
continue
|
|
326
|
+
raise SecretsConnectionError(f"Request to {self.base_url} failed: {type(exc).__name__}") from None
|
|
327
|
+
|
|
328
|
+
status = response.status_code
|
|
329
|
+
retryable = status in (429, 503) or (idempotent and status >= 500)
|
|
330
|
+
if retryable and attempt < self.max_retries:
|
|
331
|
+
delay = self._retry_after(response) if status in (429, 503) else None
|
|
332
|
+
self._sleep(min(self.max_retry_delay, delay if delay is not None else self._backoff(attempt)))
|
|
333
|
+
attempt += 1
|
|
334
|
+
continue
|
|
335
|
+
|
|
336
|
+
if status < 400:
|
|
337
|
+
return response
|
|
338
|
+
self._raise_for_status(response, key, project)
|
|
339
|
+
|
|
340
|
+
@staticmethod
|
|
341
|
+
def _raise_for_status(response: httpx.Response, key: Optional[str], project: Optional[str]) -> None:
|
|
342
|
+
status = response.status_code
|
|
343
|
+
where = f"project '{project}'" if project else "the token's project"
|
|
344
|
+
if status == 401:
|
|
345
|
+
raise SecretsAuthError("Unauthorized (401): the access token is invalid, expired or revoked.", 401)
|
|
346
|
+
if status == 403:
|
|
347
|
+
raise SecretsForbiddenError(f"Forbidden (403): the token is not allowed to perform this operation on {where}.", 403)
|
|
348
|
+
if status == 404:
|
|
349
|
+
if key is None:
|
|
350
|
+
raise SecretNotFound(f"Not found (404): {where} does not exist or is not in the token's scope.")
|
|
351
|
+
raise SecretNotFound(f"Not found (404): secret '{key}' does not exist in {where}.", key=key)
|
|
352
|
+
if status == 429:
|
|
353
|
+
raise SecretsRateLimitError("Rate limited (429): too many requests; retries exhausted.",
|
|
354
|
+
SecretsClient._retry_after(response))
|
|
355
|
+
# Only the ProblemDetails title is used: the body/detail could echo a submitted value.
|
|
356
|
+
title = None
|
|
357
|
+
try:
|
|
358
|
+
body = response.json()
|
|
359
|
+
if isinstance(body, dict) and isinstance(body.get("title"), str):
|
|
360
|
+
title = body["title"][:200]
|
|
361
|
+
except ValueError:
|
|
362
|
+
pass
|
|
363
|
+
raise SecretsError(f"The Flowbyte Secrets server returned {status} ({title or response.reason_phrase}).", status)
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
"""Dagster resource (requires ``pip install flowbyte-secrets[dagster]``).
|
|
2
|
+
|
|
3
|
+
.. code-block:: python
|
|
4
|
+
|
|
5
|
+
from dagster import Definitions, EnvVar, asset
|
|
6
|
+
from flowbyte_secrets.dagster import FlowbyteSecretsResource
|
|
7
|
+
|
|
8
|
+
@asset
|
|
9
|
+
def my_asset(secrets: FlowbyteSecretsResource):
|
|
10
|
+
pw = secrets.get("DB_PASSWORD")
|
|
11
|
+
|
|
12
|
+
defs = Definitions(
|
|
13
|
+
assets=[my_asset],
|
|
14
|
+
resources={"secrets": FlowbyteSecretsResource(base_url=EnvVar("FBS_URL"), token=EnvVar("FBS_TOKEN"), project="dagster-prod")},
|
|
15
|
+
)
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
from typing import Dict, List, Optional
|
|
21
|
+
|
|
22
|
+
try:
|
|
23
|
+
from dagster import ConfigurableResource
|
|
24
|
+
from pydantic import Field, PrivateAttr
|
|
25
|
+
except ImportError as exc: # pragma: no cover - exercised only without the extra
|
|
26
|
+
raise ImportError(
|
|
27
|
+
"flowbyte_secrets.dagster requires Dagster. Install with: pip install 'flowbyte-secrets[dagster]'"
|
|
28
|
+
) from exc
|
|
29
|
+
|
|
30
|
+
from .client import Secret, SecretsClient
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class FlowbyteSecretsResource(ConfigurableResource):
|
|
34
|
+
"""Reads secrets from Flowbyte Secrets with a machine access token."""
|
|
35
|
+
|
|
36
|
+
base_url: str = Field(description="Server URL, e.g. https://secrets.gmrlapp.com (use EnvVar('FBS_URL')).")
|
|
37
|
+
token: str = Field(description="Access token fbs_... (use EnvVar('FBS_TOKEN'); never hard-code).")
|
|
38
|
+
project: Optional[str] = Field(default=None, description="Project slug or id; optional for single-project tokens.")
|
|
39
|
+
timeout: float = Field(default=10.0, description="HTTP timeout in seconds.")
|
|
40
|
+
max_retries: int = Field(default=3, description="Retries on connection errors, 5xx and 429.")
|
|
41
|
+
|
|
42
|
+
_client: Optional[SecretsClient] = PrivateAttr(default=None)
|
|
43
|
+
|
|
44
|
+
def _get_client(self) -> SecretsClient:
|
|
45
|
+
if self._client is None:
|
|
46
|
+
self._client = SecretsClient(
|
|
47
|
+
base_url=self.base_url,
|
|
48
|
+
token=self.token,
|
|
49
|
+
project=self.project,
|
|
50
|
+
timeout=self.timeout,
|
|
51
|
+
max_retries=self.max_retries,
|
|
52
|
+
)
|
|
53
|
+
return self._client
|
|
54
|
+
|
|
55
|
+
def teardown_after_execution(self, context) -> None: # type: ignore[override]
|
|
56
|
+
if self._client is not None:
|
|
57
|
+
self._client.close()
|
|
58
|
+
self._client = None
|
|
59
|
+
|
|
60
|
+
def get(self, key: str, project: Optional[str] = None) -> str:
|
|
61
|
+
return self._get_client().get(key, project)
|
|
62
|
+
|
|
63
|
+
def get_or_default(self, key: str, default: Optional[str] = None, project: Optional[str] = None) -> Optional[str]:
|
|
64
|
+
return self._get_client().get_or_default(key, default, project)
|
|
65
|
+
|
|
66
|
+
def list(self, project: Optional[str] = None) -> List[Secret]:
|
|
67
|
+
return self._get_client().list(project)
|
|
68
|
+
|
|
69
|
+
def as_dict(self, project: Optional[str] = None) -> Dict[str, str]:
|
|
70
|
+
return self._get_client().as_dict(project)
|
|
71
|
+
|
|
72
|
+
def load_into_env(self, prefix: str = "", project: Optional[str] = None, *, override: bool = True, map_dots: bool = False) -> List[str]:
|
|
73
|
+
return self._get_client().load_into_env(prefix, project, override=override, map_dots=map_dots)
|
flowbyte_secrets/env.py
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
"""Environment-variable helpers (same mapping rules as ``fbs run``)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
import re
|
|
7
|
+
import warnings
|
|
8
|
+
from typing import Dict, Iterable, List, MutableMapping, Optional
|
|
9
|
+
|
|
10
|
+
from .client import Secret
|
|
11
|
+
|
|
12
|
+
_INVALID = re.compile(r"[^A-Za-z0-9_]")
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def to_env_name(key: str, prefix: str = "", map_dots: bool = False) -> str:
|
|
16
|
+
"""Map a secret key to an environment variable name.
|
|
17
|
+
|
|
18
|
+
With ``map_dots``, ``.`` becomes ``__`` (``Database.Password`` -> ``Database__Password``);
|
|
19
|
+
every other character outside ``[A-Za-z0-9_]`` (including ``.`` and ``-``) becomes ``_``.
|
|
20
|
+
``prefix`` is prepended verbatim; case is preserved.
|
|
21
|
+
"""
|
|
22
|
+
name = key.replace(".", "__") if map_dots else key
|
|
23
|
+
name = (prefix or "") + _INVALID.sub("_", name)
|
|
24
|
+
if name[:1].isdigit():
|
|
25
|
+
name = "_" + name
|
|
26
|
+
return name
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def to_env(secrets: Iterable[Secret], prefix: str = "", map_dots: bool = False) -> Dict[str, str]:
|
|
30
|
+
"""``{ENV_NAME: value}``; on name collisions the later key wins (a warning names the keys, never values)."""
|
|
31
|
+
result: Dict[str, str] = {}
|
|
32
|
+
origin: Dict[str, str] = {}
|
|
33
|
+
for s in secrets:
|
|
34
|
+
name = to_env_name(s.key, prefix, map_dots)
|
|
35
|
+
if name in origin:
|
|
36
|
+
warnings.warn(f"secret keys '{origin[name]}' and '{s.key}' both map to {name}; using '{s.key}'", stacklevel=2)
|
|
37
|
+
result[name] = s.value
|
|
38
|
+
origin[name] = s.key
|
|
39
|
+
return result
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def load_into_env(
|
|
43
|
+
secrets: Iterable[Secret],
|
|
44
|
+
prefix: str = "",
|
|
45
|
+
*,
|
|
46
|
+
override: bool = True,
|
|
47
|
+
map_dots: bool = False,
|
|
48
|
+
environ: Optional[MutableMapping[str, str]] = None,
|
|
49
|
+
) -> List[str]:
|
|
50
|
+
"""Set the secrets in ``os.environ`` (or ``environ``). Returns the names that were set."""
|
|
51
|
+
target = os.environ if environ is None else environ
|
|
52
|
+
names: List[str] = []
|
|
53
|
+
for name, value in to_env(secrets, prefix, map_dots).items():
|
|
54
|
+
if not override and name in target:
|
|
55
|
+
continue
|
|
56
|
+
target[name] = value
|
|
57
|
+
names.append(name)
|
|
58
|
+
return names
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
"""Typed errors. Messages never contain secret values or the access token."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Optional
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class SecretsError(Exception):
|
|
9
|
+
"""Base class for Flowbyte Secrets errors."""
|
|
10
|
+
|
|
11
|
+
def __init__(self, message: str, status_code: Optional[int] = None) -> None:
|
|
12
|
+
super().__init__(message)
|
|
13
|
+
self.status_code = status_code
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class SecretsConfigError(SecretsError):
|
|
17
|
+
"""Missing or malformed client settings (base URL, token)."""
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class SecretsConnectionError(SecretsError):
|
|
21
|
+
"""The server could not be reached (after retries) or timed out."""
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class SecretsAuthError(SecretsError):
|
|
25
|
+
"""401 (bad/expired/revoked token) or 403 (token not scoped for this project/operation)."""
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class SecretsForbiddenError(SecretsAuthError):
|
|
29
|
+
"""403: the token is valid but lacks the scope (e.g. a Read token writing)."""
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class SecretNotFound(SecretsError):
|
|
33
|
+
"""404: the secret or project does not exist (or is outside the token's scope)."""
|
|
34
|
+
|
|
35
|
+
def __init__(self, message: str, key: Optional[str] = None) -> None:
|
|
36
|
+
super().__init__(message, 404)
|
|
37
|
+
self.key = key
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class SecretsRateLimitError(SecretsError):
|
|
41
|
+
"""429 after retries were exhausted."""
|
|
42
|
+
|
|
43
|
+
def __init__(self, message: str, retry_after: Optional[float] = None) -> None:
|
|
44
|
+
super().__init__(message, 429)
|
|
45
|
+
self.retry_after = retry_after
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: flowbyte-secrets
|
|
3
|
+
Version: 1.0.1
|
|
4
|
+
Summary: Python client for the Flowbyte Secrets machine API (fbs_ access tokens), with an optional Dagster resource.
|
|
5
|
+
Author: Flowbyte
|
|
6
|
+
License: Proprietary
|
|
7
|
+
Keywords: configuration,dagster,flowbyte,secrets
|
|
8
|
+
Classifier: Programming Language :: Python :: 3
|
|
9
|
+
Classifier: Typing :: Typed
|
|
10
|
+
Requires-Python: >=3.9
|
|
11
|
+
Requires-Dist: httpx>=0.27
|
|
12
|
+
Provides-Extra: dagster
|
|
13
|
+
Requires-Dist: dagster>=1.7; extra == 'dagster'
|
|
14
|
+
Provides-Extra: dev
|
|
15
|
+
Requires-Dist: pytest>=8; extra == 'dev'
|
|
16
|
+
Requires-Dist: respx>=0.21; extra == 'dev'
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
|
|
19
|
+
# flowbyte-secrets
|
|
20
|
+
|
|
21
|
+
Python client for the Flowbyte Secrets machine API (`fbs_` access tokens), with an optional Dagster resource.
|
|
22
|
+
|
|
23
|
+
```bash
|
|
24
|
+
pip install flowbyte-secrets # or: pip install "flowbyte-secrets[dagster]"
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
```python
|
|
28
|
+
import os
|
|
29
|
+
from flowbyte_secrets import SecretsClient, SecretNotFound
|
|
30
|
+
|
|
31
|
+
c = SecretsClient(base_url="https://secrets.gmrlapp.com", token=os.environ["FBS_TOKEN"], project="dagster-prod")
|
|
32
|
+
c.get("DB_PASSWORD") # str, raises SecretNotFound
|
|
33
|
+
c.list() # [Secret(key, value, version, modified_on)]
|
|
34
|
+
c.set("K", "v") # ReadWrite tokens only
|
|
35
|
+
c.load_into_env(prefix="") # os.environ injection
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
`base_url`, `token` and `project` fall back to `FBS_URL`, `FBS_TOKEN` and `FBS_PROJECT`.
|
|
39
|
+
|
|
40
|
+
Dagster:
|
|
41
|
+
|
|
42
|
+
```python
|
|
43
|
+
from dagster import Definitions, EnvVar, asset
|
|
44
|
+
from flowbyte_secrets.dagster import FlowbyteSecretsResource
|
|
45
|
+
|
|
46
|
+
@asset
|
|
47
|
+
def my_asset(secrets: FlowbyteSecretsResource):
|
|
48
|
+
pw = secrets.get("DB_PASSWORD")
|
|
49
|
+
|
|
50
|
+
defs = Definitions(assets=[my_asset], resources={
|
|
51
|
+
"secrets": FlowbyteSecretsResource(base_url=EnvVar("FBS_URL"), token=EnvVar("FBS_TOKEN"), project="dagster-prod"),
|
|
52
|
+
})
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
See `docs/CLIENTS.md` in the repository for details.
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
flowbyte_secrets/__init__.py,sha256=wWPAxjasA4Vq-KWfINHu_0Fz3_qvkupLhxSTGxcj6Yc,1075
|
|
2
|
+
flowbyte_secrets/_version.py,sha256=d4QHYmS_30j0hPN8NmNPnQ_Z0TphDRbu4MtQj9cT9e8,22
|
|
3
|
+
flowbyte_secrets/client.py,sha256=HacRzXFMK0UIo_Sjmk5LX7iaRatEPx-svisPtIgrsfo,14426
|
|
4
|
+
flowbyte_secrets/dagster.py,sha256=9dvEmTn3Dt7LwRFkqzecwHWehRHfU-SVlR0wrKO13YM,3082
|
|
5
|
+
flowbyte_secrets/env.py,sha256=O0oZsFDp8paP-tL6zo6zFExHoBDo3XriTFm4v7MyYwk,2073
|
|
6
|
+
flowbyte_secrets/errors.py,sha256=aPUbrtREMbdZBqqqSuA2NjlGI4iAAhgrQz5wr5e-u0E,1432
|
|
7
|
+
flowbyte_secrets-1.0.1.dist-info/METADATA,sha256=uDYcs9jYcLP2jxaanQbNCKRC0-nZYDG_vdXWBhgdJDw,1821
|
|
8
|
+
flowbyte_secrets-1.0.1.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
|
|
9
|
+
flowbyte_secrets-1.0.1.dist-info/RECORD,,
|