velocity-python 0.13.0__py3-none-any.whl → 0.13.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- velocity/__init__.py +1 -1
- velocity/aws/ssm_config.py +191 -16
- {velocity_python-0.13.0.dist-info → velocity_python-0.13.2.dist-info}/METADATA +1 -1
- {velocity_python-0.13.0.dist-info → velocity_python-0.13.2.dist-info}/RECORD +8 -8
- {velocity_python-0.13.0.dist-info → velocity_python-0.13.2.dist-info}/WHEEL +0 -0
- {velocity_python-0.13.0.dist-info → velocity_python-0.13.2.dist-info}/entry_points.txt +0 -0
- {velocity_python-0.13.0.dist-info → velocity_python-0.13.2.dist-info}/licenses/LICENSE +0 -0
- {velocity_python-0.13.0.dist-info → velocity_python-0.13.2.dist-info}/top_level.txt +0 -0
velocity/__init__.py
CHANGED
velocity/aws/ssm_config.py
CHANGED
|
@@ -15,7 +15,16 @@ when they can resolve both project and AppStage from the runtime environment.
|
|
|
15
15
|
|
|
16
16
|
On an SSM miss or any other SSM error, ``getenv`` falls back to ``os.environ``.
|
|
17
17
|
Results are cached for the process lifetime so SSM is called at most once per
|
|
18
|
-
key per Lambda cold start
|
|
18
|
+
key per Lambda cold start; a transient AWS failure (throttle, timeout, 5xx) is
|
|
19
|
+
not cached. One client serves the process (``ssm_client``), and ``prefetch``
|
|
20
|
+
fills the cache for many keys in batched ``GetParameters`` calls.
|
|
21
|
+
|
|
22
|
+
The first read lists each path once (``GetParametersByPath``, one level, not
|
|
23
|
+
decrypted), so a key SSM does not hold — one that lives only in
|
|
24
|
+
``os.environ``, or only under ``/Global`` — costs no further call, and a
|
|
25
|
+
plain ``String`` comes straight from the listing. A ``SecureString`` is still
|
|
26
|
+
read (and decrypted) by name, only when asked for. A path the role may not
|
|
27
|
+
list is read key by key, as before.
|
|
19
28
|
|
|
20
29
|
Outside Lambda, or when project/AppStage cannot be resolved, ``getenv`` remains a
|
|
21
30
|
thin wrapper around ``os.environ.get`` with no SSM call.
|
|
@@ -157,26 +166,189 @@ def _ssm_parameter_names(key: str) -> list[str]:
|
|
|
157
166
|
return parameter_names
|
|
158
167
|
|
|
159
168
|
|
|
160
|
-
|
|
161
|
-
|
|
169
|
+
# One SSM client per process, with timeouts (AWS-6): a client per key cost a
|
|
170
|
+
# botocore client build on every cold-start read.
|
|
171
|
+
_client = None
|
|
172
|
+
_CONNECT_TIMEOUT = 3
|
|
173
|
+
_READ_TIMEOUT = 5
|
|
174
|
+
# A GetParameters call names at most 10 parameters.
|
|
175
|
+
_BATCH = 10
|
|
176
|
+
# Error codes that mean "not here" (cached as a miss, and the project path
|
|
177
|
+
# falls through to /Global). AccessDenied stays a miss: a role scoped to one
|
|
178
|
+
# path reads the other that way today. Anything else from AWS — a throttle, a
|
|
179
|
+
# 5xx, a timeout — is transient: not cached, no fall-through.
|
|
180
|
+
_MISS_CODES = frozenset(
|
|
181
|
+
{
|
|
182
|
+
"ParameterNotFound",
|
|
183
|
+
"ParameterVersionNotFound",
|
|
184
|
+
"AccessDeniedException",
|
|
185
|
+
"ValidationException",
|
|
186
|
+
}
|
|
187
|
+
)
|
|
188
|
+
# _fetch's answer when a read failed transiently (distinct from a miss).
|
|
189
|
+
_TRANSIENT = object()
|
|
190
|
+
|
|
191
|
+
# Per-process listing of each SSM path: path -> {parameter name: plain value,
|
|
192
|
+
# or _SECURE for a SecureString (read by name when asked for)}, or _UNLISTED
|
|
193
|
+
# when the path may not be listed (read key by key). A transient listing
|
|
194
|
+
# failure caches nothing.
|
|
195
|
+
_listing: dict[str, object] = {}
|
|
196
|
+
_SECURE = object()
|
|
197
|
+
_UNLISTED = object()
|
|
198
|
+
# GetParametersByPath returns at most 10 parameters a page.
|
|
199
|
+
_PAGE = 10
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def ssm_client():
|
|
203
|
+
"""The process's SSM client (built once, in :func:`get_region`, with
|
|
204
|
+
connect/read timeouts and standard retries)."""
|
|
205
|
+
global _client
|
|
206
|
+
if _client is None:
|
|
207
|
+
import boto3
|
|
208
|
+
from botocore.config import Config
|
|
209
|
+
|
|
210
|
+
_client = boto3.client(
|
|
211
|
+
"ssm",
|
|
212
|
+
region_name=get_region("us-east-1"),
|
|
213
|
+
config=Config(
|
|
214
|
+
connect_timeout=_CONNECT_TIMEOUT,
|
|
215
|
+
read_timeout=_READ_TIMEOUT,
|
|
216
|
+
retries={"max_attempts": 3, "mode": "standard"},
|
|
217
|
+
),
|
|
218
|
+
)
|
|
219
|
+
return _client
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def _is_transient(exc: Exception) -> bool:
|
|
223
|
+
try:
|
|
224
|
+
from botocore.exceptions import BotoCoreError, ClientError
|
|
225
|
+
except ImportError: # pragma: no cover - boto3 always brings botocore
|
|
226
|
+
return False
|
|
227
|
+
if isinstance(exc, ClientError):
|
|
228
|
+
code = (getattr(exc, "response", None) or {}).get("Error", {}).get("Code")
|
|
229
|
+
return code not in _MISS_CODES
|
|
230
|
+
return isinstance(exc, BotoCoreError)
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def _fetch(key: str):
|
|
234
|
+
"""Resolve one key from project SSM first, then Global fallback: the
|
|
235
|
+
value, None on a miss, or :data:`_TRANSIENT` when AWS failed."""
|
|
162
236
|
parameter_names = _ssm_parameter_names(key)
|
|
163
237
|
if not parameter_names:
|
|
164
238
|
return None
|
|
165
239
|
|
|
166
240
|
try:
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
ssm = boto3.client("ssm", region_name=get_region("us-east-1"))
|
|
170
|
-
for param_name in parameter_names:
|
|
171
|
-
try:
|
|
172
|
-
resp = ssm.get_parameter(Name=param_name, WithDecryption=True)
|
|
173
|
-
return resp["Parameter"]["Value"]
|
|
174
|
-
except Exception as exc:
|
|
175
|
-
logger.debug("SSM miss for %s: %s", param_name, exc)
|
|
176
|
-
return None
|
|
241
|
+
ssm = ssm_client()
|
|
177
242
|
except Exception as exc:
|
|
178
243
|
logger.debug("SSM client failure while resolving %s: %s", key, exc)
|
|
179
244
|
return None
|
|
245
|
+
for param_name in parameter_names:
|
|
246
|
+
listed = _list_path(ssm, param_name.rsplit("/", 1)[0])
|
|
247
|
+
if isinstance(listed, dict):
|
|
248
|
+
if param_name not in listed:
|
|
249
|
+
continue # SSM does not hold it: no call
|
|
250
|
+
if listed[param_name] is not _SECURE:
|
|
251
|
+
return listed[param_name]
|
|
252
|
+
try:
|
|
253
|
+
resp = ssm.get_parameter(Name=param_name, WithDecryption=True)
|
|
254
|
+
return resp["Parameter"]["Value"]
|
|
255
|
+
except Exception as exc:
|
|
256
|
+
if _is_transient(exc):
|
|
257
|
+
logger.warning(
|
|
258
|
+
"SSM read of %s failed (%s); not cached",
|
|
259
|
+
param_name,
|
|
260
|
+
type(exc).__name__,
|
|
261
|
+
)
|
|
262
|
+
return _TRANSIENT
|
|
263
|
+
logger.debug("SSM miss for %s: %s", param_name, exc)
|
|
264
|
+
return None
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
def _list_path(ssm, path: str):
|
|
268
|
+
"""The listing of ``path`` (cached per process): a dict of the names
|
|
269
|
+
directly under it, :data:`_UNLISTED` when it may not be listed, or
|
|
270
|
+
:data:`_TRANSIENT` when AWS failed (nothing cached; read by name)."""
|
|
271
|
+
cached = _listing.get(path)
|
|
272
|
+
if cached is not None:
|
|
273
|
+
return cached
|
|
274
|
+
found: dict[str, object] = {}
|
|
275
|
+
kwargs = {
|
|
276
|
+
"Path": path,
|
|
277
|
+
"Recursive": False,
|
|
278
|
+
"WithDecryption": False,
|
|
279
|
+
"MaxResults": _PAGE,
|
|
280
|
+
}
|
|
281
|
+
try:
|
|
282
|
+
while True:
|
|
283
|
+
page = ssm.get_parameters_by_path(**kwargs)
|
|
284
|
+
for parameter in page.get("Parameters") or ():
|
|
285
|
+
found[parameter["Name"]] = (
|
|
286
|
+
_SECURE
|
|
287
|
+
if parameter.get("Type") == "SecureString"
|
|
288
|
+
else parameter["Value"]
|
|
289
|
+
)
|
|
290
|
+
token = page.get("NextToken")
|
|
291
|
+
if not token:
|
|
292
|
+
break
|
|
293
|
+
kwargs["NextToken"] = token
|
|
294
|
+
except Exception as exc:
|
|
295
|
+
if _is_transient(exc):
|
|
296
|
+
logger.warning(
|
|
297
|
+
"SSM listing of %s failed (%s); reading by name",
|
|
298
|
+
path,
|
|
299
|
+
type(exc).__name__,
|
|
300
|
+
)
|
|
301
|
+
return _TRANSIENT
|
|
302
|
+
logger.debug("SSM listing of %s unavailable: %s", path, exc)
|
|
303
|
+
_listing[path] = _UNLISTED
|
|
304
|
+
return _UNLISTED
|
|
305
|
+
_listing[path] = found
|
|
306
|
+
return found
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
def prefetch(keys) -> list[str]:
|
|
310
|
+
"""Resolve ``keys`` into the cache in batched ``GetParameters`` calls
|
|
311
|
+
(both the project and the ``/Global`` name of each, 10 names a call)
|
|
312
|
+
instead of one or two ``GetParameter`` calls per key.
|
|
313
|
+
|
|
314
|
+
For a cold start that reads many keys: the same answers :func:`getenv`
|
|
315
|
+
would give (project value first, then Global, else a miss), cached the
|
|
316
|
+
same way. All or nothing: any failed call caches nothing, and
|
|
317
|
+
:func:`getenv` then reads key by key as before. Needs
|
|
318
|
+
``ssm:GetParameters`` on those names (velocity's ``ssm_read_statement``
|
|
319
|
+
grants it). Returns the keys found in SSM, sorted; [] when SSM is off.
|
|
320
|
+
"""
|
|
321
|
+
if not _is_enabled():
|
|
322
|
+
return []
|
|
323
|
+
wanted = [k for k in dict.fromkeys(keys) if k not in _cache]
|
|
324
|
+
names = {}
|
|
325
|
+
for key in wanted:
|
|
326
|
+
for rank, name in enumerate(_ssm_parameter_names(key)):
|
|
327
|
+
names[name] = (key, rank)
|
|
328
|
+
if not names:
|
|
329
|
+
return []
|
|
330
|
+
try:
|
|
331
|
+
ssm = ssm_client()
|
|
332
|
+
found = {}
|
|
333
|
+
ordered = list(names)
|
|
334
|
+
for start in range(0, len(ordered), _BATCH):
|
|
335
|
+
resp = ssm.get_parameters(
|
|
336
|
+
Names=ordered[start : start + _BATCH], WithDecryption=True
|
|
337
|
+
)
|
|
338
|
+
for parameter in resp.get("Parameters") or ():
|
|
339
|
+
key, rank = names[parameter["Name"]]
|
|
340
|
+
if key not in found or rank < found[key][0]:
|
|
341
|
+
found[key] = (rank, parameter["Value"])
|
|
342
|
+
except Exception as exc:
|
|
343
|
+
logger.warning(
|
|
344
|
+
"SSM prefetch of %d keys failed (%s); reading key by key",
|
|
345
|
+
len(wanted),
|
|
346
|
+
type(exc).__name__,
|
|
347
|
+
)
|
|
348
|
+
return []
|
|
349
|
+
for key in wanted:
|
|
350
|
+
_cache[key] = found[key][1] if key in found else _MISS
|
|
351
|
+
return sorted(found)
|
|
180
352
|
|
|
181
353
|
|
|
182
354
|
def getenv(
|
|
@@ -205,8 +377,9 @@ def getenv(
|
|
|
205
377
|
if _is_enabled():
|
|
206
378
|
if key not in _cache:
|
|
207
379
|
value = _fetch(key)
|
|
208
|
-
|
|
209
|
-
|
|
380
|
+
if value is not _TRANSIENT:
|
|
381
|
+
_cache[key] = value if value is not None else _MISS
|
|
382
|
+
cached = _cache.get(key, _MISS)
|
|
210
383
|
if cached is not _MISS:
|
|
211
384
|
if raise_on_missing and cached == "":
|
|
212
385
|
raise KeyError(f"{key} not found in SSM or environment")
|
|
@@ -309,5 +482,7 @@ def export_path(
|
|
|
309
482
|
def clear_cache() -> None:
|
|
310
483
|
"""Clear the SSM cache (useful in tests)."""
|
|
311
484
|
_cache.clear()
|
|
312
|
-
|
|
485
|
+
_listing.clear()
|
|
486
|
+
global _SSM_ENABLED, _client
|
|
313
487
|
_SSM_ENABLED = None
|
|
488
|
+
_client = None
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
velocity/__init__.py,sha256
|
|
1
|
+
velocity/__init__.py,sha256=-p_wnPMwH7aEXhWHLXLK-bW2cEDvIg6FJEASvUKhYSs,363
|
|
2
2
|
velocity/cache.py,sha256=HacQGQXMcrEveDxs-AYVGceRsyi_ZHZ9dcW8YizoGEE,9659
|
|
3
3
|
velocity/hooks.py,sha256=jY_qtAapNHvptmryFSD-Zc7RvLhV6e5nJEH3x2xTgCw,20575
|
|
4
4
|
velocity/logging.py,sha256=MAsZap1cDSdOBNRc_9ezQxup5dHOS0ARxU9NI_-UK3E,4037
|
|
@@ -24,7 +24,7 @@ velocity/aws/scheduler.py,sha256=bn6GxaQaCsZyVzUSJhyfA64j6H4ZtRzIGx4f1SZI-RY,219
|
|
|
24
24
|
velocity/aws/schema.py,sha256=65hUgWCxPRb2jBxHGQV3LBBJyATj2NIV4ZBm-zujYuc,6885
|
|
25
25
|
velocity/aws/settings.py,sha256=aAQL7_dtcC5VSBnk60SX2KwU6jYDmKuyBQVp12Mr5as,39464
|
|
26
26
|
velocity/aws/sms.py,sha256=2CdcLlIEy_V5K5Qx_pZ2vrakvEK7IhP7y1JUttBgyG4,16253
|
|
27
|
-
velocity/aws/ssm_config.py,sha256=
|
|
27
|
+
velocity/aws/ssm_config.py,sha256=T89j0sBr1D1ra2qBg70G7I7x-cHtm0H3F9m43NDsA_U,16647
|
|
28
28
|
velocity/aws/tenancy.py,sha256=GmraxV3tJBeNJhqnJedH29LECCuOxNY_UIimvzOakSQ,23335
|
|
29
29
|
velocity/aws/tiles.py,sha256=_38EH9ngxQxLeiZnTHBta-TXg1VkO2hPvaoPHWEKAlc,28011
|
|
30
30
|
velocity/aws/user_data.py,sha256=qLdmzUpkapUToEWaQ_jSeJa6hHgTHIqz0p0Cp2JuFx4,15564
|
|
@@ -291,9 +291,9 @@ velocity/workflow/points.py,sha256=g_vQ3cdHdx_JloVCS5dVjo_jJoSolO92K5p3suDzxlE,1
|
|
|
291
291
|
velocity/workflow/schema.py,sha256=1ZAkGZ_4YrTAfY6TIPCviWQJGLhhmAJmkxisfLyy2og,21597
|
|
292
292
|
velocity/workflow/traveler.py,sha256=Ik1InoEo69sZB1wVV-UinVo4P0P5QlcV_hh858p3OWE,20562
|
|
293
293
|
velocity/workflow/views.py,sha256=oTPjpuheND8y4VbcTY8c-4fMdSHKb0jgGzXMuT_G7_s,32175
|
|
294
|
-
velocity_python-0.13.
|
|
295
|
-
velocity_python-0.13.
|
|
296
|
-
velocity_python-0.13.
|
|
297
|
-
velocity_python-0.13.
|
|
298
|
-
velocity_python-0.13.
|
|
299
|
-
velocity_python-0.13.
|
|
294
|
+
velocity_python-0.13.2.dist-info/licenses/LICENSE,sha256=aoN245GG8s9oRUU89KNiGTU4_4OtnNmVi4hQeChg6rM,1076
|
|
295
|
+
velocity_python-0.13.2.dist-info/METADATA,sha256=M5jOfVPSKsv6zLd-ps8xPJkqBs2DoZnRJLNfhEG95CI,17709
|
|
296
|
+
velocity_python-0.13.2.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
|
|
297
|
+
velocity_python-0.13.2.dist-info/entry_points.txt,sha256=nXUKJHkOGQCSPoagdmMNd_qycrle1LVU-GhYknLGM1M,56
|
|
298
|
+
velocity_python-0.13.2.dist-info/top_level.txt,sha256=JW2vJPmodgdgSz7H6yoZvnxF8S3fTMIv-YJWCT1sNW0,9
|
|
299
|
+
velocity_python-0.13.2.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|