velocity-python 0.13.0__py3-none-any.whl → 0.13.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
velocity/__init__.py CHANGED
@@ -1,4 +1,4 @@
1
- __version__ = version = "0.13.0"
1
+ __version__ = version = "0.13.2"
2
2
 
3
3
  import importlib as _importlib
4
4
 
@@ -15,7 +15,16 @@ when they can resolve both project and AppStage from the runtime environment.
15
15
 
16
16
  On an SSM miss or any other SSM error, ``getenv`` falls back to ``os.environ``.
17
17
  Results are cached for the process lifetime so SSM is called at most once per
18
- key per Lambda cold start.
18
+ key per Lambda cold start; a transient AWS failure (throttle, timeout, 5xx) is
19
+ not cached. One client serves the process (``ssm_client``), and ``prefetch``
20
+ fills the cache for many keys in batched ``GetParameters`` calls.
21
+
22
+ The first read lists each path once (``GetParametersByPath``, one level, not
23
+ decrypted), so a key SSM does not hold — one that lives only in
24
+ ``os.environ``, or only under ``/Global`` — costs no further call, and a
25
+ plain ``String`` comes straight from the listing. A ``SecureString`` is still
26
+ read (and decrypted) by name, only when asked for. A path the role may not
27
+ list is read key by key, as before.
19
28
 
20
29
  Outside Lambda, or when project/AppStage cannot be resolved, ``getenv`` remains a
21
30
  thin wrapper around ``os.environ.get`` with no SSM call.
@@ -157,26 +166,189 @@ def _ssm_parameter_names(key: str) -> list[str]:
157
166
  return parameter_names
158
167
 
159
168
 
160
- def _fetch(key: str) -> str | None:
161
- """Resolve one key from project SSM first, then Global fallback."""
169
+ # One SSM client per process, with timeouts (AWS-6): a client per key cost a
170
+ # botocore client build on every cold-start read.
171
+ _client = None
172
+ _CONNECT_TIMEOUT = 3
173
+ _READ_TIMEOUT = 5
174
+ # A GetParameters call names at most 10 parameters.
175
+ _BATCH = 10
176
+ # Error codes that mean "not here" (cached as a miss, and the project path
177
+ # falls through to /Global). AccessDenied stays a miss: a role scoped to one
178
+ # path reads the other that way today. Anything else from AWS — a throttle, a
179
+ # 5xx, a timeout — is transient: not cached, no fall-through.
180
+ _MISS_CODES = frozenset(
181
+ {
182
+ "ParameterNotFound",
183
+ "ParameterVersionNotFound",
184
+ "AccessDeniedException",
185
+ "ValidationException",
186
+ }
187
+ )
188
+ # _fetch's answer when a read failed transiently (distinct from a miss).
189
+ _TRANSIENT = object()
190
+
191
+ # Per-process listing of each SSM path: path -> {parameter name: plain value,
192
+ # or _SECURE for a SecureString (read by name when asked for)}, or _UNLISTED
193
+ # when the path may not be listed (read key by key). A transient listing
194
+ # failure caches nothing.
195
+ _listing: dict[str, object] = {}
196
+ _SECURE = object()
197
+ _UNLISTED = object()
198
+ # GetParametersByPath returns at most 10 parameters a page.
199
+ _PAGE = 10
200
+
201
+
202
+ def ssm_client():
203
+ """The process's SSM client (built once, in :func:`get_region`, with
204
+ connect/read timeouts and standard retries)."""
205
+ global _client
206
+ if _client is None:
207
+ import boto3
208
+ from botocore.config import Config
209
+
210
+ _client = boto3.client(
211
+ "ssm",
212
+ region_name=get_region("us-east-1"),
213
+ config=Config(
214
+ connect_timeout=_CONNECT_TIMEOUT,
215
+ read_timeout=_READ_TIMEOUT,
216
+ retries={"max_attempts": 3, "mode": "standard"},
217
+ ),
218
+ )
219
+ return _client
220
+
221
+
222
+ def _is_transient(exc: Exception) -> bool:
223
+ try:
224
+ from botocore.exceptions import BotoCoreError, ClientError
225
+ except ImportError: # pragma: no cover - boto3 always brings botocore
226
+ return False
227
+ if isinstance(exc, ClientError):
228
+ code = (getattr(exc, "response", None) or {}).get("Error", {}).get("Code")
229
+ return code not in _MISS_CODES
230
+ return isinstance(exc, BotoCoreError)
231
+
232
+
233
+ def _fetch(key: str):
234
+ """Resolve one key from project SSM first, then Global fallback: the
235
+ value, None on a miss, or :data:`_TRANSIENT` when AWS failed."""
162
236
  parameter_names = _ssm_parameter_names(key)
163
237
  if not parameter_names:
164
238
  return None
165
239
 
166
240
  try:
167
- import boto3
168
-
169
- ssm = boto3.client("ssm", region_name=get_region("us-east-1"))
170
- for param_name in parameter_names:
171
- try:
172
- resp = ssm.get_parameter(Name=param_name, WithDecryption=True)
173
- return resp["Parameter"]["Value"]
174
- except Exception as exc:
175
- logger.debug("SSM miss for %s: %s", param_name, exc)
176
- return None
241
+ ssm = ssm_client()
177
242
  except Exception as exc:
178
243
  logger.debug("SSM client failure while resolving %s: %s", key, exc)
179
244
  return None
245
+ for param_name in parameter_names:
246
+ listed = _list_path(ssm, param_name.rsplit("/", 1)[0])
247
+ if isinstance(listed, dict):
248
+ if param_name not in listed:
249
+ continue # SSM does not hold it: no call
250
+ if listed[param_name] is not _SECURE:
251
+ return listed[param_name]
252
+ try:
253
+ resp = ssm.get_parameter(Name=param_name, WithDecryption=True)
254
+ return resp["Parameter"]["Value"]
255
+ except Exception as exc:
256
+ if _is_transient(exc):
257
+ logger.warning(
258
+ "SSM read of %s failed (%s); not cached",
259
+ param_name,
260
+ type(exc).__name__,
261
+ )
262
+ return _TRANSIENT
263
+ logger.debug("SSM miss for %s: %s", param_name, exc)
264
+ return None
265
+
266
+
267
+ def _list_path(ssm, path: str):
268
+ """The listing of ``path`` (cached per process): a dict of the names
269
+ directly under it, :data:`_UNLISTED` when it may not be listed, or
270
+ :data:`_TRANSIENT` when AWS failed (nothing cached; read by name)."""
271
+ cached = _listing.get(path)
272
+ if cached is not None:
273
+ return cached
274
+ found: dict[str, object] = {}
275
+ kwargs = {
276
+ "Path": path,
277
+ "Recursive": False,
278
+ "WithDecryption": False,
279
+ "MaxResults": _PAGE,
280
+ }
281
+ try:
282
+ while True:
283
+ page = ssm.get_parameters_by_path(**kwargs)
284
+ for parameter in page.get("Parameters") or ():
285
+ found[parameter["Name"]] = (
286
+ _SECURE
287
+ if parameter.get("Type") == "SecureString"
288
+ else parameter["Value"]
289
+ )
290
+ token = page.get("NextToken")
291
+ if not token:
292
+ break
293
+ kwargs["NextToken"] = token
294
+ except Exception as exc:
295
+ if _is_transient(exc):
296
+ logger.warning(
297
+ "SSM listing of %s failed (%s); reading by name",
298
+ path,
299
+ type(exc).__name__,
300
+ )
301
+ return _TRANSIENT
302
+ logger.debug("SSM listing of %s unavailable: %s", path, exc)
303
+ _listing[path] = _UNLISTED
304
+ return _UNLISTED
305
+ _listing[path] = found
306
+ return found
307
+
308
+
309
+ def prefetch(keys) -> list[str]:
310
+ """Resolve ``keys`` into the cache in batched ``GetParameters`` calls
311
+ (both the project and the ``/Global`` name of each, 10 names a call)
312
+ instead of one or two ``GetParameter`` calls per key.
313
+
314
+ For a cold start that reads many keys: the same answers :func:`getenv`
315
+ would give (project value first, then Global, else a miss), cached the
316
+ same way. All or nothing: any failed call caches nothing, and
317
+ :func:`getenv` then reads key by key as before. Needs
318
+ ``ssm:GetParameters`` on those names (velocity's ``ssm_read_statement``
319
+ grants it). Returns the keys found in SSM, sorted; [] when SSM is off.
320
+ """
321
+ if not _is_enabled():
322
+ return []
323
+ wanted = [k for k in dict.fromkeys(keys) if k not in _cache]
324
+ names = {}
325
+ for key in wanted:
326
+ for rank, name in enumerate(_ssm_parameter_names(key)):
327
+ names[name] = (key, rank)
328
+ if not names:
329
+ return []
330
+ try:
331
+ ssm = ssm_client()
332
+ found = {}
333
+ ordered = list(names)
334
+ for start in range(0, len(ordered), _BATCH):
335
+ resp = ssm.get_parameters(
336
+ Names=ordered[start : start + _BATCH], WithDecryption=True
337
+ )
338
+ for parameter in resp.get("Parameters") or ():
339
+ key, rank = names[parameter["Name"]]
340
+ if key not in found or rank < found[key][0]:
341
+ found[key] = (rank, parameter["Value"])
342
+ except Exception as exc:
343
+ logger.warning(
344
+ "SSM prefetch of %d keys failed (%s); reading key by key",
345
+ len(wanted),
346
+ type(exc).__name__,
347
+ )
348
+ return []
349
+ for key in wanted:
350
+ _cache[key] = found[key][1] if key in found else _MISS
351
+ return sorted(found)
180
352
 
181
353
 
182
354
  def getenv(
@@ -205,8 +377,9 @@ def getenv(
205
377
  if _is_enabled():
206
378
  if key not in _cache:
207
379
  value = _fetch(key)
208
- _cache[key] = value if value is not None else _MISS
209
- cached = _cache[key]
380
+ if value is not _TRANSIENT:
381
+ _cache[key] = value if value is not None else _MISS
382
+ cached = _cache.get(key, _MISS)
210
383
  if cached is not _MISS:
211
384
  if raise_on_missing and cached == "":
212
385
  raise KeyError(f"{key} not found in SSM or environment")
@@ -309,5 +482,7 @@ def export_path(
309
482
  def clear_cache() -> None:
310
483
  """Clear the SSM cache (useful in tests)."""
311
484
  _cache.clear()
312
- global _SSM_ENABLED
485
+ _listing.clear()
486
+ global _SSM_ENABLED, _client
313
487
  _SSM_ENABLED = None
488
+ _client = None
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: velocity-python
3
- Version: 0.13.0
3
+ Version: 0.13.2
4
4
  Summary: A rapid application development library for interfacing with data storage
5
5
  Author-email: Velocity Team <info@codeclubs.org>
6
6
  License-Expression: MIT
@@ -1,4 +1,4 @@
1
- velocity/__init__.py,sha256=O369AuZrUSzRvRgmgF7ZwAY_ClYjAzUHBAJnlJO6V34,363
1
+ velocity/__init__.py,sha256=-p_wnPMwH7aEXhWHLXLK-bW2cEDvIg6FJEASvUKhYSs,363
2
2
  velocity/cache.py,sha256=HacQGQXMcrEveDxs-AYVGceRsyi_ZHZ9dcW8YizoGEE,9659
3
3
  velocity/hooks.py,sha256=jY_qtAapNHvptmryFSD-Zc7RvLhV6e5nJEH3x2xTgCw,20575
4
4
  velocity/logging.py,sha256=MAsZap1cDSdOBNRc_9ezQxup5dHOS0ARxU9NI_-UK3E,4037
@@ -24,7 +24,7 @@ velocity/aws/scheduler.py,sha256=bn6GxaQaCsZyVzUSJhyfA64j6H4ZtRzIGx4f1SZI-RY,219
24
24
  velocity/aws/schema.py,sha256=65hUgWCxPRb2jBxHGQV3LBBJyATj2NIV4ZBm-zujYuc,6885
25
25
  velocity/aws/settings.py,sha256=aAQL7_dtcC5VSBnk60SX2KwU6jYDmKuyBQVp12Mr5as,39464
26
26
  velocity/aws/sms.py,sha256=2CdcLlIEy_V5K5Qx_pZ2vrakvEK7IhP7y1JUttBgyG4,16253
27
- velocity/aws/ssm_config.py,sha256=VKjP86WOGBfRZnkMpU22Pu-alxNS1rPUvngyQlTSMcs,10049
27
+ velocity/aws/ssm_config.py,sha256=T89j0sBr1D1ra2qBg70G7I7x-cHtm0H3F9m43NDsA_U,16647
28
28
  velocity/aws/tenancy.py,sha256=GmraxV3tJBeNJhqnJedH29LECCuOxNY_UIimvzOakSQ,23335
29
29
  velocity/aws/tiles.py,sha256=_38EH9ngxQxLeiZnTHBta-TXg1VkO2hPvaoPHWEKAlc,28011
30
30
  velocity/aws/user_data.py,sha256=qLdmzUpkapUToEWaQ_jSeJa6hHgTHIqz0p0Cp2JuFx4,15564
@@ -291,9 +291,9 @@ velocity/workflow/points.py,sha256=g_vQ3cdHdx_JloVCS5dVjo_jJoSolO92K5p3suDzxlE,1
291
291
  velocity/workflow/schema.py,sha256=1ZAkGZ_4YrTAfY6TIPCviWQJGLhhmAJmkxisfLyy2og,21597
292
292
  velocity/workflow/traveler.py,sha256=Ik1InoEo69sZB1wVV-UinVo4P0P5QlcV_hh858p3OWE,20562
293
293
  velocity/workflow/views.py,sha256=oTPjpuheND8y4VbcTY8c-4fMdSHKb0jgGzXMuT_G7_s,32175
294
- velocity_python-0.13.0.dist-info/licenses/LICENSE,sha256=aoN245GG8s9oRUU89KNiGTU4_4OtnNmVi4hQeChg6rM,1076
295
- velocity_python-0.13.0.dist-info/METADATA,sha256=wKlYvuOFymuED_rPB3xHg-fGP76i3hj27BjF-YqkCJ0,17709
296
- velocity_python-0.13.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
297
- velocity_python-0.13.0.dist-info/entry_points.txt,sha256=nXUKJHkOGQCSPoagdmMNd_qycrle1LVU-GhYknLGM1M,56
298
- velocity_python-0.13.0.dist-info/top_level.txt,sha256=JW2vJPmodgdgSz7H6yoZvnxF8S3fTMIv-YJWCT1sNW0,9
299
- velocity_python-0.13.0.dist-info/RECORD,,
294
+ velocity_python-0.13.2.dist-info/licenses/LICENSE,sha256=aoN245GG8s9oRUU89KNiGTU4_4OtnNmVi4hQeChg6rM,1076
295
+ velocity_python-0.13.2.dist-info/METADATA,sha256=M5jOfVPSKsv6zLd-ps8xPJkqBs2DoZnRJLNfhEG95CI,17709
296
+ velocity_python-0.13.2.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
297
+ velocity_python-0.13.2.dist-info/entry_points.txt,sha256=nXUKJHkOGQCSPoagdmMNd_qycrle1LVU-GhYknLGM1M,56
298
+ velocity_python-0.13.2.dist-info/top_level.txt,sha256=JW2vJPmodgdgSz7H6yoZvnxF8S3fTMIv-YJWCT1sNW0,9
299
+ velocity_python-0.13.2.dist-info/RECORD,,