agentenv-framework 0.9.1254__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agent_env/__init__.py +0 -0
- agent_env/a2a_agent/__init__.py +23 -0
- agent_env/a2a_agent/a2a_agent.py +600 -0
- agent_env/a2a_agent/conversation_store.py +356 -0
- agent_env/a2a_agent/object_transfer.py +609 -0
- agent_env/a2a_agent/protocol.py +238 -0
- agent_env/a2a_agent/store.py +201 -0
- agent_env/a2a_agent/validator.py +901 -0
- agent_env/artifact/__init__.py +48 -0
- agent_env/artifact/artifact.py +74 -0
- agent_env/artifact/artifacts/__init__.py +0 -0
- agent_env/artifact/artifacts/cli.py +83 -0
- agent_env/artifact/artifacts/docker_image.py +457 -0
- agent_env/artifact/artifacts/environment.py +89 -0
- agent_env/artifact/artifacts/environment_universe.py +156 -0
- agent_env/artifact/artifacts/file.py +214 -0
- agent_env/artifact/artifacts/file_artifact_universe.py +206 -0
- agent_env/artifact/artifacts/skill.py +206 -0
- agent_env/artifact/artifacts/vm_image.py +95 -0
- agent_env/artifact/ref.py +15 -0
- agent_env/artifact/registry.py +251 -0
- agent_env/artifact/store.py +295 -0
- agent_env/artifact/universe.py +38 -0
- agent_env/attribution.py +41 -0
- agent_env/bundle/__init__.py +4 -0
- agent_env/bundle/_fs.py +128 -0
- agent_env/bundle/authoring.py +239 -0
- agent_env/bundle/installed.py +175 -0
- agent_env/bundle/ledger.py +269 -0
- agent_env/bundle/materialize.py +204 -0
- agent_env/bundle/parse.py +594 -0
- agent_env/bundle/plan.py +414 -0
- agent_env/bundle/resolve.py +404 -0
- agent_env/bundle/run.py +361 -0
- agent_env/cli/__init__.py +69 -0
- agent_env/cli/__main__.py +10 -0
- agent_env/cli/_installers.py +576 -0
- agent_env/cli/_plugin_changes.py +947 -0
- agent_env/cli/a2a_agent/__init__.py +22 -0
- agent_env/cli/a2a_agent/add_skill.py +97 -0
- agent_env/cli/a2a_agent/deploy.py +60 -0
- agent_env/cli/a2a_agent/get.py +15 -0
- agent_env/cli/a2a_agent/get_instance.py +26 -0
- agent_env/cli/a2a_agent/put.py +78 -0
- agent_env/cli/a2a_agent/validate.py +56 -0
- agent_env/cli/artifact/__init__.py +20 -0
- agent_env/cli/artifact/cli.py +83 -0
- agent_env/cli/artifact/environment.py +44 -0
- agent_env/cli/artifact/environment_universe.py +163 -0
- agent_env/cli/artifact/file_artifact_universe.py +281 -0
- agent_env/cli/artifact/skill.py +128 -0
- agent_env/cli/banner.py +25 -0
- agent_env/cli/config.py +263 -0
- agent_env/cli/env/__init__.py +31 -0
- agent_env/cli/env/deploy.py +112 -0
- agent_env/cli/env/gateway.py +57 -0
- agent_env/cli/env/get_instance.py +40 -0
- agent_env/cli/env/mcp_server.py +410 -0
- agent_env/cli/env/multi.py +282 -0
- agent_env/cli/env/service_db.py +99 -0
- agent_env/cli/env/snapshot.py +35 -0
- agent_env/cli/env/state/__init__.py +6 -0
- agent_env/cli/env/state/init.py +108 -0
- agent_env/cli/env/state/teardown.py +44 -0
- agent_env/cli/env/website.py +216 -0
- agent_env/cli/env/website_browser.py +67 -0
- agent_env/cli/eval/__init__.py +16 -0
- agent_env/cli/eval/add_tasks.py +63 -0
- agent_env/cli/eval/create.py +49 -0
- agent_env/cli/eval/run.py +194 -0
- agent_env/cli/identity/__init__.py +5 -0
- agent_env/cli/identity/client_identity.py +37 -0
- agent_env/cli/plugin.py +438 -0
- agent_env/cli/run.py +285 -0
- agent_env/cli/task/__init__.py +21 -0
- agent_env/cli/task/create.py +109 -0
- agent_env/cli/task/get.py +17 -0
- agent_env/cli/task/get_instance.py +34 -0
- agent_env/cli/task/run.py +615 -0
- agent_env/cli/task/validate.py +31 -0
- agent_env/cli/up.py +123 -0
- agent_env/cli/utils.py +183 -0
- agent_env/config/__init__.py +50 -0
- agent_env/config/describe.py +1067 -0
- agent_env/config/errors.py +13 -0
- agent_env/config/loader.py +245 -0
- agent_env/config/model.py +154 -0
- agent_env/config/paths.py +25 -0
- agent_env/config/plugin_tables.py +101 -0
- agent_env/config/provenance.py +134 -0
- agent_env/config/runtime.py +1064 -0
- agent_env/config/snapshot.py +81 -0
- agent_env/entity_refs.py +186 -0
- agent_env/env/__init__.py +49 -0
- agent_env/env/env.py +433 -0
- agent_env/env/env_artifact_store.py +87 -0
- agent_env/env/envs/__init__.py +7 -0
- agent_env/env/envs/_deployment.py +281 -0
- agent_env/env/envs/gateway_server.py +32 -0
- agent_env/env/envs/mcp_server.py +387 -0
- agent_env/env/envs/multi_env.py +780 -0
- agent_env/env/envs/service_db/Dockerfile +20 -0
- agent_env/env/envs/service_db/Dockerfile.db-mcp +42 -0
- agent_env/env/envs/service_db/Dockerfile.db-web +8 -0
- agent_env/env/envs/service_db.py +92 -0
- agent_env/env/envs/website.py +294 -0
- agent_env/env/envs/website_browser/Dockerfile +40 -0
- agent_env/env/envs/website_browser/__init__.py +17 -0
- agent_env/env/envs/website_browser/entrypoint.sh +13 -0
- agent_env/env/gateway/Dockerfile +14 -0
- agent_env/env/gateway/__init__.py +37 -0
- agent_env/env/gateway/clock.py +184 -0
- agent_env/env/gateway/constants.py +190 -0
- agent_env/env/gateway/entrypoint.py +156 -0
- agent_env/env/gateway/gateway.py +1095 -0
- agent_env/env/gateway/get_time.py +60 -0
- agent_env/env/gateway/requirements.txt +4 -0
- agent_env/env/gateway/triggers.py +1193 -0
- agent_env/env/legacy_protocol.py +146 -0
- agent_env/env/registry.py +93 -0
- agent_env/env/snapshot_store.py +372 -0
- agent_env/env/store.py +411 -0
- agent_env/eval/__init__.py +14 -0
- agent_env/eval/eval.py +68 -0
- agent_env/eval/store.py +116 -0
- agent_env/examples/hello/README.md +9 -0
- agent_env/examples/hello/artifacts/greeting/check.sh +1 -0
- agent_env/examples/hello/artifacts/greeting/hello.txt +1 -0
- agent_env/examples/hello/tasks/hello.json +10 -0
- agent_env/explorer/__init__.py +12 -0
- agent_env/explorer/app.py +277 -0
- agent_env/explorer/openapi_docs.py +242 -0
- agent_env/explorer/plugin.py +95 -0
- agent_env/explorer/routers/__init__.py +1 -0
- agent_env/explorer/routers/common.py +174 -0
- agent_env/explorer/routers/conversations.py +25 -0
- agent_env/explorer/routers/objects.py +112 -0
- agent_env/explorer/routers/runs.py +475 -0
- agent_env/plugins/__init__.py +59 -0
- agent_env/plugins/_cli.py +244 -0
- agent_env/plugins/_discovery.py +97 -0
- agent_env/plugins/_inventory.py +262 -0
- agent_env/plugins/_registration.py +268 -0
- agent_env/plugins/_report.py +47 -0
- agent_env/plugins/_requirements.py +55 -0
- agent_env/providers/__init__.py +37 -0
- agent_env/providers/env_providers/__init__.py +12 -0
- agent_env/providers/env_providers/constants.py +13 -0
- agent_env/providers/env_providers/env_gateway_provider.py +1354 -0
- agent_env/providers/env_providers/env_provider.py +237 -0
- agent_env/providers/env_providers/env_server_provider.py +133 -0
- agent_env/providers/env_state/__init__.py +49 -0
- agent_env/providers/env_state/env_state_provider.py +554 -0
- agent_env/providers/env_state/local_postgres.py +390 -0
- agent_env/providers/env_state/store.py +112 -0
- agent_env/providers/sandbox_providers/__init__.py +29 -0
- agent_env/providers/sandbox_providers/chained_sandbox_provider.py +93 -0
- agent_env/providers/sandbox_providers/e2b/__init__.py +6 -0
- agent_env/providers/sandbox_providers/e2b/provider.py +376 -0
- agent_env/providers/sandbox_providers/e2b/sandbox.py +342 -0
- agent_env/providers/sandbox_providers/e2b/template.py +158 -0
- agent_env/providers/sandbox_providers/local_sandbox.py +408 -0
- agent_env/providers/sandbox_providers/modal_sandbox.py +500 -0
- agent_env/providers/sandbox_providers/modal_vm_sandbox.py +466 -0
- agent_env/providers/sandbox_providers/sandbox.py +387 -0
- agent_env/providers/sandbox_providers/sandbox_provider.py +614 -0
- agent_env/py.typed +0 -0
- agent_env/runner/__init__.py +16 -0
- agent_env/runner/local_runner.py +190 -0
- agent_env/runner/runner.py +127 -0
- agent_env/runner/store.py +94 -0
- agent_env/store/__init__.py +126 -0
- agent_env/store/_google.py +68 -0
- agent_env/store/base.py +33 -0
- agent_env/store/document_store/__init__.py +60 -0
- agent_env/store/document_store/document_store.py +519 -0
- agent_env/store/document_store/dynamodb_document_store.py +306 -0
- agent_env/store/document_store/evaluation.py +199 -0
- agent_env/store/document_store/firestore_mongo_document_store.py +242 -0
- agent_env/store/document_store/mongo_document_store.py +259 -0
- agent_env/store/document_store/sqlite_document_store.py +314 -0
- agent_env/store/ids.py +123 -0
- agent_env/store/image_store/__init__.py +34 -0
- agent_env/store/image_store/ecr_image_store.py +131 -0
- agent_env/store/image_store/google_credentials.py +113 -0
- agent_env/store/image_store/image_store.py +113 -0
- agent_env/store/image_store/local_registry_image_store.py +106 -0
- agent_env/store/image_store/oci_registry_credentials.py +196 -0
- agent_env/store/local_state.py +27 -0
- agent_env/store/object_store/__init__.py +24 -0
- agent_env/store/object_store/gcs_object_store.py +415 -0
- agent_env/store/object_store/local_object_store.py +143 -0
- agent_env/store/object_store/object_store.py +198 -0
- agent_env/store/object_store/s3_object_store.py +354 -0
- agent_env/store/query.py +112 -0
- agent_env/store/routing.py +609 -0
- agent_env/store/secret_store/__init__.py +19 -0
- agent_env/store/secret_store/aws_secrets_manager_secret_store.py +293 -0
- agent_env/store/secret_store/gcp_secret_manager_secret_store.py +319 -0
- agent_env/store/secret_store/local_secret_store.py +47 -0
- agent_env/store/secret_store/secret_store.py +19 -0
- agent_env/task/__init__.py +27 -0
- agent_env/task/interrupts.py +109 -0
- agent_env/task/registry.py +27 -0
- agent_env/task/step_journal.py +99 -0
- agent_env/task/store.py +940 -0
- agent_env/task/task.py +815 -0
- agent_env/task/teardown.py +142 -0
- agent_env/task_step/__init__.py +47 -0
- agent_env/task_step/context.py +195 -0
- agent_env/task_step/context_ops.py +224 -0
- agent_env/task_step/registry.py +182 -0
- agent_env/task_step/review_store.py +118 -0
- agent_env/task_step/snapshot_utils/__init__.py +2 -0
- agent_env/task_step/snapshot_utils/agent_state_capture.py +261 -0
- agent_env/task_step/snapshot_utils/snapshot_series.py +631 -0
- agent_env/task_step/store.py +129 -0
- agent_env/task_step/task_step.py +173 -0
- agent_env/task_step/task_steps/__init__.py +27 -0
- agent_env/task_step/task_steps/a2a_agent_validator/__init__.py +31 -0
- agent_env/task_step/task_steps/a2a_agent_validator/fixtures/README.md +158 -0
- agent_env/task_step/task_steps/a2a_agent_validator/fixtures/clip.m4a +0 -0
- agent_env/task_step/task_steps/a2a_agent_validator/fixtures/clip.mp3 +0 -0
- agent_env/task_step/task_steps/a2a_agent_validator/fixtures/clip.mp4 +0 -0
- agent_env/task_step/task_steps/a2a_agent_validator/fixtures/clip.ogg +0 -0
- agent_env/task_step/task_steps/a2a_agent_validator/fixtures/clip.wav +0 -0
- agent_env/task_step/task_steps/a2a_agent_validator/fixtures/document.pdf +0 -0
- agent_env/task_step/task_steps/a2a_agent_validator/fixtures/red.gif +0 -0
- agent_env/task_step/task_steps/a2a_agent_validator/fixtures/red.jpg +0 -0
- agent_env/task_step/task_steps/a2a_agent_validator/fixtures/red.png +0 -0
- agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_agent_card.py +103 -0
- agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_agent_config_identity.py +69 -0
- agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_agent_mcp.py +178 -0
- agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_core_protocol.py +67 -0
- agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_install.py +115 -0
- agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_litellm_attribution.py +124 -0
- agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_litellm_attribution_runtime.py +173 -0
- agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_modalities.py +270 -0
- agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_peer_agents.py +102 -0
- agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_role.py +151 -0
- agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_skill_config.py +194 -0
- agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_snapshot.py +111 -0
- agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_system_prompt.py +215 -0
- agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_trajectory.py +164 -0
- agent_env/task_step/task_steps/add_skills.py +240 -0
- agent_env/task_step/task_steps/apply_server_config.py +288 -0
- agent_env/task_step/task_steps/collect_artifacts.py +940 -0
- agent_env/task_step/task_steps/deploy_agent.py +626 -0
- agent_env/task_step/task_steps/deploy_env.py +197 -0
- agent_env/task_step/task_steps/deploy_human_agent.py +100 -0
- agent_env/task_step/task_steps/deploy_sandbox.py +170 -0
- agent_env/task_step/task_steps/env_card_validator/__init__.py +4 -0
- agent_env/task_step/task_steps/env_card_validator/verify_env_card.py +109 -0
- agent_env/task_step/task_steps/env_card_validator/verify_env_core_protocol.py +108 -0
- agent_env/task_step/task_steps/install_agent.py +376 -0
- agent_env/task_step/task_steps/load_artifact.py +775 -0
- agent_env/task_step/task_steps/mcp_cli_builder/__init__.py +4 -0
- agent_env/task_step/task_steps/mcp_cli_builder/build_mcp_cli.py +201 -0
- agent_env/task_step/task_steps/mcp_cli_builder/codegen.py +627 -0
- agent_env/task_step/task_steps/mcp_env_validator/__init__.py +44 -0
- agent_env/task_step/task_steps/mcp_env_validator/validation_gate_aggregator.py +102 -0
- agent_env/task_step/task_steps/mcp_env_validator/verify_mcp_env_assessment.py +78 -0
- agent_env/task_step/task_steps/mcp_env_validator/verify_mcp_tool_schema.py +152 -0
- agent_env/task_step/task_steps/mcp_env_validator/verify_spec_conformance.py +486 -0
- agent_env/task_step/task_steps/modify_env_tool_access.py +80 -0
- agent_env/task_step/task_steps/multienv_validator/__init__.py +3 -0
- agent_env/task_step/task_steps/multienv_validator/combine_universe_verdicts.py +107 -0
- agent_env/task_step/task_steps/multienv_validator/universe_comparison.py +253 -0
- agent_env/task_step/task_steps/multienv_validator/verify_universe_agent_judge.py +180 -0
- agent_env/task_step/task_steps/multienv_validator/verify_universe_roundtrip.py +241 -0
- agent_env/task_step/task_steps/peer_agents.py +92 -0
- agent_env/task_step/task_steps/prompt_agent.py +900 -0
- agent_env/task_step/task_steps/register_agent_triggers.py +113 -0
- agent_env/task_step/task_steps/register_env_triggers.py +97 -0
- agent_env/task_step/task_steps/reset_env.py +71 -0
- agent_env/task_step/task_steps/review.py +152 -0
- agent_env/task_step/task_steps/run_code.py +461 -0
- agent_env/task_step/task_steps/run_code_runner.py +52 -0
- agent_env/task_step/task_steps/run_docker_container.py +380 -0
- agent_env/task_step/task_steps/sandbox_utils/__init__.py +0 -0
- agent_env/task_step/task_steps/sandbox_utils/sandbox_utils.py +79 -0
- agent_env/task_step/task_steps/snapshot_agent_state.py +205 -0
- agent_env/task_step/task_steps/snapshot_env.py +637 -0
- agent_env/task_step/task_steps/sync_env_clock.py +132 -0
- agent_env/task_step/task_steps/teardown_sandboxes.py +170 -0
- agent_env/task_step/task_steps/verifiers/__init__.py +23 -0
- agent_env/task_step/task_steps/verifiers/agent_prompt_response_verifier.py +166 -0
- agent_env/task_step/task_steps/verifiers/aggregate_verifiers.py +90 -0
- agent_env/task_step/task_steps/verifiers/env_outcome_verifier.py +127 -0
- agent_env/task_step/task_steps/verifiers/judge_utils/__init__.py +2 -0
- agent_env/task_step/task_steps/verifiers/judge_utils/frame_selection.py +377 -0
- agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py +1293 -0
- agent_env/task_step/task_steps/verifiers/judge_utils/trajectory_filter.py +514 -0
- agent_env/task_step/task_steps/verifiers/rubrics_verifier.py +1318 -0
- agent_env/task_step/task_steps/verifiers/run_container_unit_tests_verifier.py +434 -0
- agent_env/task_step/task_steps/verifiers/scoring.py +40 -0
- agent_env/task_step/task_steps/verifiers/verify_sandbox.py +308 -0
- agent_env/task_step/thread_work.py +61 -0
- agent_env/utils/__init__.py +0 -0
- agent_env/utils/card_naming.py +108 -0
- agent_env/utils/deprecation.py +109 -0
- agent_env/utils/docker_build.py +29 -0
- agent_env/utils/exec_retry.py +77 -0
- agent_env/utils/litellm_attribution.py +59 -0
- agent_env/utils/paths.py +20 -0
- agentenv_framework-0.9.1254.dist-info/METADATA +394 -0
- agentenv_framework-0.9.1254.dist-info/RECORD +312 -0
- agentenv_framework-0.9.1254.dist-info/WHEEL +4 -0
- agentenv_framework-0.9.1254.dist-info/entry_points.txt +5 -0
- agentenv_framework-0.9.1254.dist-info/licenses/LICENSE +202 -0
- agentenv_framework-0.9.1254.dist-info/licenses/NOTICE +4 -0
- agentenv_framework-0.9.1254.dist-info/licenses/THIRD_PARTY_NOTICES.md +7665 -0
|
@@ -0,0 +1,293 @@
|
|
|
1
|
+
"""AWS Secrets Manager secret store backend."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import contextlib
|
|
6
|
+
import logging
|
|
7
|
+
import threading
|
|
8
|
+
import time
|
|
9
|
+
from collections.abc import Iterator, ItemsView, KeysView, Mapping, ValuesView
|
|
10
|
+
from typing import Any, Optional
|
|
11
|
+
|
|
12
|
+
import boto3
|
|
13
|
+
import yaml
|
|
14
|
+
from botocore.config import Config as BotocoreConfig
|
|
15
|
+
|
|
16
|
+
from agent_env.store.secret_store.secret_store import SecretStore
|
|
17
|
+
|
|
18
|
+
logger = logging.getLogger(__name__)
|
|
19
|
+
|
|
20
|
+
# GetSecretValue is $0.05/10k calls: a handful of pods on a 5-minute TTL is cents per month.
|
|
21
|
+
_DEFAULT_TTL_SECONDS = 300.0
|
|
22
|
+
|
|
23
|
+
# Floor between forced refresh() calls; doubles as the retry backoff after a failed re-fetch.
|
|
24
|
+
_DEFAULT_MIN_REFRESH_INTERVAL = 10.0
|
|
25
|
+
|
|
26
|
+
# Re-fetches run on the read path: fail in seconds, not botocore's default 60s x retries.
|
|
27
|
+
_BOTO_CONFIG = BotocoreConfig(
|
|
28
|
+
connect_timeout=5, read_timeout=10, retries={"max_attempts": 3, "mode": "standard"}
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class _BundleView(Mapping):
|
|
33
|
+
"""Live read-only view of the store's bundle: consumers that memoize ``_load()``'s
|
|
34
|
+
return for the process lifetime (``Config._get_secret()`` does) still observe TTL
|
|
35
|
+
re-fetches. Accessors bind to one snapshot each; not json/pickle/deepcopy-serializable.
|
|
36
|
+
"""
|
|
37
|
+
|
|
38
|
+
def __init__(self, store: "AwsSecretsManagerSecretStore") -> None:
|
|
39
|
+
self._store = store
|
|
40
|
+
|
|
41
|
+
def __getitem__(self, key: str) -> Any:
|
|
42
|
+
return self._store._current()[key]
|
|
43
|
+
|
|
44
|
+
def __iter__(self) -> Iterator[str]:
|
|
45
|
+
return iter(self._store._current())
|
|
46
|
+
|
|
47
|
+
def __len__(self) -> int:
|
|
48
|
+
return len(self._store._current())
|
|
49
|
+
|
|
50
|
+
def get(self, key: str, default: Any = None) -> Any:
|
|
51
|
+
return self._store._current().get(key, default)
|
|
52
|
+
|
|
53
|
+
def keys(self) -> KeysView:
|
|
54
|
+
return self._store._current().keys()
|
|
55
|
+
|
|
56
|
+
def items(self) -> ItemsView:
|
|
57
|
+
return self._store._current().items()
|
|
58
|
+
|
|
59
|
+
def values(self) -> ValuesView:
|
|
60
|
+
return self._store._current().values()
|
|
61
|
+
|
|
62
|
+
def __eq__(self, other: Any) -> bool:
|
|
63
|
+
return self._store._current() == other
|
|
64
|
+
|
|
65
|
+
def __repr__(self) -> str: # never render values
|
|
66
|
+
return f"<_BundleView of {self._store._secret_name!r}>"
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
#: Shared by every caller, because the loggers being changed are process-global.
|
|
70
|
+
_suppress_guard = threading.Lock()
|
|
71
|
+
_suppress_depth = 0
|
|
72
|
+
_suppress_saved: list = []
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
@contextlib.contextmanager
|
|
76
|
+
def suppress_aws_body_logging():
|
|
77
|
+
"""Keep an AWS response body out of the logs for the duration of a call.
|
|
78
|
+
|
|
79
|
+
botocore logs full HTTP request and response bodies at DEBUG. For most calls
|
|
80
|
+
that is a useful trace; for ``GetSecretValue`` the response body *is* the
|
|
81
|
+
secret, so any process that turns on DEBUG logging — a developer debugging an
|
|
82
|
+
unrelated timeout, a verbose CI job — writes the entire bundle to stdout in
|
|
83
|
+
plaintext. That has happened.
|
|
84
|
+
|
|
85
|
+
Raising the level only around the fetch is deliberate: it does not fight the
|
|
86
|
+
caller's logging config, and it does not silence botocore anywhere else. The
|
|
87
|
+
previous level is always restored, including on the error path, because the
|
|
88
|
+
exception carrying us out may itself need to be debugged.
|
|
89
|
+
|
|
90
|
+
Reference-counted, because the loggers are process-global and the fetches
|
|
91
|
+
are not serialised against each other — the secret store guards itself with
|
|
92
|
+
its own lock, and any other caller reaching Secrets Manager directly is a
|
|
93
|
+
different object with a different lock. Without the count, two overlapping
|
|
94
|
+
fetches restore DEBUG when the *first* finishes, and the second one's
|
|
95
|
+
``SecretString`` goes straight to the handlers. Only the outermost context
|
|
96
|
+
restores, and only it captured the levels worth restoring to.
|
|
97
|
+
"""
|
|
98
|
+
global _suppress_depth, _suppress_saved
|
|
99
|
+
|
|
100
|
+
loggers = [logging.getLogger(name) for name in
|
|
101
|
+
("botocore.parsers", "botocore.endpoint", "botocore.httpsession",
|
|
102
|
+
"botocore.awsrequest", "urllib3.connectionpool")]
|
|
103
|
+
|
|
104
|
+
with _suppress_guard:
|
|
105
|
+
if _suppress_depth == 0:
|
|
106
|
+
_suppress_saved = [(lg, lg.level) for lg in loggers]
|
|
107
|
+
for lg in loggers:
|
|
108
|
+
if lg.level < logging.INFO:
|
|
109
|
+
lg.setLevel(logging.INFO)
|
|
110
|
+
_suppress_depth += 1
|
|
111
|
+
try:
|
|
112
|
+
yield
|
|
113
|
+
finally:
|
|
114
|
+
with _suppress_guard:
|
|
115
|
+
_suppress_depth -= 1
|
|
116
|
+
if _suppress_depth == 0:
|
|
117
|
+
for lg, level in _suppress_saved:
|
|
118
|
+
lg.setLevel(level)
|
|
119
|
+
_suppress_saved = []
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
class AwsSecretsManagerSecretStore(SecretStore):
|
|
123
|
+
"""Reads secrets from one AWS Secrets Manager secret holding a YAML/JSON mapping
|
|
124
|
+
(a combined bundle secret).
|
|
125
|
+
|
|
126
|
+
The bundle is cached (thread-safe) and re-fetched once older than ``ttl_seconds``
|
|
127
|
+
(0 = cache for the process lifetime); ``refresh()`` forces a re-fetch, rate-limited
|
|
128
|
+
to one per ``min_refresh_interval``. A failed TTL re-fetch serves the last-good
|
|
129
|
+
bundle; only the first-ever fetch and an explicit ``refresh()`` raise.
|
|
130
|
+
"""
|
|
131
|
+
|
|
132
|
+
def __init__(
|
|
133
|
+
self,
|
|
134
|
+
secret_name: str,
|
|
135
|
+
region_name: str,
|
|
136
|
+
ttl_seconds: float = _DEFAULT_TTL_SECONDS,
|
|
137
|
+
min_refresh_interval: float = _DEFAULT_MIN_REFRESH_INTERVAL,
|
|
138
|
+
) -> None:
|
|
139
|
+
self._secret_name = secret_name
|
|
140
|
+
self._region_name = region_name
|
|
141
|
+
self._ttl_seconds = float(ttl_seconds)
|
|
142
|
+
self._min_refresh_interval = float(min_refresh_interval)
|
|
143
|
+
self._values: Optional[dict] = None
|
|
144
|
+
self._fetched_at: Optional[float] = None
|
|
145
|
+
self._last_forced_refresh_at: Optional[float] = None
|
|
146
|
+
self._last_failed_fetch_at: Optional[float] = None
|
|
147
|
+
self._lock = threading.Lock()
|
|
148
|
+
self._view = _BundleView(self)
|
|
149
|
+
|
|
150
|
+
@classmethod
|
|
151
|
+
def from_config(
|
|
152
|
+
cls,
|
|
153
|
+
*,
|
|
154
|
+
secret_name: str,
|
|
155
|
+
region: str,
|
|
156
|
+
ttl_seconds: float = _DEFAULT_TTL_SECONDS,
|
|
157
|
+
min_refresh_interval: float = _DEFAULT_MIN_REFRESH_INTERVAL,
|
|
158
|
+
) -> AwsSecretsManagerSecretStore:
|
|
159
|
+
"""Build from a ``[stores.secret.config]`` table (literal values only)."""
|
|
160
|
+
return cls(
|
|
161
|
+
secret_name,
|
|
162
|
+
region,
|
|
163
|
+
ttl_seconds=ttl_seconds,
|
|
164
|
+
min_refresh_interval=min_refresh_interval,
|
|
165
|
+
)
|
|
166
|
+
|
|
167
|
+
def _fetch(self) -> dict:
|
|
168
|
+
"""Fetch and parse the bundle. Caller holds ``self._lock``."""
|
|
169
|
+
client = boto3.client(
|
|
170
|
+
"secretsmanager", region_name=self._region_name, config=_BOTO_CONFIG
|
|
171
|
+
)
|
|
172
|
+
with suppress_aws_body_logging():
|
|
173
|
+
response = client.get_secret_value(SecretId=self._secret_name)
|
|
174
|
+
if "SecretString" not in response:
|
|
175
|
+
raise ValueError(
|
|
176
|
+
f"AWS secret {self._secret_name!r} holds binary data; expected a YAML/JSON mapping"
|
|
177
|
+
)
|
|
178
|
+
try:
|
|
179
|
+
loaded = yaml.load(response["SecretString"], Loader=yaml.BaseLoader) or {}
|
|
180
|
+
except yaml.YAMLError as exc:
|
|
181
|
+
# YAML error marks embed the raw secret line: re-raise sanitized, context dropped.
|
|
182
|
+
mark = getattr(exc, "problem_mark", None)
|
|
183
|
+
where = f" at line {mark.line + 1}, column {mark.column + 1}" if mark else ""
|
|
184
|
+
raise ValueError(
|
|
185
|
+
f"AWS secret {self._secret_name!r} is not valid YAML/JSON{where}"
|
|
186
|
+
) from None
|
|
187
|
+
if not isinstance(loaded, dict):
|
|
188
|
+
raise ValueError(
|
|
189
|
+
f"AWS secret {self._secret_name!r} must be a YAML/JSON "
|
|
190
|
+
f"mapping, got {type(loaded).__name__}"
|
|
191
|
+
)
|
|
192
|
+
return loaded
|
|
193
|
+
|
|
194
|
+
def _needs_fetch(self) -> bool:
|
|
195
|
+
if self._values is None or self._fetched_at is None:
|
|
196
|
+
return True
|
|
197
|
+
if self._ttl_seconds <= 0:
|
|
198
|
+
return False # TTL disabled: cache for the process lifetime
|
|
199
|
+
if (time.monotonic() - self._fetched_at) < self._ttl_seconds:
|
|
200
|
+
return False
|
|
201
|
+
# Outage backoff: serve stale without re-probing AWS on every read.
|
|
202
|
+
if self._last_failed_fetch_at is not None and (
|
|
203
|
+
time.monotonic() - self._last_failed_fetch_at
|
|
204
|
+
) < self._min_refresh_interval:
|
|
205
|
+
return False
|
|
206
|
+
return True
|
|
207
|
+
|
|
208
|
+
def _current(self) -> dict:
|
|
209
|
+
"""Freshest available bundle. Only the first-ever load blocks: a reader that
|
|
210
|
+
finds a re-fetch already in flight serves the stale bundle immediately."""
|
|
211
|
+
if self._needs_fetch():
|
|
212
|
+
first_load = self._values is None
|
|
213
|
+
acquired = self._lock.acquire(blocking=first_load)
|
|
214
|
+
if acquired:
|
|
215
|
+
try:
|
|
216
|
+
if self._needs_fetch():
|
|
217
|
+
self._refetch_locked()
|
|
218
|
+
finally:
|
|
219
|
+
self._lock.release()
|
|
220
|
+
assert self._values is not None # first fetch either populated it or raised
|
|
221
|
+
return self._values
|
|
222
|
+
|
|
223
|
+
def _refetch_locked(self) -> None:
|
|
224
|
+
"""One TTL re-fetch attempt. Caller holds ``self._lock``."""
|
|
225
|
+
try:
|
|
226
|
+
values = self._fetch()
|
|
227
|
+
except ValueError:
|
|
228
|
+
if self._values is None:
|
|
229
|
+
raise
|
|
230
|
+
self._last_failed_fetch_at = time.monotonic()
|
|
231
|
+
# Config rot, not an outage — escalate so monitoring can tell them apart.
|
|
232
|
+
logger.error(
|
|
233
|
+
"AWS secret %r was rotated to invalid content; serving the cached "
|
|
234
|
+
"bundle (age %.0fs) until the secret is fixed.",
|
|
235
|
+
self._secret_name,
|
|
236
|
+
time.monotonic() - (self._fetched_at or 0.0),
|
|
237
|
+
exc_info=True,
|
|
238
|
+
)
|
|
239
|
+
except Exception:
|
|
240
|
+
if self._values is None:
|
|
241
|
+
raise
|
|
242
|
+
self._last_failed_fetch_at = time.monotonic()
|
|
243
|
+
logger.warning(
|
|
244
|
+
"Re-fetching AWS secret %r failed; serving the cached bundle "
|
|
245
|
+
"(age %.0fs). Will retry in %.0fs.",
|
|
246
|
+
self._secret_name,
|
|
247
|
+
time.monotonic() - (self._fetched_at or 0.0),
|
|
248
|
+
self._min_refresh_interval,
|
|
249
|
+
exc_info=True,
|
|
250
|
+
)
|
|
251
|
+
else:
|
|
252
|
+
# Post-fetch clock read: the TTL measures from arrival, not request send.
|
|
253
|
+
self._values = values
|
|
254
|
+
self._fetched_at = time.monotonic()
|
|
255
|
+
self._last_failed_fetch_at = None
|
|
256
|
+
|
|
257
|
+
def _load(self) -> Mapping:
|
|
258
|
+
"""The bundle as a live view. Primes eagerly so a first-ever fetch failure
|
|
259
|
+
raises at the call site, exactly as it did pre-TTL."""
|
|
260
|
+
self._current()
|
|
261
|
+
return self._view
|
|
262
|
+
|
|
263
|
+
def refresh(self) -> Mapping:
|
|
264
|
+
"""Force a re-fetch (rate-limited to one per ``min_refresh_interval``) so a key
|
|
265
|
+
added to AWS resolves on the first request that references it. Raises on
|
|
266
|
+
failure — the caller asked for fresh data — leaving the cached bundle intact."""
|
|
267
|
+
with self._lock:
|
|
268
|
+
now = time.monotonic()
|
|
269
|
+
within_cooldown = (
|
|
270
|
+
self._last_forced_refresh_at is not None
|
|
271
|
+
and (now - self._last_forced_refresh_at) < self._min_refresh_interval
|
|
272
|
+
)
|
|
273
|
+
if self._values is not None and within_cooldown:
|
|
274
|
+
return self._view
|
|
275
|
+
# Cooldown and TTL are stamped only after a successful fetch, off a post-fetch
|
|
276
|
+
# clock read: a failed attempt must not suppress its own retry, and a fetch
|
|
277
|
+
# slower than the cooldown must not hand back an already-expired one.
|
|
278
|
+
try:
|
|
279
|
+
values = self._fetch()
|
|
280
|
+
except Exception:
|
|
281
|
+
if self._values is not None:
|
|
282
|
+
self._last_failed_fetch_at = time.monotonic() # arm the read-path backoff
|
|
283
|
+
raise
|
|
284
|
+
fetched_at = time.monotonic()
|
|
285
|
+
self._values = values
|
|
286
|
+
self._last_forced_refresh_at = fetched_at
|
|
287
|
+
self._fetched_at = fetched_at
|
|
288
|
+
self._last_failed_fetch_at = None
|
|
289
|
+
return self._view
|
|
290
|
+
|
|
291
|
+
def get(self, name: str) -> str | None:
|
|
292
|
+
value = self._current().get(name)
|
|
293
|
+
return None if value is None else str(value)
|
|
@@ -0,0 +1,319 @@
|
|
|
1
|
+
"""Google Cloud Secret Manager secret store backend (the ``gcp`` extra)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import contextlib
|
|
6
|
+
import logging
|
|
7
|
+
import threading
|
|
8
|
+
import time
|
|
9
|
+
from collections.abc import Iterator, ItemsView, KeysView, Mapping, ValuesView
|
|
10
|
+
from typing import Any, Optional
|
|
11
|
+
|
|
12
|
+
import google.auth
|
|
13
|
+
import yaml
|
|
14
|
+
from google.api_core.retry import Retry, if_transient_error
|
|
15
|
+
from google.auth.exceptions import DefaultCredentialsError
|
|
16
|
+
from google.cloud import secretmanager
|
|
17
|
+
|
|
18
|
+
from agent_env.config.errors import ConfigError
|
|
19
|
+
from agent_env.store import _google
|
|
20
|
+
from agent_env.store.secret_store.secret_store import SecretStore
|
|
21
|
+
|
|
22
|
+
logger = logging.getLogger(__name__)
|
|
23
|
+
|
|
24
|
+
# A rotation reaches every process within five minutes, at one access per process each time.
|
|
25
|
+
_DEFAULT_TTL_SECONDS = 300.0
|
|
26
|
+
|
|
27
|
+
# Floor between forced refresh() calls; doubles as the retry backoff after a failed re-fetch.
|
|
28
|
+
_DEFAULT_MIN_REFRESH_INTERVAL = 10.0
|
|
29
|
+
|
|
30
|
+
# Re-fetches run on the read path: fail in seconds, not after the client's minute of retries.
|
|
31
|
+
_RETRY = Retry(predicate=if_transient_error, initial=0.5, maximum=4.0, timeout=15.0)
|
|
32
|
+
_TIMEOUT_SECONDS = 10.0
|
|
33
|
+
|
|
34
|
+
# The client library logs request and response payloads at DEBUG; the payload is the bundle.
|
|
35
|
+
# The transport's logger is named as well as its package because the SDK's logging scope
|
|
36
|
+
# (GOOGLE_SDK_PYTHON_LOGGING_SCOPE) can set DEBUG on it directly, which its package's level
|
|
37
|
+
# would not override.
|
|
38
|
+
_PAYLOAD_LOGGERS = (
|
|
39
|
+
"google.cloud.secretmanager_v1",
|
|
40
|
+
"google.cloud.secretmanager_v1.services.secret_manager_service.transports.rest",
|
|
41
|
+
"google.api_core",
|
|
42
|
+
"google.auth",
|
|
43
|
+
"urllib3",
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class _BundleView(Mapping):
|
|
48
|
+
"""Live read-only view of the store's bundle: consumers that memoize ``_load()``'s
|
|
49
|
+
return for the process lifetime (``Config._get_secret()`` does) still observe TTL
|
|
50
|
+
re-fetches. Accessors bind to one snapshot each; not json/pickle/deepcopy-serializable.
|
|
51
|
+
"""
|
|
52
|
+
|
|
53
|
+
def __init__(self, store: "GcpSecretManagerSecretStore") -> None:
|
|
54
|
+
self._store = store
|
|
55
|
+
|
|
56
|
+
def __getitem__(self, key: str) -> Any:
|
|
57
|
+
return self._store._current()[key]
|
|
58
|
+
|
|
59
|
+
def __iter__(self) -> Iterator[str]:
|
|
60
|
+
return iter(self._store._current())
|
|
61
|
+
|
|
62
|
+
def __len__(self) -> int:
|
|
63
|
+
return len(self._store._current())
|
|
64
|
+
|
|
65
|
+
def get(self, key: str, default: Any = None) -> Any:
|
|
66
|
+
return self._store._current().get(key, default)
|
|
67
|
+
|
|
68
|
+
def keys(self) -> KeysView:
|
|
69
|
+
return self._store._current().keys()
|
|
70
|
+
|
|
71
|
+
def items(self) -> ItemsView:
|
|
72
|
+
return self._store._current().items()
|
|
73
|
+
|
|
74
|
+
def values(self) -> ValuesView:
|
|
75
|
+
return self._store._current().values()
|
|
76
|
+
|
|
77
|
+
def __eq__(self, other: Any) -> bool:
|
|
78
|
+
return self._store._current() == other
|
|
79
|
+
|
|
80
|
+
def __repr__(self) -> str: # never render values
|
|
81
|
+
return f"<_BundleView of {self._store._secret_name!r}>"
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
#: Shared by every caller, because the loggers being changed are process-global.
|
|
85
|
+
_suppress_guard = threading.Lock()
|
|
86
|
+
_suppress_depth = 0
|
|
87
|
+
_suppress_saved: list = []
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
@contextlib.contextmanager
|
|
91
|
+
def _suppress_payload_logging() -> Iterator[None]:
|
|
92
|
+
"""Keep a Secret Manager payload out of the logs for the duration of a call.
|
|
93
|
+
|
|
94
|
+
Raising the level only around the fetch does not fight the caller's logging config or
|
|
95
|
+
silence the library anywhere else, and the previous levels are restored on the error
|
|
96
|
+
path too. Reference-counted, because fetches from different stores overlap: only the
|
|
97
|
+
outermost context captured the levels worth restoring, so only it restores them.
|
|
98
|
+
"""
|
|
99
|
+
global _suppress_depth, _suppress_saved
|
|
100
|
+
|
|
101
|
+
loggers = [logging.getLogger(name) for name in _PAYLOAD_LOGGERS]
|
|
102
|
+
with _suppress_guard:
|
|
103
|
+
if _suppress_depth == 0:
|
|
104
|
+
_suppress_saved = [(lg, lg.level) for lg in loggers]
|
|
105
|
+
for lg in loggers:
|
|
106
|
+
if lg.level < logging.INFO:
|
|
107
|
+
lg.setLevel(logging.INFO)
|
|
108
|
+
_suppress_depth += 1
|
|
109
|
+
try:
|
|
110
|
+
yield
|
|
111
|
+
finally:
|
|
112
|
+
with _suppress_guard:
|
|
113
|
+
_suppress_depth -= 1
|
|
114
|
+
if _suppress_depth == 0:
|
|
115
|
+
for lg, level in _suppress_saved:
|
|
116
|
+
lg.setLevel(level)
|
|
117
|
+
_suppress_saved = []
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
class GcpSecretManagerSecretStore(SecretStore):
|
|
121
|
+
"""Reads secrets from one Secret Manager secret version holding a YAML/JSON mapping (a
|
|
122
|
+
combined bundle secret). Authenticates through Application Default Credentials.
|
|
123
|
+
|
|
124
|
+
The bundle is cached (thread-safe) and re-fetched once older than ``ttl_seconds``
|
|
125
|
+
(0 = cache for the process lifetime); ``refresh()`` forces a re-fetch, rate-limited
|
|
126
|
+
to one per ``min_refresh_interval``. A failed TTL re-fetch serves the last-good
|
|
127
|
+
bundle; only the first-ever fetch and an explicit ``refresh()`` raise.
|
|
128
|
+
"""
|
|
129
|
+
|
|
130
|
+
def __init__(
|
|
131
|
+
self,
|
|
132
|
+
secret_name: str,
|
|
133
|
+
project: str | None = None,
|
|
134
|
+
version: str = "latest",
|
|
135
|
+
ttl_seconds: float = _DEFAULT_TTL_SECONDS,
|
|
136
|
+
min_refresh_interval: float = _DEFAULT_MIN_REFRESH_INTERVAL,
|
|
137
|
+
) -> None:
|
|
138
|
+
self._secret_name = secret_name
|
|
139
|
+
self._project = project
|
|
140
|
+
self._version = version
|
|
141
|
+
self._ttl_seconds = float(ttl_seconds)
|
|
142
|
+
self._min_refresh_interval = float(min_refresh_interval)
|
|
143
|
+
self._client: secretmanager.SecretManagerServiceClient | None = None
|
|
144
|
+
self._values: Optional[dict] = None
|
|
145
|
+
self._fetched_at: Optional[float] = None
|
|
146
|
+
self._last_forced_refresh_at: Optional[float] = None
|
|
147
|
+
self._last_failed_fetch_at: Optional[float] = None
|
|
148
|
+
self._lock = threading.Lock()
|
|
149
|
+
self._view = _BundleView(self)
|
|
150
|
+
|
|
151
|
+
@classmethod
|
|
152
|
+
def from_config(
|
|
153
|
+
cls,
|
|
154
|
+
*,
|
|
155
|
+
secret_name: str,
|
|
156
|
+
project: str | None = None,
|
|
157
|
+
version: str = "latest",
|
|
158
|
+
ttl_seconds: float = _DEFAULT_TTL_SECONDS,
|
|
159
|
+
min_refresh_interval: float = _DEFAULT_MIN_REFRESH_INTERVAL,
|
|
160
|
+
) -> GcpSecretManagerSecretStore:
|
|
161
|
+
"""Build from a ``[stores.secret.config]`` table (literal values only). ``project``
|
|
162
|
+
defaults to the one Application Default Credentials resolve; ``version`` to the latest."""
|
|
163
|
+
return cls(
|
|
164
|
+
secret_name,
|
|
165
|
+
project=project,
|
|
166
|
+
version=version,
|
|
167
|
+
ttl_seconds=ttl_seconds,
|
|
168
|
+
min_refresh_interval=min_refresh_interval,
|
|
169
|
+
)
|
|
170
|
+
|
|
171
|
+
def _fetch(self) -> dict:
|
|
172
|
+
"""Fetch and parse the bundle. Caller holds ``self._lock``."""
|
|
173
|
+
client = self._connect()
|
|
174
|
+
name = client.secret_version_path(self._project, self._secret_name, self._version)
|
|
175
|
+
with _suppress_payload_logging():
|
|
176
|
+
response = client.access_secret_version(name=name, retry=_RETRY, timeout=_TIMEOUT_SECONDS)
|
|
177
|
+
data = response.payload.data
|
|
178
|
+
try:
|
|
179
|
+
text = data.decode("utf-8")
|
|
180
|
+
except UnicodeDecodeError:
|
|
181
|
+
raise ValueError(
|
|
182
|
+
f"Secret Manager secret {self._secret_name!r} holds binary data; expected a YAML/JSON mapping"
|
|
183
|
+
) from None
|
|
184
|
+
try:
|
|
185
|
+
loaded = yaml.load(text, Loader=yaml.BaseLoader) or {}
|
|
186
|
+
except yaml.YAMLError as exc:
|
|
187
|
+
# YAML error marks embed the raw secret line: re-raise sanitized, context dropped.
|
|
188
|
+
mark = getattr(exc, "problem_mark", None)
|
|
189
|
+
where = f" at line {mark.line + 1}, column {mark.column + 1}" if mark else ""
|
|
190
|
+
raise ValueError(
|
|
191
|
+
f"Secret Manager secret {self._secret_name!r} is not valid YAML/JSON{where}"
|
|
192
|
+
) from None
|
|
193
|
+
if not isinstance(loaded, dict):
|
|
194
|
+
raise ValueError(
|
|
195
|
+
f"Secret Manager secret {self._secret_name!r} must be a YAML/JSON "
|
|
196
|
+
f"mapping, got {type(loaded).__name__}"
|
|
197
|
+
)
|
|
198
|
+
return loaded
|
|
199
|
+
|
|
200
|
+
def _connect(self) -> secretmanager.SecretManagerServiceClient:
|
|
201
|
+
"""The client, built on first use. REST rather than gRPC: a process that forks after a
|
|
202
|
+
fetch hands its client to the child, and a gRPC channel does not survive that. A REST
|
|
203
|
+
connection the two then share fails once, and the retry opens a new one."""
|
|
204
|
+
if self._client is None:
|
|
205
|
+
try:
|
|
206
|
+
credentials, default_project = google.auth.default(scopes=_google.SCOPES)
|
|
207
|
+
except (DefaultCredentialsError, OSError) as e:
|
|
208
|
+
raise ConfigError(
|
|
209
|
+
f"No Google credentials for Secret Manager secret {self._secret_name!r}: {e}"
|
|
210
|
+
) from e
|
|
211
|
+
self._project = self._project or default_project
|
|
212
|
+
if not self._project:
|
|
213
|
+
raise ConfigError(
|
|
214
|
+
f"Secret Manager secret {self._secret_name!r} needs a project: set one, "
|
|
215
|
+
"or GOOGLE_CLOUD_PROJECT for the credentials"
|
|
216
|
+
)
|
|
217
|
+
self._client = secretmanager.SecretManagerServiceClient(credentials=credentials, transport="rest")
|
|
218
|
+
return self._client
|
|
219
|
+
|
|
220
|
+
def _needs_fetch(self) -> bool:
|
|
221
|
+
if self._values is None or self._fetched_at is None:
|
|
222
|
+
return True
|
|
223
|
+
if self._ttl_seconds <= 0:
|
|
224
|
+
return False # TTL disabled: cache for the process lifetime
|
|
225
|
+
if (time.monotonic() - self._fetched_at) < self._ttl_seconds:
|
|
226
|
+
return False
|
|
227
|
+
# Outage backoff: serve stale without re-probing Secret Manager on every read.
|
|
228
|
+
if self._last_failed_fetch_at is not None and (
|
|
229
|
+
time.monotonic() - self._last_failed_fetch_at
|
|
230
|
+
) < self._min_refresh_interval:
|
|
231
|
+
return False
|
|
232
|
+
return True
|
|
233
|
+
|
|
234
|
+
def _current(self) -> dict:
|
|
235
|
+
"""Freshest available bundle. Only the first-ever load blocks: a reader that
|
|
236
|
+
finds a re-fetch already in flight serves the stale bundle immediately."""
|
|
237
|
+
if self._needs_fetch():
|
|
238
|
+
first_load = self._values is None
|
|
239
|
+
acquired = self._lock.acquire(blocking=first_load)
|
|
240
|
+
if acquired:
|
|
241
|
+
try:
|
|
242
|
+
if self._needs_fetch():
|
|
243
|
+
self._refetch_locked()
|
|
244
|
+
finally:
|
|
245
|
+
self._lock.release()
|
|
246
|
+
assert self._values is not None # first fetch either populated it or raised
|
|
247
|
+
return self._values
|
|
248
|
+
|
|
249
|
+
def _refetch_locked(self) -> None:
|
|
250
|
+
"""One TTL re-fetch attempt. Caller holds ``self._lock``."""
|
|
251
|
+
try:
|
|
252
|
+
values = self._fetch()
|
|
253
|
+
except ValueError:
|
|
254
|
+
if self._values is None:
|
|
255
|
+
raise
|
|
256
|
+
self._last_failed_fetch_at = time.monotonic()
|
|
257
|
+
# Config rot, not an outage — escalate so monitoring can tell them apart.
|
|
258
|
+
logger.error(
|
|
259
|
+
"Secret Manager secret %r was rotated to invalid content; serving the cached "
|
|
260
|
+
"bundle (age %.0fs) until the secret is fixed.",
|
|
261
|
+
self._secret_name,
|
|
262
|
+
time.monotonic() - (self._fetched_at or 0.0),
|
|
263
|
+
exc_info=True,
|
|
264
|
+
)
|
|
265
|
+
except Exception:
|
|
266
|
+
if self._values is None:
|
|
267
|
+
raise
|
|
268
|
+
self._last_failed_fetch_at = time.monotonic()
|
|
269
|
+
logger.warning(
|
|
270
|
+
"Re-fetching Secret Manager secret %r failed; serving the cached bundle "
|
|
271
|
+
"(age %.0fs). Will retry in %.0fs.",
|
|
272
|
+
self._secret_name,
|
|
273
|
+
time.monotonic() - (self._fetched_at or 0.0),
|
|
274
|
+
self._min_refresh_interval,
|
|
275
|
+
exc_info=True,
|
|
276
|
+
)
|
|
277
|
+
else:
|
|
278
|
+
# Post-fetch clock read: the TTL measures from arrival, not request send.
|
|
279
|
+
self._values = values
|
|
280
|
+
self._fetched_at = time.monotonic()
|
|
281
|
+
self._last_failed_fetch_at = None
|
|
282
|
+
|
|
283
|
+
def _load(self) -> Mapping:
|
|
284
|
+
"""The bundle as a live view. Primes eagerly so a first-ever fetch failure
|
|
285
|
+
raises at the call site."""
|
|
286
|
+
self._current()
|
|
287
|
+
return self._view
|
|
288
|
+
|
|
289
|
+
def refresh(self) -> Mapping:
|
|
290
|
+
"""Force a re-fetch (rate-limited to one per ``min_refresh_interval``) so a key
|
|
291
|
+
added to the secret resolves on the first request that references it. Raises on
|
|
292
|
+
failure — the caller asked for fresh data — leaving the cached bundle intact."""
|
|
293
|
+
with self._lock:
|
|
294
|
+
now = time.monotonic()
|
|
295
|
+
within_cooldown = (
|
|
296
|
+
self._last_forced_refresh_at is not None
|
|
297
|
+
and (now - self._last_forced_refresh_at) < self._min_refresh_interval
|
|
298
|
+
)
|
|
299
|
+
if self._values is not None and within_cooldown:
|
|
300
|
+
return self._view
|
|
301
|
+
# Cooldown and TTL are stamped only after a successful fetch, off a post-fetch
|
|
302
|
+
# clock read: a failed attempt must not suppress its own retry, and a fetch
|
|
303
|
+
# slower than the cooldown must not hand back an already-expired one.
|
|
304
|
+
try:
|
|
305
|
+
values = self._fetch()
|
|
306
|
+
except Exception:
|
|
307
|
+
if self._values is not None:
|
|
308
|
+
self._last_failed_fetch_at = time.monotonic() # arm the read-path backoff
|
|
309
|
+
raise
|
|
310
|
+
fetched_at = time.monotonic()
|
|
311
|
+
self._values = values
|
|
312
|
+
self._last_forced_refresh_at = fetched_at
|
|
313
|
+
self._fetched_at = fetched_at
|
|
314
|
+
self._last_failed_fetch_at = None
|
|
315
|
+
return self._view
|
|
316
|
+
|
|
317
|
+
def get(self, name: str) -> str | None:
|
|
318
|
+
value = self._current().get(name)
|
|
319
|
+
return None if value is None else str(value)
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
"""Local secret store backend — env vars + an optional YAML/JSON file, no cloud services."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
from typing import Mapping, Optional
|
|
7
|
+
|
|
8
|
+
import yaml
|
|
9
|
+
|
|
10
|
+
from agent_env.store.secret_store.secret_store import SecretStore
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class LocalSecretStore(SecretStore):
|
|
14
|
+
"""No-AWS secret backend: process env vars (checked first when ``use_env``) over an
|
|
15
|
+
optional flat YAML/JSON ``file_path`` mapping of ``name -> value``."""
|
|
16
|
+
|
|
17
|
+
def __init__(
|
|
18
|
+
self,
|
|
19
|
+
values: Optional[Mapping[str, object]] = None,
|
|
20
|
+
*,
|
|
21
|
+
file_path: Optional[str] = None,
|
|
22
|
+
use_env: bool = True,
|
|
23
|
+
) -> None:
|
|
24
|
+
merged: dict[str, object] = {}
|
|
25
|
+
if file_path:
|
|
26
|
+
if not os.path.isfile(file_path):
|
|
27
|
+
raise ValueError(f"Secret file {file_path!r} does not exist.")
|
|
28
|
+
with open(file_path) as f:
|
|
29
|
+
loaded = yaml.safe_load(f) or {}
|
|
30
|
+
if not isinstance(loaded, Mapping):
|
|
31
|
+
raise ValueError(f"Secret file {file_path!r} must be a flat mapping of name -> value.")
|
|
32
|
+
merged.update(loaded)
|
|
33
|
+
if values:
|
|
34
|
+
merged.update(values)
|
|
35
|
+
self._values = merged
|
|
36
|
+
self._use_env = use_env
|
|
37
|
+
|
|
38
|
+
def get(self, name: str) -> str | None:
|
|
39
|
+
if self._use_env and name in os.environ:
|
|
40
|
+
return os.environ[name]
|
|
41
|
+
value = self._values.get(name)
|
|
42
|
+
return None if value is None else str(value)
|
|
43
|
+
|
|
44
|
+
def _load(self) -> dict:
|
|
45
|
+
"""The name→value mapping this store was built from (file + values, no env), for
|
|
46
|
+
the whole-bundle reader ``Config._get_secret`` (callers apply their own env overrides)."""
|
|
47
|
+
return {name: str(value) for name, value in self._values.items()}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"""Backend-agnostic secret store abstraction over string-keyed secrets."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from abc import ABC, abstractmethod
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class SecretStore(ABC):
|
|
9
|
+
"""Backend-selectable secret source, selected by ``AGENT_ENV_SECRET_STORE`` /
|
|
10
|
+
``[stores.secret]`` — local env/file (default), AWS Secrets Manager, or a custom backend."""
|
|
11
|
+
|
|
12
|
+
@classmethod
|
|
13
|
+
def from_config(cls, **config) -> SecretStore:
|
|
14
|
+
"""Construct from a resolved config table; backends that build a client override this."""
|
|
15
|
+
return cls(**config)
|
|
16
|
+
|
|
17
|
+
@abstractmethod
|
|
18
|
+
def get(self, name: str) -> str | None:
|
|
19
|
+
"""The secret value for ``name``, or ``None`` if it is not present."""
|