agentenv-framework 0.9.1254__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agent_env/__init__.py +0 -0
- agent_env/a2a_agent/__init__.py +23 -0
- agent_env/a2a_agent/a2a_agent.py +600 -0
- agent_env/a2a_agent/conversation_store.py +356 -0
- agent_env/a2a_agent/object_transfer.py +609 -0
- agent_env/a2a_agent/protocol.py +238 -0
- agent_env/a2a_agent/store.py +201 -0
- agent_env/a2a_agent/validator.py +901 -0
- agent_env/artifact/__init__.py +48 -0
- agent_env/artifact/artifact.py +74 -0
- agent_env/artifact/artifacts/__init__.py +0 -0
- agent_env/artifact/artifacts/cli.py +83 -0
- agent_env/artifact/artifacts/docker_image.py +457 -0
- agent_env/artifact/artifacts/environment.py +89 -0
- agent_env/artifact/artifacts/environment_universe.py +156 -0
- agent_env/artifact/artifacts/file.py +214 -0
- agent_env/artifact/artifacts/file_artifact_universe.py +206 -0
- agent_env/artifact/artifacts/skill.py +206 -0
- agent_env/artifact/artifacts/vm_image.py +95 -0
- agent_env/artifact/ref.py +15 -0
- agent_env/artifact/registry.py +251 -0
- agent_env/artifact/store.py +295 -0
- agent_env/artifact/universe.py +38 -0
- agent_env/attribution.py +41 -0
- agent_env/bundle/__init__.py +4 -0
- agent_env/bundle/_fs.py +128 -0
- agent_env/bundle/authoring.py +239 -0
- agent_env/bundle/installed.py +175 -0
- agent_env/bundle/ledger.py +269 -0
- agent_env/bundle/materialize.py +204 -0
- agent_env/bundle/parse.py +594 -0
- agent_env/bundle/plan.py +414 -0
- agent_env/bundle/resolve.py +404 -0
- agent_env/bundle/run.py +361 -0
- agent_env/cli/__init__.py +69 -0
- agent_env/cli/__main__.py +10 -0
- agent_env/cli/_installers.py +576 -0
- agent_env/cli/_plugin_changes.py +947 -0
- agent_env/cli/a2a_agent/__init__.py +22 -0
- agent_env/cli/a2a_agent/add_skill.py +97 -0
- agent_env/cli/a2a_agent/deploy.py +60 -0
- agent_env/cli/a2a_agent/get.py +15 -0
- agent_env/cli/a2a_agent/get_instance.py +26 -0
- agent_env/cli/a2a_agent/put.py +78 -0
- agent_env/cli/a2a_agent/validate.py +56 -0
- agent_env/cli/artifact/__init__.py +20 -0
- agent_env/cli/artifact/cli.py +83 -0
- agent_env/cli/artifact/environment.py +44 -0
- agent_env/cli/artifact/environment_universe.py +163 -0
- agent_env/cli/artifact/file_artifact_universe.py +281 -0
- agent_env/cli/artifact/skill.py +128 -0
- agent_env/cli/banner.py +25 -0
- agent_env/cli/config.py +263 -0
- agent_env/cli/env/__init__.py +31 -0
- agent_env/cli/env/deploy.py +112 -0
- agent_env/cli/env/gateway.py +57 -0
- agent_env/cli/env/get_instance.py +40 -0
- agent_env/cli/env/mcp_server.py +410 -0
- agent_env/cli/env/multi.py +282 -0
- agent_env/cli/env/service_db.py +99 -0
- agent_env/cli/env/snapshot.py +35 -0
- agent_env/cli/env/state/__init__.py +6 -0
- agent_env/cli/env/state/init.py +108 -0
- agent_env/cli/env/state/teardown.py +44 -0
- agent_env/cli/env/website.py +216 -0
- agent_env/cli/env/website_browser.py +67 -0
- agent_env/cli/eval/__init__.py +16 -0
- agent_env/cli/eval/add_tasks.py +63 -0
- agent_env/cli/eval/create.py +49 -0
- agent_env/cli/eval/run.py +194 -0
- agent_env/cli/identity/__init__.py +5 -0
- agent_env/cli/identity/client_identity.py +37 -0
- agent_env/cli/plugin.py +438 -0
- agent_env/cli/run.py +285 -0
- agent_env/cli/task/__init__.py +21 -0
- agent_env/cli/task/create.py +109 -0
- agent_env/cli/task/get.py +17 -0
- agent_env/cli/task/get_instance.py +34 -0
- agent_env/cli/task/run.py +615 -0
- agent_env/cli/task/validate.py +31 -0
- agent_env/cli/up.py +123 -0
- agent_env/cli/utils.py +183 -0
- agent_env/config/__init__.py +50 -0
- agent_env/config/describe.py +1067 -0
- agent_env/config/errors.py +13 -0
- agent_env/config/loader.py +245 -0
- agent_env/config/model.py +154 -0
- agent_env/config/paths.py +25 -0
- agent_env/config/plugin_tables.py +101 -0
- agent_env/config/provenance.py +134 -0
- agent_env/config/runtime.py +1064 -0
- agent_env/config/snapshot.py +81 -0
- agent_env/entity_refs.py +186 -0
- agent_env/env/__init__.py +49 -0
- agent_env/env/env.py +433 -0
- agent_env/env/env_artifact_store.py +87 -0
- agent_env/env/envs/__init__.py +7 -0
- agent_env/env/envs/_deployment.py +281 -0
- agent_env/env/envs/gateway_server.py +32 -0
- agent_env/env/envs/mcp_server.py +387 -0
- agent_env/env/envs/multi_env.py +780 -0
- agent_env/env/envs/service_db/Dockerfile +20 -0
- agent_env/env/envs/service_db/Dockerfile.db-mcp +42 -0
- agent_env/env/envs/service_db/Dockerfile.db-web +8 -0
- agent_env/env/envs/service_db.py +92 -0
- agent_env/env/envs/website.py +294 -0
- agent_env/env/envs/website_browser/Dockerfile +40 -0
- agent_env/env/envs/website_browser/__init__.py +17 -0
- agent_env/env/envs/website_browser/entrypoint.sh +13 -0
- agent_env/env/gateway/Dockerfile +14 -0
- agent_env/env/gateway/__init__.py +37 -0
- agent_env/env/gateway/clock.py +184 -0
- agent_env/env/gateway/constants.py +190 -0
- agent_env/env/gateway/entrypoint.py +156 -0
- agent_env/env/gateway/gateway.py +1095 -0
- agent_env/env/gateway/get_time.py +60 -0
- agent_env/env/gateway/requirements.txt +4 -0
- agent_env/env/gateway/triggers.py +1193 -0
- agent_env/env/legacy_protocol.py +146 -0
- agent_env/env/registry.py +93 -0
- agent_env/env/snapshot_store.py +372 -0
- agent_env/env/store.py +411 -0
- agent_env/eval/__init__.py +14 -0
- agent_env/eval/eval.py +68 -0
- agent_env/eval/store.py +116 -0
- agent_env/examples/hello/README.md +9 -0
- agent_env/examples/hello/artifacts/greeting/check.sh +1 -0
- agent_env/examples/hello/artifacts/greeting/hello.txt +1 -0
- agent_env/examples/hello/tasks/hello.json +10 -0
- agent_env/explorer/__init__.py +12 -0
- agent_env/explorer/app.py +277 -0
- agent_env/explorer/openapi_docs.py +242 -0
- agent_env/explorer/plugin.py +95 -0
- agent_env/explorer/routers/__init__.py +1 -0
- agent_env/explorer/routers/common.py +174 -0
- agent_env/explorer/routers/conversations.py +25 -0
- agent_env/explorer/routers/objects.py +112 -0
- agent_env/explorer/routers/runs.py +475 -0
- agent_env/plugins/__init__.py +59 -0
- agent_env/plugins/_cli.py +244 -0
- agent_env/plugins/_discovery.py +97 -0
- agent_env/plugins/_inventory.py +262 -0
- agent_env/plugins/_registration.py +268 -0
- agent_env/plugins/_report.py +47 -0
- agent_env/plugins/_requirements.py +55 -0
- agent_env/providers/__init__.py +37 -0
- agent_env/providers/env_providers/__init__.py +12 -0
- agent_env/providers/env_providers/constants.py +13 -0
- agent_env/providers/env_providers/env_gateway_provider.py +1354 -0
- agent_env/providers/env_providers/env_provider.py +237 -0
- agent_env/providers/env_providers/env_server_provider.py +133 -0
- agent_env/providers/env_state/__init__.py +49 -0
- agent_env/providers/env_state/env_state_provider.py +554 -0
- agent_env/providers/env_state/local_postgres.py +390 -0
- agent_env/providers/env_state/store.py +112 -0
- agent_env/providers/sandbox_providers/__init__.py +29 -0
- agent_env/providers/sandbox_providers/chained_sandbox_provider.py +93 -0
- agent_env/providers/sandbox_providers/e2b/__init__.py +6 -0
- agent_env/providers/sandbox_providers/e2b/provider.py +376 -0
- agent_env/providers/sandbox_providers/e2b/sandbox.py +342 -0
- agent_env/providers/sandbox_providers/e2b/template.py +158 -0
- agent_env/providers/sandbox_providers/local_sandbox.py +408 -0
- agent_env/providers/sandbox_providers/modal_sandbox.py +500 -0
- agent_env/providers/sandbox_providers/modal_vm_sandbox.py +466 -0
- agent_env/providers/sandbox_providers/sandbox.py +387 -0
- agent_env/providers/sandbox_providers/sandbox_provider.py +614 -0
- agent_env/py.typed +0 -0
- agent_env/runner/__init__.py +16 -0
- agent_env/runner/local_runner.py +190 -0
- agent_env/runner/runner.py +127 -0
- agent_env/runner/store.py +94 -0
- agent_env/store/__init__.py +126 -0
- agent_env/store/_google.py +68 -0
- agent_env/store/base.py +33 -0
- agent_env/store/document_store/__init__.py +60 -0
- agent_env/store/document_store/document_store.py +519 -0
- agent_env/store/document_store/dynamodb_document_store.py +306 -0
- agent_env/store/document_store/evaluation.py +199 -0
- agent_env/store/document_store/firestore_mongo_document_store.py +242 -0
- agent_env/store/document_store/mongo_document_store.py +259 -0
- agent_env/store/document_store/sqlite_document_store.py +314 -0
- agent_env/store/ids.py +123 -0
- agent_env/store/image_store/__init__.py +34 -0
- agent_env/store/image_store/ecr_image_store.py +131 -0
- agent_env/store/image_store/google_credentials.py +113 -0
- agent_env/store/image_store/image_store.py +113 -0
- agent_env/store/image_store/local_registry_image_store.py +106 -0
- agent_env/store/image_store/oci_registry_credentials.py +196 -0
- agent_env/store/local_state.py +27 -0
- agent_env/store/object_store/__init__.py +24 -0
- agent_env/store/object_store/gcs_object_store.py +415 -0
- agent_env/store/object_store/local_object_store.py +143 -0
- agent_env/store/object_store/object_store.py +198 -0
- agent_env/store/object_store/s3_object_store.py +354 -0
- agent_env/store/query.py +112 -0
- agent_env/store/routing.py +609 -0
- agent_env/store/secret_store/__init__.py +19 -0
- agent_env/store/secret_store/aws_secrets_manager_secret_store.py +293 -0
- agent_env/store/secret_store/gcp_secret_manager_secret_store.py +319 -0
- agent_env/store/secret_store/local_secret_store.py +47 -0
- agent_env/store/secret_store/secret_store.py +19 -0
- agent_env/task/__init__.py +27 -0
- agent_env/task/interrupts.py +109 -0
- agent_env/task/registry.py +27 -0
- agent_env/task/step_journal.py +99 -0
- agent_env/task/store.py +940 -0
- agent_env/task/task.py +815 -0
- agent_env/task/teardown.py +142 -0
- agent_env/task_step/__init__.py +47 -0
- agent_env/task_step/context.py +195 -0
- agent_env/task_step/context_ops.py +224 -0
- agent_env/task_step/registry.py +182 -0
- agent_env/task_step/review_store.py +118 -0
- agent_env/task_step/snapshot_utils/__init__.py +2 -0
- agent_env/task_step/snapshot_utils/agent_state_capture.py +261 -0
- agent_env/task_step/snapshot_utils/snapshot_series.py +631 -0
- agent_env/task_step/store.py +129 -0
- agent_env/task_step/task_step.py +173 -0
- agent_env/task_step/task_steps/__init__.py +27 -0
- agent_env/task_step/task_steps/a2a_agent_validator/__init__.py +31 -0
- agent_env/task_step/task_steps/a2a_agent_validator/fixtures/README.md +158 -0
- agent_env/task_step/task_steps/a2a_agent_validator/fixtures/clip.m4a +0 -0
- agent_env/task_step/task_steps/a2a_agent_validator/fixtures/clip.mp3 +0 -0
- agent_env/task_step/task_steps/a2a_agent_validator/fixtures/clip.mp4 +0 -0
- agent_env/task_step/task_steps/a2a_agent_validator/fixtures/clip.ogg +0 -0
- agent_env/task_step/task_steps/a2a_agent_validator/fixtures/clip.wav +0 -0
- agent_env/task_step/task_steps/a2a_agent_validator/fixtures/document.pdf +0 -0
- agent_env/task_step/task_steps/a2a_agent_validator/fixtures/red.gif +0 -0
- agent_env/task_step/task_steps/a2a_agent_validator/fixtures/red.jpg +0 -0
- agent_env/task_step/task_steps/a2a_agent_validator/fixtures/red.png +0 -0
- agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_agent_card.py +103 -0
- agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_agent_config_identity.py +69 -0
- agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_agent_mcp.py +178 -0
- agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_core_protocol.py +67 -0
- agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_install.py +115 -0
- agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_litellm_attribution.py +124 -0
- agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_litellm_attribution_runtime.py +173 -0
- agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_modalities.py +270 -0
- agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_peer_agents.py +102 -0
- agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_role.py +151 -0
- agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_skill_config.py +194 -0
- agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_snapshot.py +111 -0
- agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_system_prompt.py +215 -0
- agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_trajectory.py +164 -0
- agent_env/task_step/task_steps/add_skills.py +240 -0
- agent_env/task_step/task_steps/apply_server_config.py +288 -0
- agent_env/task_step/task_steps/collect_artifacts.py +940 -0
- agent_env/task_step/task_steps/deploy_agent.py +626 -0
- agent_env/task_step/task_steps/deploy_env.py +197 -0
- agent_env/task_step/task_steps/deploy_human_agent.py +100 -0
- agent_env/task_step/task_steps/deploy_sandbox.py +170 -0
- agent_env/task_step/task_steps/env_card_validator/__init__.py +4 -0
- agent_env/task_step/task_steps/env_card_validator/verify_env_card.py +109 -0
- agent_env/task_step/task_steps/env_card_validator/verify_env_core_protocol.py +108 -0
- agent_env/task_step/task_steps/install_agent.py +376 -0
- agent_env/task_step/task_steps/load_artifact.py +775 -0
- agent_env/task_step/task_steps/mcp_cli_builder/__init__.py +4 -0
- agent_env/task_step/task_steps/mcp_cli_builder/build_mcp_cli.py +201 -0
- agent_env/task_step/task_steps/mcp_cli_builder/codegen.py +627 -0
- agent_env/task_step/task_steps/mcp_env_validator/__init__.py +44 -0
- agent_env/task_step/task_steps/mcp_env_validator/validation_gate_aggregator.py +102 -0
- agent_env/task_step/task_steps/mcp_env_validator/verify_mcp_env_assessment.py +78 -0
- agent_env/task_step/task_steps/mcp_env_validator/verify_mcp_tool_schema.py +152 -0
- agent_env/task_step/task_steps/mcp_env_validator/verify_spec_conformance.py +486 -0
- agent_env/task_step/task_steps/modify_env_tool_access.py +80 -0
- agent_env/task_step/task_steps/multienv_validator/__init__.py +3 -0
- agent_env/task_step/task_steps/multienv_validator/combine_universe_verdicts.py +107 -0
- agent_env/task_step/task_steps/multienv_validator/universe_comparison.py +253 -0
- agent_env/task_step/task_steps/multienv_validator/verify_universe_agent_judge.py +180 -0
- agent_env/task_step/task_steps/multienv_validator/verify_universe_roundtrip.py +241 -0
- agent_env/task_step/task_steps/peer_agents.py +92 -0
- agent_env/task_step/task_steps/prompt_agent.py +900 -0
- agent_env/task_step/task_steps/register_agent_triggers.py +113 -0
- agent_env/task_step/task_steps/register_env_triggers.py +97 -0
- agent_env/task_step/task_steps/reset_env.py +71 -0
- agent_env/task_step/task_steps/review.py +152 -0
- agent_env/task_step/task_steps/run_code.py +461 -0
- agent_env/task_step/task_steps/run_code_runner.py +52 -0
- agent_env/task_step/task_steps/run_docker_container.py +380 -0
- agent_env/task_step/task_steps/sandbox_utils/__init__.py +0 -0
- agent_env/task_step/task_steps/sandbox_utils/sandbox_utils.py +79 -0
- agent_env/task_step/task_steps/snapshot_agent_state.py +205 -0
- agent_env/task_step/task_steps/snapshot_env.py +637 -0
- agent_env/task_step/task_steps/sync_env_clock.py +132 -0
- agent_env/task_step/task_steps/teardown_sandboxes.py +170 -0
- agent_env/task_step/task_steps/verifiers/__init__.py +23 -0
- agent_env/task_step/task_steps/verifiers/agent_prompt_response_verifier.py +166 -0
- agent_env/task_step/task_steps/verifiers/aggregate_verifiers.py +90 -0
- agent_env/task_step/task_steps/verifiers/env_outcome_verifier.py +127 -0
- agent_env/task_step/task_steps/verifiers/judge_utils/__init__.py +2 -0
- agent_env/task_step/task_steps/verifiers/judge_utils/frame_selection.py +377 -0
- agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py +1293 -0
- agent_env/task_step/task_steps/verifiers/judge_utils/trajectory_filter.py +514 -0
- agent_env/task_step/task_steps/verifiers/rubrics_verifier.py +1318 -0
- agent_env/task_step/task_steps/verifiers/run_container_unit_tests_verifier.py +434 -0
- agent_env/task_step/task_steps/verifiers/scoring.py +40 -0
- agent_env/task_step/task_steps/verifiers/verify_sandbox.py +308 -0
- agent_env/task_step/thread_work.py +61 -0
- agent_env/utils/__init__.py +0 -0
- agent_env/utils/card_naming.py +108 -0
- agent_env/utils/deprecation.py +109 -0
- agent_env/utils/docker_build.py +29 -0
- agent_env/utils/exec_retry.py +77 -0
- agent_env/utils/litellm_attribution.py +59 -0
- agent_env/utils/paths.py +20 -0
- agentenv_framework-0.9.1254.dist-info/METADATA +394 -0
- agentenv_framework-0.9.1254.dist-info/RECORD +312 -0
- agentenv_framework-0.9.1254.dist-info/WHEEL +4 -0
- agentenv_framework-0.9.1254.dist-info/entry_points.txt +5 -0
- agentenv_framework-0.9.1254.dist-info/licenses/LICENSE +202 -0
- agentenv_framework-0.9.1254.dist-info/licenses/NOTICE +4 -0
- agentenv_framework-0.9.1254.dist-info/licenses/THIRD_PARTY_NOTICES.md +7665 -0
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
"""Legacy (pre-/v1) data-plane wire protocol for MCP server / website envs.
|
|
2
|
+
|
|
3
|
+
Standalone async helpers for the current gateway endpoints: ``POST /api/reset``
|
|
4
|
+
(with an MCP-tool fallback for older servers), ``POST /api/add``, and
|
|
5
|
+
``GET /export-state``. Mirrors the module-level function style of
|
|
6
|
+
``agent_env.a2a_agent.protocol`` — callers pass URLs + params, not ``self``.
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import asyncio
|
|
11
|
+
import logging
|
|
12
|
+
from typing import TYPE_CHECKING, Optional
|
|
13
|
+
|
|
14
|
+
import httpx
|
|
15
|
+
from agentenv_protocol import RPC_PATH, client as protocol_v1
|
|
16
|
+
|
|
17
|
+
if TYPE_CHECKING:
|
|
18
|
+
from agent_env.env.env import DeployedEnv
|
|
19
|
+
|
|
20
|
+
logger = logging.getLogger(__name__)
|
|
21
|
+
|
|
22
|
+
DEFAULT_MCP_MAX_RETRIES = 5
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def environment_base_url(gateway_url: Optional[str], environment_name: str, mcp: bool = True) -> str:
|
|
26
|
+
"""Gateway path for a service: ``/svc/mcp-{name}`` (MCP) or ``/svc/{name}`` (website)."""
|
|
27
|
+
if not gateway_url: # an env deployed without a gateway has no gateway paths
|
|
28
|
+
from agent_env.env.env import EnvNeedsGateway
|
|
29
|
+
|
|
30
|
+
raise EnvNeedsGateway(f"Reaching '{environment_name}' here")
|
|
31
|
+
prefix = "mcp-" if mcp else ""
|
|
32
|
+
return f"{gateway_url}/svc/{prefix}{environment_name}"
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
async def v1_base_url(deployed: Optional[DeployedEnv], gateway_url: Optional[str], environment_name: str, mcp: bool = True) -> Optional[str]:
|
|
36
|
+
"""A child env's v1 data-plane base from the stored env card, or None for legacy; without a stored card, the live probe decides."""
|
|
37
|
+
if deployed is None or not deployed.environment_card:
|
|
38
|
+
base_url = environment_base_url(gateway_url, environment_name, mcp=mcp)
|
|
39
|
+
return base_url if await protocol_v1.supports_v1(base_url) else None
|
|
40
|
+
child = deployed.get_child_env_card(environment_name)
|
|
41
|
+
return deployed.environment_url + child.get("url", RPC_PATH).removesuffix(RPC_PATH) if child else None
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
async def child_env_card(deployed: Optional[DeployedEnv], gateway_url: Optional[str], environment_name: str, timeout: int = 10) -> tuple[str, Optional[dict]]:
|
|
46
|
+
"""An MCP child env's card and the base its endpoints resolve against: from the stored env card, or without one a live read (None on 404)."""
|
|
47
|
+
if deployed is None or not deployed.environment_card:
|
|
48
|
+
base_url = environment_base_url((gateway_url or "").rstrip("/"), environment_name)
|
|
49
|
+
try:
|
|
50
|
+
return base_url, await protocol_v1.get_card(base_url, timeout=timeout)
|
|
51
|
+
except httpx.HTTPStatusError as e:
|
|
52
|
+
if e.response.status_code == 404:
|
|
53
|
+
return base_url, None
|
|
54
|
+
raise
|
|
55
|
+
return deployed.environment_url, deployed.get_child_env_card(environment_name)
|
|
56
|
+
|
|
57
|
+
async def reset_via_rest(
|
|
58
|
+
base_url: str,
|
|
59
|
+
mock_data_path: Optional[str] = None,
|
|
60
|
+
timeout: int = 30,
|
|
61
|
+
verify: bool = True,
|
|
62
|
+
) -> dict:
|
|
63
|
+
"""POST ``{base_url}/api/reset``. Sends the seed path when given. Raises on non-2xx."""
|
|
64
|
+
body = {"mock_data_path": mock_data_path} if mock_data_path is not None else None
|
|
65
|
+
async with httpx.AsyncClient(verify=verify) as client:
|
|
66
|
+
response = await client.post(f"{base_url}/api/reset", json=body, timeout=timeout)
|
|
67
|
+
response.raise_for_status()
|
|
68
|
+
result = response.json()
|
|
69
|
+
logger.info(f"load_environment_artifact: REST reset succeeded: {result}")
|
|
70
|
+
return result
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
async def add_via_rest(base_url: str, file_path: str, timeout: int = 120, verify: bool = True) -> dict:
|
|
74
|
+
"""POST ``{base_url}/api/add`` with a backend file path. Raises on non-2xx."""
|
|
75
|
+
async with httpx.AsyncClient(verify=verify) as client:
|
|
76
|
+
response = await client.post(f"{base_url}/api/add", json={"file_path": file_path}, timeout=timeout)
|
|
77
|
+
response.raise_for_status()
|
|
78
|
+
result = response.json()
|
|
79
|
+
logger.info(f"load_environment_artifact: add response: {result}")
|
|
80
|
+
return result
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
async def export_state(gateway_url: str, environment_name: str, timeout: int = 60, verify: bool = True) -> dict:
|
|
84
|
+
"""GET ``/svc/mcp-{name}/export-state`` — the legacy state snapshot for one service."""
|
|
85
|
+
base_url = environment_base_url(gateway_url, environment_name, mcp=True)
|
|
86
|
+
async with httpx.AsyncClient(verify=verify) as client:
|
|
87
|
+
response = await client.get(f"{base_url}/export-state", timeout=timeout)
|
|
88
|
+
response.raise_for_status()
|
|
89
|
+
return response.json()
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
async def reset_via_mcp_tool(
|
|
93
|
+
mcp_url: str,
|
|
94
|
+
environment_name: str,
|
|
95
|
+
mock_data_path: str,
|
|
96
|
+
max_retries: int = DEFAULT_MCP_MAX_RETRIES,
|
|
97
|
+
) -> None:
|
|
98
|
+
"""Call the ``{environment_name}_reset`` MCP tool, retrying on transient failures."""
|
|
99
|
+
from mcp import ClientSession
|
|
100
|
+
from mcp.client.streamable_http import streamable_http_client
|
|
101
|
+
|
|
102
|
+
for attempt in range(max_retries):
|
|
103
|
+
reset_done = False
|
|
104
|
+
try:
|
|
105
|
+
async with streamable_http_client(mcp_url) as (read_stream, write_stream, _):
|
|
106
|
+
async with ClientSession(read_stream, write_stream) as session:
|
|
107
|
+
await session.initialize()
|
|
108
|
+
reset_tool_name = f"{environment_name}_reset"
|
|
109
|
+
result = await session.call_tool(reset_tool_name, {"mock_data_path": mock_data_path})
|
|
110
|
+
result_text = "".join(c.text for c in result.content if hasattr(c, "text"))
|
|
111
|
+
logger.info(f"load_environment_artifact: {reset_tool_name} returned: {result_text}")
|
|
112
|
+
reset_done = True
|
|
113
|
+
except BaseExceptionGroup as eg:
|
|
114
|
+
if reset_done:
|
|
115
|
+
logger.info(f"load_environment_artifact: succeeded (ignoring cleanup ExceptionGroup)")
|
|
116
|
+
break
|
|
117
|
+
logger.info(f"load_environment_artifact attempt {attempt + 1}/{max_retries} failed: {eg}")
|
|
118
|
+
if attempt < max_retries - 1:
|
|
119
|
+
await asyncio.sleep(5)
|
|
120
|
+
else:
|
|
121
|
+
raise
|
|
122
|
+
except Exception as e:
|
|
123
|
+
logger.info(f"load_environment_artifact attempt {attempt + 1}/{max_retries} failed: {type(e).__name__}")
|
|
124
|
+
if attempt < max_retries - 1:
|
|
125
|
+
await asyncio.sleep(5)
|
|
126
|
+
else:
|
|
127
|
+
raise
|
|
128
|
+
else:
|
|
129
|
+
break
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
async def reset(
|
|
133
|
+
gateway_url: str,
|
|
134
|
+
environment_name: str,
|
|
135
|
+
mock_data_path: str,
|
|
136
|
+
max_retries: int = DEFAULT_MCP_MAX_RETRIES,
|
|
137
|
+
) -> Optional[dict]:
|
|
138
|
+
"""Reset an MCP service to a seed: try REST ``/api/reset``, fall back to the MCP tool."""
|
|
139
|
+
base_url = environment_base_url(gateway_url, environment_name, mcp=True)
|
|
140
|
+
try:
|
|
141
|
+
return await reset_via_rest(base_url, mock_data_path)
|
|
142
|
+
except Exception as rest_err:
|
|
143
|
+
# TODO remove MCP tool fallback once all usages of MCP servers are on REST
|
|
144
|
+
logger.info(f"load_environment_artifact: REST reset failed ({type(rest_err).__name__}), falling back to MCP tool")
|
|
145
|
+
await reset_via_mcp_tool(f"{gateway_url}/mcp", environment_name, mock_data_path, max_retries=max_retries)
|
|
146
|
+
return None
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
"""Env registry for type-based deserialization."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import TYPE_CHECKING
|
|
6
|
+
|
|
7
|
+
from agent_env.plugins import _registration
|
|
8
|
+
|
|
9
|
+
if TYPE_CHECKING:
|
|
10
|
+
from agent_env.env.env import Env
|
|
11
|
+
from agent_env.config.runtime import Config
|
|
12
|
+
|
|
13
|
+
# Env defines this only to raise, so a registered class must override it. Not ``deploy``: a
|
|
14
|
+
# component env such as ``service_db`` is deployed by the env that contains it.
|
|
15
|
+
_MUST_IMPLEMENT = ("from_dict",)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _get_type(cls: type["Env"]) -> str:
|
|
19
|
+
return cls.type
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _builtin_registry() -> dict[str, type["Env"]]:
|
|
23
|
+
"""The built-in env types, by type."""
|
|
24
|
+
from agent_env.env.envs.gateway_server import GatewayEnv
|
|
25
|
+
from agent_env.env.envs.mcp_server import MCPServerEnv
|
|
26
|
+
from agent_env.env.envs.multi_env import MultiEnv
|
|
27
|
+
from agent_env.env.envs.service_db import ServiceDBEnv
|
|
28
|
+
from agent_env.env.envs.website import WebsiteEnv
|
|
29
|
+
|
|
30
|
+
return {
|
|
31
|
+
_get_type(GatewayEnv): GatewayEnv,
|
|
32
|
+
_get_type(MCPServerEnv): MCPServerEnv,
|
|
33
|
+
_get_type(MultiEnv): MultiEnv,
|
|
34
|
+
_get_type(ServiceDBEnv): ServiceDBEnv,
|
|
35
|
+
_get_type(WebsiteEnv): WebsiteEnv,
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _build_registry(source: Config | None = None) -> dict[str, type["Env"]]:
|
|
40
|
+
"""The built-in env types, then ``agent_env.envs`` plugins, then ``[envs] impls`` in ``source``."""
|
|
41
|
+
from agent_env.env.env import Env
|
|
42
|
+
|
|
43
|
+
registry = _builtin_registry()
|
|
44
|
+
validate = _registration.typed_validator(Env, builtins=frozenset(registry), required=_MUST_IMPLEMENT)
|
|
45
|
+
from_plugins = _registration.merge(registry, _registration.ENVS, validate, source=source)
|
|
46
|
+
_merge_config_toml_envs(registry, source=source, from_plugins=from_plugins)
|
|
47
|
+
return registry
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def get_env_registry() -> dict[str, type["Env"]]:
|
|
51
|
+
from agent_env.config import runtime
|
|
52
|
+
|
|
53
|
+
return runtime.get_config().env_registry()
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _merge_config_toml_envs(
|
|
57
|
+
registry: dict[str, type["Env"]],
|
|
58
|
+
*,
|
|
59
|
+
source: Config | None = None,
|
|
60
|
+
from_plugins: _registration.Registrations | None = None,
|
|
61
|
+
) -> None:
|
|
62
|
+
from agent_env.config import runtime
|
|
63
|
+
from agent_env.config import ConfigError, load_impl
|
|
64
|
+
from agent_env.env.env import Env
|
|
65
|
+
|
|
66
|
+
section = (source or runtime.get_config()).section("envs")
|
|
67
|
+
impls = section.get("impls", [])
|
|
68
|
+
if not isinstance(impls, list):
|
|
69
|
+
raise ConfigError(
|
|
70
|
+
f"[envs] impls must be a list of 'module:Class' strings, got {type(impls).__name__}"
|
|
71
|
+
)
|
|
72
|
+
from_plugins = from_plugins if from_plugins is not None else _registration.Registrations.empty()
|
|
73
|
+
for impl in impls:
|
|
74
|
+
if not isinstance(impl, str):
|
|
75
|
+
raise ConfigError(
|
|
76
|
+
f"[envs] impl must be a 'module:Class' string, got {type(impl).__name__}: {impl!r}"
|
|
77
|
+
)
|
|
78
|
+
cls = load_impl(impl, Env)
|
|
79
|
+
if cls.type == Env.type:
|
|
80
|
+
raise ConfigError(
|
|
81
|
+
f"[envs] impl {impl!r} does not define its own 'type' "
|
|
82
|
+
f"(inherits the base default {Env.type!r}); set a unique 'type' ClassVar"
|
|
83
|
+
)
|
|
84
|
+
if problem := _registration.unimplemented(cls, Env, _MUST_IMPLEMENT):
|
|
85
|
+
raise ConfigError(f"[envs] impl {impl!r}: {problem}")
|
|
86
|
+
if from_plugins.refuse(cls.type, f"[envs] impl {impl!r}"):
|
|
87
|
+
continue
|
|
88
|
+
if not from_plugins.release(cls.type, "envs", impl, cls) and cls.type in registry:
|
|
89
|
+
raise ConfigError(
|
|
90
|
+
f"[envs] impl {impl!r} type {cls.type!r} is already registered "
|
|
91
|
+
f"(conflicts with a built-in or another custom env)"
|
|
92
|
+
)
|
|
93
|
+
registry[cls.type] = cls
|
|
@@ -0,0 +1,372 @@
|
|
|
1
|
+
"""Env snapshot store for caching pre-loaded servicedb images."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import hashlib
|
|
7
|
+
import logging
|
|
8
|
+
from dataclasses import dataclass
|
|
9
|
+
from datetime import datetime, timezone
|
|
10
|
+
from typing import Callable, Optional
|
|
11
|
+
|
|
12
|
+
from agent_env.providers.sandbox_providers.sandbox_provider import SANDBOX_MODE_CONTAINER
|
|
13
|
+
from agent_env.config import get_config
|
|
14
|
+
from agent_env.store.document_store import AbsentOrNull, Filter, Sort
|
|
15
|
+
from agent_env.store.routing import refuse_local_derivation
|
|
16
|
+
|
|
17
|
+
logger = logging.getLogger(__name__)
|
|
18
|
+
|
|
19
|
+
ENV_SNAPSHOTS_COLLECTION = "env_snapshots"
|
|
20
|
+
|
|
21
|
+
ProgressCallback = Callable[[str, str, int], None]
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def compute_env_fingerprint(env) -> str:
|
|
25
|
+
"""A stable digest of the service images a MultiEnv is built from.
|
|
26
|
+
|
|
27
|
+
Snapshots carry pre-ingested PGDATA, and the schema in that PGDATA is produced by
|
|
28
|
+
the *service images* — not by the env id. So the env id is the wrong cache key in
|
|
29
|
+
both directions: re-registering the same images under a new id needlessly orphans a
|
|
30
|
+
valid snapshot, while rebuilding a service under the same id would silently serve a
|
|
31
|
+
snapshot whose schema no longer matches.
|
|
32
|
+
|
|
33
|
+
Keying on the images fixes both. Sorted by environment name so dict/list ordering
|
|
34
|
+
can't change the digest, and truncated because this is a cache key, not a security
|
|
35
|
+
claim.
|
|
36
|
+
"""
|
|
37
|
+
parts: list[str] = []
|
|
38
|
+
for mcp_env in getattr(env, "mcp_server_envs", None) or []:
|
|
39
|
+
image = getattr(getattr(mcp_env, "docker_image_artifact", None), "image_name", "")
|
|
40
|
+
parts.append(f"mcp:{mcp_env.environment_name}={image}")
|
|
41
|
+
for site_env in getattr(env, "website_envs", None) or []:
|
|
42
|
+
backend = getattr(getattr(site_env, "backend_docker_image_artifact", None), "image_name", "")
|
|
43
|
+
frontend = getattr(getattr(site_env, "frontend_docker_image_artifact", None), "image_name", "")
|
|
44
|
+
parts.append(f"web:{site_env.environment_name}={backend}+{frontend}")
|
|
45
|
+
digest = hashlib.sha256("\n".join(sorted(parts)).encode()).hexdigest()
|
|
46
|
+
return digest[:32]
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
@dataclass
|
|
50
|
+
class EnvSnapshot:
|
|
51
|
+
env_id: str
|
|
52
|
+
environment_universe_id: str
|
|
53
|
+
environment_universe_version: int
|
|
54
|
+
instance_id: str
|
|
55
|
+
is_clean: bool
|
|
56
|
+
db_image_artifact_id: str
|
|
57
|
+
db_image_artifact_version: int
|
|
58
|
+
created_at_utc: datetime | None = None
|
|
59
|
+
# Absent on rows written before image-fingerprint keying; those match on env_id only.
|
|
60
|
+
env_fingerprint: str | None = None
|
|
61
|
+
|
|
62
|
+
@classmethod
|
|
63
|
+
def from_dict(cls, data: dict) -> EnvSnapshot:
|
|
64
|
+
return cls(
|
|
65
|
+
env_id=data["env_id"],
|
|
66
|
+
environment_universe_id=data["service_universe_id"],
|
|
67
|
+
environment_universe_version=data["service_universe_version"],
|
|
68
|
+
instance_id=data["instance_id"],
|
|
69
|
+
is_clean=data["is_clean"],
|
|
70
|
+
db_image_artifact_id=data["db_image_artifact_id"],
|
|
71
|
+
db_image_artifact_version=data["db_image_artifact_version"],
|
|
72
|
+
created_at_utc=data.get("created_at_utc"),
|
|
73
|
+
env_fingerprint=data.get("env_fingerprint"),
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
@classmethod
|
|
77
|
+
async def create(cls, instance_id: str, on_progress: ProgressCallback | None = None) -> EnvSnapshot:
|
|
78
|
+
"""Create a snapshot of a deployed MultiEnv's servicedb.
|
|
79
|
+
|
|
80
|
+
Reconnects to the sandbox, checks changelog status, exports the
|
|
81
|
+
PostgreSQL data directory, builds a Docker image with the data
|
|
82
|
+
pre-loaded, stores it as a DockerImageArtifact, and persists an
|
|
83
|
+
EnvSnapshot record.
|
|
84
|
+
"""
|
|
85
|
+
from agent_env.artifact import DockerImageArtifact
|
|
86
|
+
from agent_env.env.env import DeployedSandboxEnv, Env
|
|
87
|
+
from agent_env.env.envs.multi_env import MultiEnv, _gateway_state_provider
|
|
88
|
+
from agent_env.env.store import get_env_instance_store
|
|
89
|
+
|
|
90
|
+
log = on_progress or (lambda step, msg, pct: None)
|
|
91
|
+
|
|
92
|
+
log("validate", "Fetching env instance...", 5)
|
|
93
|
+
deployed_env = get_env_instance_store().get(instance_id)
|
|
94
|
+
refuse_local_derivation(deployed_env.env_id, "env", "snapshotting")
|
|
95
|
+
env = Env.get(deployed_env.env_id, deployed_env.env_version)
|
|
96
|
+
if not isinstance(env, MultiEnv):
|
|
97
|
+
raise ValueError(
|
|
98
|
+
f"Env '{deployed_env.env_id}' is type '{env.type}', "
|
|
99
|
+
"only MultiEnv is supported"
|
|
100
|
+
)
|
|
101
|
+
|
|
102
|
+
env_id = deployed_env.env_id
|
|
103
|
+
without_our_gateway = (f"Snapshot capture needs our gateway and its local Postgres store; env '{env_id}' was deployed by "
|
|
104
|
+
f"env_provider_type '{deployed_env.env_provider_type}'")
|
|
105
|
+
if not isinstance(deployed_env, DeployedSandboxEnv):
|
|
106
|
+
raise NotImplementedError(without_our_gateway)
|
|
107
|
+
|
|
108
|
+
environment_universe = get_env_instance_store().get_environment_universe(instance_id)
|
|
109
|
+
if environment_universe is None:
|
|
110
|
+
raise ValueError(f"Instance '{instance_id}' has no service_universe loaded")
|
|
111
|
+
universe_id = environment_universe["id"]
|
|
112
|
+
universe_version = environment_universe["version"]
|
|
113
|
+
log("reconnect", f"Reconnecting to sandbox {deployed_env.sandbox_id}...", 10)
|
|
114
|
+
multi_env = await MultiEnv.from_deployed_env(deployed_env)
|
|
115
|
+
if multi_env._sandbox is None:
|
|
116
|
+
raise NotImplementedError(without_our_gateway)
|
|
117
|
+
if multi_env._sandbox.mode == SANDBOX_MODE_CONTAINER:
|
|
118
|
+
raise NotImplementedError("Snapshot creation not supported on container mode")
|
|
119
|
+
|
|
120
|
+
# Snapshot capture bakes the local servicedb container into an image (docker cp PGDATA →
|
|
121
|
+
# image), so it's only meaningful for a backend that can also restore from that image
|
|
122
|
+
state_provider = _gateway_state_provider(multi_env)
|
|
123
|
+
if state_provider is None or not state_provider.supports_restore_from_snapshot:
|
|
124
|
+
raise NotImplementedError(
|
|
125
|
+
"Snapshot capture is only supported for the local Postgres backend "
|
|
126
|
+
f"(this env's state backend does not support it: {type(state_provider).__name__ if state_provider else None}). "
|
|
127
|
+
"Remote-backed deploys use the normal per-service load."
|
|
128
|
+
)
|
|
129
|
+
|
|
130
|
+
log("check_changelog", "Checking changelog...", 20)
|
|
131
|
+
is_clean = await _check_changelog_empty(multi_env._sandbox)
|
|
132
|
+
logger.info(f"Changelog empty: {is_clean} ({'clean' if is_clean else 'dirty'} snapshot)")
|
|
133
|
+
log("check_changelog", f"Changelog: {'clean' if is_clean else 'dirty'}", 30)
|
|
134
|
+
|
|
135
|
+
log("snapshot_db", "Exporting servicedb data...", 40)
|
|
136
|
+
tar_gz_s3_url = await _snapshot_servicedb(multi_env._sandbox, env_id, universe_id, log)
|
|
137
|
+
|
|
138
|
+
log("store_artifact", "Storing Docker image artifact...", 70)
|
|
139
|
+
logger.info("Storing snapshot as DockerImageArtifact...")
|
|
140
|
+
image_tag = f"env-snapshot-{env_id}-{universe_id}"
|
|
141
|
+
artifact = DockerImageArtifact.put_tar(
|
|
142
|
+
id=f"env-snapshot-{env_id}",
|
|
143
|
+
description=(
|
|
144
|
+
f"Snapshot of servicedb for env={env_id} "
|
|
145
|
+
f"universe={universe_id} v{universe_version}"
|
|
146
|
+
),
|
|
147
|
+
image_name=image_tag,
|
|
148
|
+
tar_gz_s3_url=tar_gz_s3_url,
|
|
149
|
+
)
|
|
150
|
+
logger.info(f"Artifact: id={artifact.id} version={artifact.version}")
|
|
151
|
+
log("store_artifact", f"Artifact: id={artifact.id} version={artifact.version}", 80)
|
|
152
|
+
|
|
153
|
+
log("store_snapshot", "Persisting snapshot record...", 90)
|
|
154
|
+
snapshot = get_env_snapshot_store().put(
|
|
155
|
+
env_id=env_id,
|
|
156
|
+
environment_universe_id=universe_id,
|
|
157
|
+
environment_universe_version=universe_version,
|
|
158
|
+
instance_id=instance_id,
|
|
159
|
+
is_clean=is_clean,
|
|
160
|
+
db_image_artifact_id=artifact.id,
|
|
161
|
+
db_image_artifact_version=artifact.version,
|
|
162
|
+
# Recorded from the env whose images produced this PGDATA, so a later
|
|
163
|
+
# image-identical env (any id) can match it — and a rebuilt one cannot.
|
|
164
|
+
env_fingerprint=compute_env_fingerprint(env),
|
|
165
|
+
)
|
|
166
|
+
log("done", "Snapshot complete", 100)
|
|
167
|
+
return snapshot
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
async def _check_changelog_empty(sandbox) -> bool:
|
|
171
|
+
"""Return True if the _changelog table is empty (no modifications since data load)."""
|
|
172
|
+
from agent_env.env.envs.service_db import DB_NAME, DB_USER
|
|
173
|
+
from agent_env.env.envs.service_db import DATABASE_SERVICE_NAME
|
|
174
|
+
from agent_env.providers.env_providers.constants import DOCKER_COMPOSE_PATH
|
|
175
|
+
|
|
176
|
+
cmd = (
|
|
177
|
+
f"docker compose -f {DOCKER_COMPOSE_PATH} exec -T {DATABASE_SERVICE_NAME} "
|
|
178
|
+
f'psql -U {DB_USER} -d {DB_NAME} -tAc "SELECT count(*) FROM public._changelog"'
|
|
179
|
+
)
|
|
180
|
+
output = await sandbox.exec_script(cmd)
|
|
181
|
+
count = int(output.strip())
|
|
182
|
+
return count == 0
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
async def _snapshot_servicedb(sandbox, env_id: str, environment_universe_id: str, log: ProgressCallback) -> str:
|
|
186
|
+
"""Export pgdata from sandbox servicedb, build Docker image on sandbox, upload to S3.
|
|
187
|
+
|
|
188
|
+
Returns the S3 URL of the uploaded tar.gz.
|
|
189
|
+
"""
|
|
190
|
+
from agent_env.env.envs.service_db import DATABASE_SERVICE_NAME
|
|
191
|
+
from agent_env.providers.env_providers.constants import DOCKER_COMPOSE_PATH
|
|
192
|
+
|
|
193
|
+
container_id_output = await sandbox.exec_script(
|
|
194
|
+
f"docker compose -f {DOCKER_COMPOSE_PATH} ps -a -q {DATABASE_SERVICE_NAME}"
|
|
195
|
+
)
|
|
196
|
+
container_id = container_id_output.strip()
|
|
197
|
+
if not container_id:
|
|
198
|
+
raise RuntimeError("servicedb container not found on sandbox")
|
|
199
|
+
|
|
200
|
+
log("snapshot_db", "Copying pgdata from container...", 45)
|
|
201
|
+
logger.info("Exporting pgdata from servicedb container...")
|
|
202
|
+
await sandbox.exec_script(f"docker cp {container_id}:/var/lib/postgresql/data /tmp/pgdata")
|
|
203
|
+
|
|
204
|
+
image_tag = f"env-snapshot-{env_id}-{environment_universe_id}"
|
|
205
|
+
log("snapshot_db", f"Building Docker image on sandbox: {image_tag}...", 50)
|
|
206
|
+
logger.info(f"Building Docker image on sandbox: {image_tag}...")
|
|
207
|
+
await sandbox.exec_script(
|
|
208
|
+
"mkdir -p /tmp/snapshot-build && "
|
|
209
|
+
"mv /tmp/pgdata /tmp/snapshot-build/pgdata && "
|
|
210
|
+
"cat > /tmp/snapshot-build/Dockerfile << 'DEOF'\n"
|
|
211
|
+
"FROM public.ecr.aws/docker/library/postgres:16-alpine\n"
|
|
212
|
+
"COPY pgdata /var/lib/postgresql/data\n"
|
|
213
|
+
"RUN chown -R postgres:postgres /var/lib/postgresql/data\n"
|
|
214
|
+
"DEOF"
|
|
215
|
+
)
|
|
216
|
+
await sandbox.exec_script(
|
|
217
|
+
f"docker build --platform linux/amd64 -t {image_tag} /tmp/snapshot-build"
|
|
218
|
+
)
|
|
219
|
+
|
|
220
|
+
log("snapshot_db", "Saving Docker image...", 55)
|
|
221
|
+
logger.info("Saving Docker image to tar.gz on sandbox...")
|
|
222
|
+
await sandbox.exec_script(f"docker save {image_tag} | gzip > /tmp/snapshot-image.tar.gz")
|
|
223
|
+
|
|
224
|
+
config = get_config()
|
|
225
|
+
object_store = config.get_object_store()
|
|
226
|
+
object_url = object_store.object_url(
|
|
227
|
+
f"{config.get_artifact_key_prefix()}env-snapshots/{env_id}/{environment_universe_id}/{image_tag}.tar.gz"
|
|
228
|
+
)
|
|
229
|
+
put_url = await asyncio.to_thread(object_store.signed_put_url, object_url)
|
|
230
|
+
if put_url is None:
|
|
231
|
+
raise RuntimeError(
|
|
232
|
+
f"{type(object_store).__name__} can't presign uploads; env snapshots need a signable object store or local execution."
|
|
233
|
+
)
|
|
234
|
+
log("snapshot_db", "Uploading Docker image...", 60)
|
|
235
|
+
logger.info(f"Uploading Docker image to {object_url}...")
|
|
236
|
+
await sandbox.exec_script(
|
|
237
|
+
f'curl -fsSL -X PUT --upload-file /tmp/snapshot-image.tar.gz "{put_url}"'
|
|
238
|
+
)
|
|
239
|
+
|
|
240
|
+
await sandbox.exec_script(f"rm -rf /tmp/snapshot-build /tmp/snapshot-image.tar.gz && docker rmi {image_tag}")
|
|
241
|
+
|
|
242
|
+
return object_url
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
class EnvSnapshotStore:
|
|
246
|
+
def __init__(self) -> None:
|
|
247
|
+
self._indexed = None
|
|
248
|
+
|
|
249
|
+
@property
|
|
250
|
+
def _doc_store(self):
|
|
251
|
+
# Resolved per call: a cached store outlives reset_config(), so a process that
|
|
252
|
+
# re-pointed would read the new config and write the old backend.
|
|
253
|
+
store = get_config().get_document_store()
|
|
254
|
+
if self._indexed is not store:
|
|
255
|
+
store.ensure_index(
|
|
256
|
+
ENV_SNAPSHOTS_COLLECTION,
|
|
257
|
+
["env_id", "service_universe_id", "instance_id"],
|
|
258
|
+
unique=True,
|
|
259
|
+
)
|
|
260
|
+
# The get_clean lookup shape — hit on every universe load, so it should
|
|
261
|
+
# not table-scan as snapshots accumulate.
|
|
262
|
+
store.ensure_index(
|
|
263
|
+
ENV_SNAPSHOTS_COLLECTION,
|
|
264
|
+
["env_fingerprint", "service_universe_id", "service_universe_version", "is_clean"],
|
|
265
|
+
)
|
|
266
|
+
self._indexed = store
|
|
267
|
+
return store
|
|
268
|
+
|
|
269
|
+
def put(
|
|
270
|
+
self,
|
|
271
|
+
env_id: str,
|
|
272
|
+
environment_universe_id: str,
|
|
273
|
+
environment_universe_version: int,
|
|
274
|
+
instance_id: str,
|
|
275
|
+
is_clean: bool,
|
|
276
|
+
db_image_artifact_id: str,
|
|
277
|
+
db_image_artifact_version: int,
|
|
278
|
+
env_fingerprint: str | None = None,
|
|
279
|
+
) -> EnvSnapshot:
|
|
280
|
+
doc = {
|
|
281
|
+
"env_id": env_id,
|
|
282
|
+
"service_universe_id": environment_universe_id,
|
|
283
|
+
"service_universe_version": environment_universe_version,
|
|
284
|
+
"instance_id": instance_id,
|
|
285
|
+
"is_clean": is_clean,
|
|
286
|
+
"db_image_artifact_id": db_image_artifact_id,
|
|
287
|
+
"db_image_artifact_version": db_image_artifact_version,
|
|
288
|
+
"created_at_utc": datetime.now(timezone.utc),
|
|
289
|
+
"env_fingerprint": env_fingerprint,
|
|
290
|
+
}
|
|
291
|
+
self._doc_store.replace(
|
|
292
|
+
ENV_SNAPSHOTS_COLLECTION,
|
|
293
|
+
# `Filter.of` keywords are stored document keys, not Python names: this one keeps
|
|
294
|
+
# the on-disk spelling while the parameter follows the rename.
|
|
295
|
+
Filter.of(env_id=env_id, service_universe_id=environment_universe_id, instance_id=instance_id),
|
|
296
|
+
doc,
|
|
297
|
+
upsert=True,
|
|
298
|
+
)
|
|
299
|
+
return EnvSnapshot.from_dict(doc)
|
|
300
|
+
|
|
301
|
+
def get(self, env_id: str, environment_universe_id: str, instance_id: str) -> EnvSnapshot | None:
|
|
302
|
+
doc = self._doc_store.find_one(
|
|
303
|
+
ENV_SNAPSHOTS_COLLECTION,
|
|
304
|
+
Filter.of(env_id=env_id, service_universe_id=environment_universe_id, instance_id=instance_id),
|
|
305
|
+
)
|
|
306
|
+
if not doc:
|
|
307
|
+
return None
|
|
308
|
+
return EnvSnapshot.from_dict(doc)
|
|
309
|
+
|
|
310
|
+
def get_clean(
|
|
311
|
+
self,
|
|
312
|
+
env_id: str,
|
|
313
|
+
environment_universe_id: str,
|
|
314
|
+
environment_universe_version: int | None = None,
|
|
315
|
+
env_fingerprint: str | None = None,
|
|
316
|
+
) -> EnvSnapshot | None:
|
|
317
|
+
"""Newest clean snapshot that can be restored into this env, or None.
|
|
318
|
+
|
|
319
|
+
With an ``env_fingerprint``, an image-identical env matches regardless of its
|
|
320
|
+
id — so re-registering the same services under a new env id reuses the existing
|
|
321
|
+
snapshot instead of paying for a fresh full load. The env_id lookup is kept as a
|
|
322
|
+
fallback for rows written before fingerprinting, which carry no digest to match.
|
|
323
|
+
|
|
324
|
+
Note the fingerprint match is deliberately NOT widened to a "close enough" env
|
|
325
|
+
id comparison: an env whose service images differ has a different schema, and
|
|
326
|
+
restoring a stale snapshot into it would corrupt the env rather than slow it down.
|
|
327
|
+
"""
|
|
328
|
+
base = {"service_universe_id": environment_universe_id, "is_clean": True}
|
|
329
|
+
if environment_universe_version is not None:
|
|
330
|
+
base["service_universe_version"] = environment_universe_version
|
|
331
|
+
|
|
332
|
+
candidates: list[Filter] = []
|
|
333
|
+
# Fingerprint first: it is the correct key, and it lets a renamed env hit.
|
|
334
|
+
if env_fingerprint:
|
|
335
|
+
candidates.append(Filter.of(**base, env_fingerprint=env_fingerprint))
|
|
336
|
+
# Legacy rows predate fingerprinting, so the exact env id is the only provenance
|
|
337
|
+
# they carry. Restricted to rows with NO fingerprint on purpose: a row that has
|
|
338
|
+
# one and did not match above was built from different service images, and
|
|
339
|
+
# matching it on env_id alone would restore a stale schema into this env.
|
|
340
|
+
candidates.append(
|
|
341
|
+
Filter.of(**base, env_id=env_id).where("env_fingerprint", AbsentOrNull())
|
|
342
|
+
)
|
|
343
|
+
|
|
344
|
+
for query in candidates:
|
|
345
|
+
doc = self._doc_store.find_one(
|
|
346
|
+
ENV_SNAPSHOTS_COLLECTION,
|
|
347
|
+
query,
|
|
348
|
+
sort=Sort.by("created_at_utc", descending=True),
|
|
349
|
+
)
|
|
350
|
+
if doc:
|
|
351
|
+
return EnvSnapshot.from_dict(doc)
|
|
352
|
+
return None
|
|
353
|
+
|
|
354
|
+
|
|
355
|
+
_env_snapshot_store: Optional[EnvSnapshotStore] = None
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
def get_env_snapshot_store() -> EnvSnapshotStore:
|
|
359
|
+
global _env_snapshot_store
|
|
360
|
+
if _env_snapshot_store is None:
|
|
361
|
+
_env_snapshot_store = EnvSnapshotStore()
|
|
362
|
+
return _env_snapshot_store
|
|
363
|
+
|
|
364
|
+
|
|
365
|
+
def set_env_snapshot_store(store: EnvSnapshotStore) -> None:
|
|
366
|
+
global _env_snapshot_store
|
|
367
|
+
_env_snapshot_store = store
|
|
368
|
+
|
|
369
|
+
|
|
370
|
+
def reset_env_snapshot_store() -> None:
|
|
371
|
+
global _env_snapshot_store
|
|
372
|
+
_env_snapshot_store = None
|