agentenv-framework 0.9.1254__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agent_env/__init__.py +0 -0
- agent_env/a2a_agent/__init__.py +23 -0
- agent_env/a2a_agent/a2a_agent.py +600 -0
- agent_env/a2a_agent/conversation_store.py +356 -0
- agent_env/a2a_agent/object_transfer.py +609 -0
- agent_env/a2a_agent/protocol.py +238 -0
- agent_env/a2a_agent/store.py +201 -0
- agent_env/a2a_agent/validator.py +901 -0
- agent_env/artifact/__init__.py +48 -0
- agent_env/artifact/artifact.py +74 -0
- agent_env/artifact/artifacts/__init__.py +0 -0
- agent_env/artifact/artifacts/cli.py +83 -0
- agent_env/artifact/artifacts/docker_image.py +457 -0
- agent_env/artifact/artifacts/environment.py +89 -0
- agent_env/artifact/artifacts/environment_universe.py +156 -0
- agent_env/artifact/artifacts/file.py +214 -0
- agent_env/artifact/artifacts/file_artifact_universe.py +206 -0
- agent_env/artifact/artifacts/skill.py +206 -0
- agent_env/artifact/artifacts/vm_image.py +95 -0
- agent_env/artifact/ref.py +15 -0
- agent_env/artifact/registry.py +251 -0
- agent_env/artifact/store.py +295 -0
- agent_env/artifact/universe.py +38 -0
- agent_env/attribution.py +41 -0
- agent_env/bundle/__init__.py +4 -0
- agent_env/bundle/_fs.py +128 -0
- agent_env/bundle/authoring.py +239 -0
- agent_env/bundle/installed.py +175 -0
- agent_env/bundle/ledger.py +269 -0
- agent_env/bundle/materialize.py +204 -0
- agent_env/bundle/parse.py +594 -0
- agent_env/bundle/plan.py +414 -0
- agent_env/bundle/resolve.py +404 -0
- agent_env/bundle/run.py +361 -0
- agent_env/cli/__init__.py +69 -0
- agent_env/cli/__main__.py +10 -0
- agent_env/cli/_installers.py +576 -0
- agent_env/cli/_plugin_changes.py +947 -0
- agent_env/cli/a2a_agent/__init__.py +22 -0
- agent_env/cli/a2a_agent/add_skill.py +97 -0
- agent_env/cli/a2a_agent/deploy.py +60 -0
- agent_env/cli/a2a_agent/get.py +15 -0
- agent_env/cli/a2a_agent/get_instance.py +26 -0
- agent_env/cli/a2a_agent/put.py +78 -0
- agent_env/cli/a2a_agent/validate.py +56 -0
- agent_env/cli/artifact/__init__.py +20 -0
- agent_env/cli/artifact/cli.py +83 -0
- agent_env/cli/artifact/environment.py +44 -0
- agent_env/cli/artifact/environment_universe.py +163 -0
- agent_env/cli/artifact/file_artifact_universe.py +281 -0
- agent_env/cli/artifact/skill.py +128 -0
- agent_env/cli/banner.py +25 -0
- agent_env/cli/config.py +263 -0
- agent_env/cli/env/__init__.py +31 -0
- agent_env/cli/env/deploy.py +112 -0
- agent_env/cli/env/gateway.py +57 -0
- agent_env/cli/env/get_instance.py +40 -0
- agent_env/cli/env/mcp_server.py +410 -0
- agent_env/cli/env/multi.py +282 -0
- agent_env/cli/env/service_db.py +99 -0
- agent_env/cli/env/snapshot.py +35 -0
- agent_env/cli/env/state/__init__.py +6 -0
- agent_env/cli/env/state/init.py +108 -0
- agent_env/cli/env/state/teardown.py +44 -0
- agent_env/cli/env/website.py +216 -0
- agent_env/cli/env/website_browser.py +67 -0
- agent_env/cli/eval/__init__.py +16 -0
- agent_env/cli/eval/add_tasks.py +63 -0
- agent_env/cli/eval/create.py +49 -0
- agent_env/cli/eval/run.py +194 -0
- agent_env/cli/identity/__init__.py +5 -0
- agent_env/cli/identity/client_identity.py +37 -0
- agent_env/cli/plugin.py +438 -0
- agent_env/cli/run.py +285 -0
- agent_env/cli/task/__init__.py +21 -0
- agent_env/cli/task/create.py +109 -0
- agent_env/cli/task/get.py +17 -0
- agent_env/cli/task/get_instance.py +34 -0
- agent_env/cli/task/run.py +615 -0
- agent_env/cli/task/validate.py +31 -0
- agent_env/cli/up.py +123 -0
- agent_env/cli/utils.py +183 -0
- agent_env/config/__init__.py +50 -0
- agent_env/config/describe.py +1067 -0
- agent_env/config/errors.py +13 -0
- agent_env/config/loader.py +245 -0
- agent_env/config/model.py +154 -0
- agent_env/config/paths.py +25 -0
- agent_env/config/plugin_tables.py +101 -0
- agent_env/config/provenance.py +134 -0
- agent_env/config/runtime.py +1064 -0
- agent_env/config/snapshot.py +81 -0
- agent_env/entity_refs.py +186 -0
- agent_env/env/__init__.py +49 -0
- agent_env/env/env.py +433 -0
- agent_env/env/env_artifact_store.py +87 -0
- agent_env/env/envs/__init__.py +7 -0
- agent_env/env/envs/_deployment.py +281 -0
- agent_env/env/envs/gateway_server.py +32 -0
- agent_env/env/envs/mcp_server.py +387 -0
- agent_env/env/envs/multi_env.py +780 -0
- agent_env/env/envs/service_db/Dockerfile +20 -0
- agent_env/env/envs/service_db/Dockerfile.db-mcp +42 -0
- agent_env/env/envs/service_db/Dockerfile.db-web +8 -0
- agent_env/env/envs/service_db.py +92 -0
- agent_env/env/envs/website.py +294 -0
- agent_env/env/envs/website_browser/Dockerfile +40 -0
- agent_env/env/envs/website_browser/__init__.py +17 -0
- agent_env/env/envs/website_browser/entrypoint.sh +13 -0
- agent_env/env/gateway/Dockerfile +14 -0
- agent_env/env/gateway/__init__.py +37 -0
- agent_env/env/gateway/clock.py +184 -0
- agent_env/env/gateway/constants.py +190 -0
- agent_env/env/gateway/entrypoint.py +156 -0
- agent_env/env/gateway/gateway.py +1095 -0
- agent_env/env/gateway/get_time.py +60 -0
- agent_env/env/gateway/requirements.txt +4 -0
- agent_env/env/gateway/triggers.py +1193 -0
- agent_env/env/legacy_protocol.py +146 -0
- agent_env/env/registry.py +93 -0
- agent_env/env/snapshot_store.py +372 -0
- agent_env/env/store.py +411 -0
- agent_env/eval/__init__.py +14 -0
- agent_env/eval/eval.py +68 -0
- agent_env/eval/store.py +116 -0
- agent_env/examples/hello/README.md +9 -0
- agent_env/examples/hello/artifacts/greeting/check.sh +1 -0
- agent_env/examples/hello/artifacts/greeting/hello.txt +1 -0
- agent_env/examples/hello/tasks/hello.json +10 -0
- agent_env/explorer/__init__.py +12 -0
- agent_env/explorer/app.py +277 -0
- agent_env/explorer/openapi_docs.py +242 -0
- agent_env/explorer/plugin.py +95 -0
- agent_env/explorer/routers/__init__.py +1 -0
- agent_env/explorer/routers/common.py +174 -0
- agent_env/explorer/routers/conversations.py +25 -0
- agent_env/explorer/routers/objects.py +112 -0
- agent_env/explorer/routers/runs.py +475 -0
- agent_env/plugins/__init__.py +59 -0
- agent_env/plugins/_cli.py +244 -0
- agent_env/plugins/_discovery.py +97 -0
- agent_env/plugins/_inventory.py +262 -0
- agent_env/plugins/_registration.py +268 -0
- agent_env/plugins/_report.py +47 -0
- agent_env/plugins/_requirements.py +55 -0
- agent_env/providers/__init__.py +37 -0
- agent_env/providers/env_providers/__init__.py +12 -0
- agent_env/providers/env_providers/constants.py +13 -0
- agent_env/providers/env_providers/env_gateway_provider.py +1354 -0
- agent_env/providers/env_providers/env_provider.py +237 -0
- agent_env/providers/env_providers/env_server_provider.py +133 -0
- agent_env/providers/env_state/__init__.py +49 -0
- agent_env/providers/env_state/env_state_provider.py +554 -0
- agent_env/providers/env_state/local_postgres.py +390 -0
- agent_env/providers/env_state/store.py +112 -0
- agent_env/providers/sandbox_providers/__init__.py +29 -0
- agent_env/providers/sandbox_providers/chained_sandbox_provider.py +93 -0
- agent_env/providers/sandbox_providers/e2b/__init__.py +6 -0
- agent_env/providers/sandbox_providers/e2b/provider.py +376 -0
- agent_env/providers/sandbox_providers/e2b/sandbox.py +342 -0
- agent_env/providers/sandbox_providers/e2b/template.py +158 -0
- agent_env/providers/sandbox_providers/local_sandbox.py +408 -0
- agent_env/providers/sandbox_providers/modal_sandbox.py +500 -0
- agent_env/providers/sandbox_providers/modal_vm_sandbox.py +466 -0
- agent_env/providers/sandbox_providers/sandbox.py +387 -0
- agent_env/providers/sandbox_providers/sandbox_provider.py +614 -0
- agent_env/py.typed +0 -0
- agent_env/runner/__init__.py +16 -0
- agent_env/runner/local_runner.py +190 -0
- agent_env/runner/runner.py +127 -0
- agent_env/runner/store.py +94 -0
- agent_env/store/__init__.py +126 -0
- agent_env/store/_google.py +68 -0
- agent_env/store/base.py +33 -0
- agent_env/store/document_store/__init__.py +60 -0
- agent_env/store/document_store/document_store.py +519 -0
- agent_env/store/document_store/dynamodb_document_store.py +306 -0
- agent_env/store/document_store/evaluation.py +199 -0
- agent_env/store/document_store/firestore_mongo_document_store.py +242 -0
- agent_env/store/document_store/mongo_document_store.py +259 -0
- agent_env/store/document_store/sqlite_document_store.py +314 -0
- agent_env/store/ids.py +123 -0
- agent_env/store/image_store/__init__.py +34 -0
- agent_env/store/image_store/ecr_image_store.py +131 -0
- agent_env/store/image_store/google_credentials.py +113 -0
- agent_env/store/image_store/image_store.py +113 -0
- agent_env/store/image_store/local_registry_image_store.py +106 -0
- agent_env/store/image_store/oci_registry_credentials.py +196 -0
- agent_env/store/local_state.py +27 -0
- agent_env/store/object_store/__init__.py +24 -0
- agent_env/store/object_store/gcs_object_store.py +415 -0
- agent_env/store/object_store/local_object_store.py +143 -0
- agent_env/store/object_store/object_store.py +198 -0
- agent_env/store/object_store/s3_object_store.py +354 -0
- agent_env/store/query.py +112 -0
- agent_env/store/routing.py +609 -0
- agent_env/store/secret_store/__init__.py +19 -0
- agent_env/store/secret_store/aws_secrets_manager_secret_store.py +293 -0
- agent_env/store/secret_store/gcp_secret_manager_secret_store.py +319 -0
- agent_env/store/secret_store/local_secret_store.py +47 -0
- agent_env/store/secret_store/secret_store.py +19 -0
- agent_env/task/__init__.py +27 -0
- agent_env/task/interrupts.py +109 -0
- agent_env/task/registry.py +27 -0
- agent_env/task/step_journal.py +99 -0
- agent_env/task/store.py +940 -0
- agent_env/task/task.py +815 -0
- agent_env/task/teardown.py +142 -0
- agent_env/task_step/__init__.py +47 -0
- agent_env/task_step/context.py +195 -0
- agent_env/task_step/context_ops.py +224 -0
- agent_env/task_step/registry.py +182 -0
- agent_env/task_step/review_store.py +118 -0
- agent_env/task_step/snapshot_utils/__init__.py +2 -0
- agent_env/task_step/snapshot_utils/agent_state_capture.py +261 -0
- agent_env/task_step/snapshot_utils/snapshot_series.py +631 -0
- agent_env/task_step/store.py +129 -0
- agent_env/task_step/task_step.py +173 -0
- agent_env/task_step/task_steps/__init__.py +27 -0
- agent_env/task_step/task_steps/a2a_agent_validator/__init__.py +31 -0
- agent_env/task_step/task_steps/a2a_agent_validator/fixtures/README.md +158 -0
- agent_env/task_step/task_steps/a2a_agent_validator/fixtures/clip.m4a +0 -0
- agent_env/task_step/task_steps/a2a_agent_validator/fixtures/clip.mp3 +0 -0
- agent_env/task_step/task_steps/a2a_agent_validator/fixtures/clip.mp4 +0 -0
- agent_env/task_step/task_steps/a2a_agent_validator/fixtures/clip.ogg +0 -0
- agent_env/task_step/task_steps/a2a_agent_validator/fixtures/clip.wav +0 -0
- agent_env/task_step/task_steps/a2a_agent_validator/fixtures/document.pdf +0 -0
- agent_env/task_step/task_steps/a2a_agent_validator/fixtures/red.gif +0 -0
- agent_env/task_step/task_steps/a2a_agent_validator/fixtures/red.jpg +0 -0
- agent_env/task_step/task_steps/a2a_agent_validator/fixtures/red.png +0 -0
- agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_agent_card.py +103 -0
- agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_agent_config_identity.py +69 -0
- agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_agent_mcp.py +178 -0
- agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_core_protocol.py +67 -0
- agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_install.py +115 -0
- agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_litellm_attribution.py +124 -0
- agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_litellm_attribution_runtime.py +173 -0
- agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_modalities.py +270 -0
- agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_peer_agents.py +102 -0
- agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_role.py +151 -0
- agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_skill_config.py +194 -0
- agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_snapshot.py +111 -0
- agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_system_prompt.py +215 -0
- agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_trajectory.py +164 -0
- agent_env/task_step/task_steps/add_skills.py +240 -0
- agent_env/task_step/task_steps/apply_server_config.py +288 -0
- agent_env/task_step/task_steps/collect_artifacts.py +940 -0
- agent_env/task_step/task_steps/deploy_agent.py +626 -0
- agent_env/task_step/task_steps/deploy_env.py +197 -0
- agent_env/task_step/task_steps/deploy_human_agent.py +100 -0
- agent_env/task_step/task_steps/deploy_sandbox.py +170 -0
- agent_env/task_step/task_steps/env_card_validator/__init__.py +4 -0
- agent_env/task_step/task_steps/env_card_validator/verify_env_card.py +109 -0
- agent_env/task_step/task_steps/env_card_validator/verify_env_core_protocol.py +108 -0
- agent_env/task_step/task_steps/install_agent.py +376 -0
- agent_env/task_step/task_steps/load_artifact.py +775 -0
- agent_env/task_step/task_steps/mcp_cli_builder/__init__.py +4 -0
- agent_env/task_step/task_steps/mcp_cli_builder/build_mcp_cli.py +201 -0
- agent_env/task_step/task_steps/mcp_cli_builder/codegen.py +627 -0
- agent_env/task_step/task_steps/mcp_env_validator/__init__.py +44 -0
- agent_env/task_step/task_steps/mcp_env_validator/validation_gate_aggregator.py +102 -0
- agent_env/task_step/task_steps/mcp_env_validator/verify_mcp_env_assessment.py +78 -0
- agent_env/task_step/task_steps/mcp_env_validator/verify_mcp_tool_schema.py +152 -0
- agent_env/task_step/task_steps/mcp_env_validator/verify_spec_conformance.py +486 -0
- agent_env/task_step/task_steps/modify_env_tool_access.py +80 -0
- agent_env/task_step/task_steps/multienv_validator/__init__.py +3 -0
- agent_env/task_step/task_steps/multienv_validator/combine_universe_verdicts.py +107 -0
- agent_env/task_step/task_steps/multienv_validator/universe_comparison.py +253 -0
- agent_env/task_step/task_steps/multienv_validator/verify_universe_agent_judge.py +180 -0
- agent_env/task_step/task_steps/multienv_validator/verify_universe_roundtrip.py +241 -0
- agent_env/task_step/task_steps/peer_agents.py +92 -0
- agent_env/task_step/task_steps/prompt_agent.py +900 -0
- agent_env/task_step/task_steps/register_agent_triggers.py +113 -0
- agent_env/task_step/task_steps/register_env_triggers.py +97 -0
- agent_env/task_step/task_steps/reset_env.py +71 -0
- agent_env/task_step/task_steps/review.py +152 -0
- agent_env/task_step/task_steps/run_code.py +461 -0
- agent_env/task_step/task_steps/run_code_runner.py +52 -0
- agent_env/task_step/task_steps/run_docker_container.py +380 -0
- agent_env/task_step/task_steps/sandbox_utils/__init__.py +0 -0
- agent_env/task_step/task_steps/sandbox_utils/sandbox_utils.py +79 -0
- agent_env/task_step/task_steps/snapshot_agent_state.py +205 -0
- agent_env/task_step/task_steps/snapshot_env.py +637 -0
- agent_env/task_step/task_steps/sync_env_clock.py +132 -0
- agent_env/task_step/task_steps/teardown_sandboxes.py +170 -0
- agent_env/task_step/task_steps/verifiers/__init__.py +23 -0
- agent_env/task_step/task_steps/verifiers/agent_prompt_response_verifier.py +166 -0
- agent_env/task_step/task_steps/verifiers/aggregate_verifiers.py +90 -0
- agent_env/task_step/task_steps/verifiers/env_outcome_verifier.py +127 -0
- agent_env/task_step/task_steps/verifiers/judge_utils/__init__.py +2 -0
- agent_env/task_step/task_steps/verifiers/judge_utils/frame_selection.py +377 -0
- agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py +1293 -0
- agent_env/task_step/task_steps/verifiers/judge_utils/trajectory_filter.py +514 -0
- agent_env/task_step/task_steps/verifiers/rubrics_verifier.py +1318 -0
- agent_env/task_step/task_steps/verifiers/run_container_unit_tests_verifier.py +434 -0
- agent_env/task_step/task_steps/verifiers/scoring.py +40 -0
- agent_env/task_step/task_steps/verifiers/verify_sandbox.py +308 -0
- agent_env/task_step/thread_work.py +61 -0
- agent_env/utils/__init__.py +0 -0
- agent_env/utils/card_naming.py +108 -0
- agent_env/utils/deprecation.py +109 -0
- agent_env/utils/docker_build.py +29 -0
- agent_env/utils/exec_retry.py +77 -0
- agent_env/utils/litellm_attribution.py +59 -0
- agent_env/utils/paths.py +20 -0
- agentenv_framework-0.9.1254.dist-info/METADATA +394 -0
- agentenv_framework-0.9.1254.dist-info/RECORD +312 -0
- agentenv_framework-0.9.1254.dist-info/WHEEL +4 -0
- agentenv_framework-0.9.1254.dist-info/entry_points.txt +5 -0
- agentenv_framework-0.9.1254.dist-info/licenses/LICENSE +202 -0
- agentenv_framework-0.9.1254.dist-info/licenses/NOTICE +4 -0
- agentenv_framework-0.9.1254.dist-info/licenses/THIRD_PARTY_NOTICES.md +7665 -0
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"""Artifact module for AgentEnv."""
|
|
2
|
+
|
|
3
|
+
from agent_env.artifact.artifact import Artifact
|
|
4
|
+
from agent_env.artifact.ref import ArtifactRef
|
|
5
|
+
from agent_env.artifact.artifacts.cli import CliArtifact
|
|
6
|
+
from agent_env.artifact.artifacts.docker_image import DockerImageArtifact
|
|
7
|
+
from agent_env.artifact.artifacts.file import FileArtifact
|
|
8
|
+
from agent_env.artifact.artifacts.file_artifact_universe import FileArtifactUniverse
|
|
9
|
+
from agent_env.artifact.artifacts.environment import EnvironmentArtifact
|
|
10
|
+
from agent_env.artifact.artifacts.environment_universe import EnvironmentUniverseArtifact
|
|
11
|
+
from agent_env.artifact.artifacts.skill import AGENT_SKILLS_SPEC_VERSION, SkillArtifact
|
|
12
|
+
from agent_env.artifact.artifacts.vm_image import VMImageArtifact
|
|
13
|
+
from agent_env.artifact.registry import (
|
|
14
|
+
ARTIFACT_REGISTRY,
|
|
15
|
+
canonical_type,
|
|
16
|
+
equivalent_types,
|
|
17
|
+
get_artifact_registry,
|
|
18
|
+
get_type_aliases,
|
|
19
|
+
)
|
|
20
|
+
from agent_env.artifact.store import ArtifactStore, get_artifact_store, reset_artifact_store, set_artifact_store
|
|
21
|
+
from agent_env.artifact.universe import Universe
|
|
22
|
+
|
|
23
|
+
__all__ = [
|
|
24
|
+
"Artifact",
|
|
25
|
+
"ArtifactRef",
|
|
26
|
+
"CliArtifact",
|
|
27
|
+
"DockerImageArtifact",
|
|
28
|
+
"FileArtifact",
|
|
29
|
+
"FileArtifactUniverse",
|
|
30
|
+
"EnvironmentArtifact",
|
|
31
|
+
"EnvironmentUniverseArtifact",
|
|
32
|
+
# Deprecated aliases. Kept in __all__ so `from agent_env.artifact import *`
|
|
33
|
+
# still binds them; served by __getattr__ below rather than imported eagerly, so
|
|
34
|
+
# importing this package does not itself trip the deprecation counter.
|
|
35
|
+
"SkillArtifact",
|
|
36
|
+
"AGENT_SKILLS_SPEC_VERSION",
|
|
37
|
+
"Universe",
|
|
38
|
+
"VMImageArtifact",
|
|
39
|
+
"ARTIFACT_REGISTRY",
|
|
40
|
+
"canonical_type",
|
|
41
|
+
"equivalent_types",
|
|
42
|
+
"get_artifact_registry",
|
|
43
|
+
"get_type_aliases",
|
|
44
|
+
"ArtifactStore",
|
|
45
|
+
"get_artifact_store",
|
|
46
|
+
"reset_artifact_store",
|
|
47
|
+
"set_artifact_store",
|
|
48
|
+
]
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
"""Artifact models for AgentEnv."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import TYPE_CHECKING, Any, ClassVar, Optional, Self
|
|
6
|
+
|
|
7
|
+
from pydantic import BaseModel, Field, field_validator
|
|
8
|
+
|
|
9
|
+
from agent_env.entity_refs import EntityRef
|
|
10
|
+
|
|
11
|
+
if TYPE_CHECKING:
|
|
12
|
+
from agent_env.bundle.authoring import AuthoringContext
|
|
13
|
+
from agent_env.artifact.ref import ArtifactRef
|
|
14
|
+
from agent_env.artifact.store import ArtifactQuery
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class Artifact(BaseModel):
|
|
18
|
+
"""Base class for all environment artifacts. Immutable - each modification creates a new version."""
|
|
19
|
+
|
|
20
|
+
id: str = Field(description="Unique artifact identifier")
|
|
21
|
+
version: int = Field(default=0, description="Artifact version (auto-calculated on put)")
|
|
22
|
+
type: str = Field(description="The type of Artifact")
|
|
23
|
+
|
|
24
|
+
toml_refs: ClassVar[tuple[EntityRef, ...]] = ()
|
|
25
|
+
|
|
26
|
+
@field_validator("type", mode="before")
|
|
27
|
+
@classmethod
|
|
28
|
+
def _canonical_type(cls, value):
|
|
29
|
+
"""Accept a spelling `[artifacts] type_aliases` maps to this type's own."""
|
|
30
|
+
from agent_env.artifact.registry import canonical_type
|
|
31
|
+
|
|
32
|
+
return canonical_type(value) if isinstance(value, str) else value
|
|
33
|
+
|
|
34
|
+
def as_ref(self) -> "ArtifactRef":
|
|
35
|
+
from agent_env.artifact.ref import ArtifactRef
|
|
36
|
+
|
|
37
|
+
if self.version < 1:
|
|
38
|
+
raise ValueError(
|
|
39
|
+
f"Cannot create a ref for unpersisted artifact {self.id!r} "
|
|
40
|
+
"(version=0). Call put() first."
|
|
41
|
+
)
|
|
42
|
+
return ArtifactRef(id=self.id, version=self.version)
|
|
43
|
+
|
|
44
|
+
@classmethod
|
|
45
|
+
def get(cls, id: str, version: Optional[int] = None) -> Self:
|
|
46
|
+
"""The artifact stored under ``id``, which must be a ``cls``: the store builds the class
|
|
47
|
+
its stored type names, so ``FileArtifact.get`` refuses an id that holds another type."""
|
|
48
|
+
from agent_env.artifact.store import get_artifact_store
|
|
49
|
+
|
|
50
|
+
artifact = get_artifact_store().get(id, version)
|
|
51
|
+
if not isinstance(artifact, cls):
|
|
52
|
+
raise TypeError(f"artifact {id!r} is a {artifact.type!r} artifact, not a {cls.__name__}")
|
|
53
|
+
return artifact
|
|
54
|
+
|
|
55
|
+
@classmethod
|
|
56
|
+
def put(cls, **kwargs: Any) -> Self:
|
|
57
|
+
from agent_env.artifact.store import get_artifact_store
|
|
58
|
+
|
|
59
|
+
instance = cls(**kwargs)
|
|
60
|
+
return get_artifact_store().put_document(instance)
|
|
61
|
+
|
|
62
|
+
@classmethod
|
|
63
|
+
def from_toml(cls, data: dict[str, Any], ctx: AuthoringContext) -> Self:
|
|
64
|
+
"""Write the artifact authored as ``data`` (its toml, with the keys ``toml_refs`` declares
|
|
65
|
+
resolved to ids) under ``ctx.id`` and return it. The default suits a type that is only a
|
|
66
|
+
document; one whose ``put`` uploads files or images overrides this."""
|
|
67
|
+
fields = {key: value for key, value in data.items() if key not in ("id", "type", "version")}
|
|
68
|
+
return cls.put(id=ctx.id, **fields)
|
|
69
|
+
|
|
70
|
+
@classmethod
|
|
71
|
+
def query(cls) -> "ArtifactQuery":
|
|
72
|
+
from agent_env.artifact.store import ArtifactQuery, get_artifact_store
|
|
73
|
+
|
|
74
|
+
return ArtifactQuery(get_artifact_store())
|
|
File without changes
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
"""CliArtifact — packages a CLI tool (entrypoint + supporting files) as a versioned artifact.
|
|
2
|
+
|
|
3
|
+
A CliArtifact references one FileArtifactUniverse bundling every file in the CLI
|
|
4
|
+
directory tree, colocated under a single S3 prefix (`artifacts/cli/<key_segment(id)>/<version>/`).
|
|
5
|
+
The `entrypoint` field is a relative path within that prefix that consumers
|
|
6
|
+
(e.g. an agent gateway) chmod +x and symlink onto PATH.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from typing import TYPE_CHECKING, Literal, Optional
|
|
13
|
+
|
|
14
|
+
from pydantic import Field
|
|
15
|
+
|
|
16
|
+
from agent_env.artifact.artifact import Artifact
|
|
17
|
+
from agent_env.config import get_config
|
|
18
|
+
from agent_env.store.ids import derive_id, key_segment
|
|
19
|
+
|
|
20
|
+
if TYPE_CHECKING:
|
|
21
|
+
from agent_env.artifact.artifacts.file_artifact_universe import FileArtifactUniverse
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class CliArtifact(Artifact):
|
|
25
|
+
type: Literal["cli"] = "cli"
|
|
26
|
+
|
|
27
|
+
cli_files_id: str = Field(description="ID of the FileArtifactUniverse bundling every file in the CLI tree")
|
|
28
|
+
cli_object_url: str = Field(alias="cli_s3_url", description="Object-store prefix where the CLI bundle lives")
|
|
29
|
+
entrypoint: str = Field(description="Relative path within the bundle to the executable file (e.g. 'bin/slack')")
|
|
30
|
+
command_name: str = Field(description="Command name installed on PATH (e.g. 'slack' -> /usr/local/bin/slack)")
|
|
31
|
+
|
|
32
|
+
env_id: Optional[str] = Field(default=None, description="Source env id this CLI was generated from, if any")
|
|
33
|
+
env_version: Optional[int] = Field(default=None, description="Source env version this CLI was generated from, if any")
|
|
34
|
+
|
|
35
|
+
@classmethod
|
|
36
|
+
def put(
|
|
37
|
+
cls,
|
|
38
|
+
id: str,
|
|
39
|
+
*,
|
|
40
|
+
command_name: str,
|
|
41
|
+
entrypoint: str,
|
|
42
|
+
cli_dir: Path,
|
|
43
|
+
env_id: Optional[str] = None,
|
|
44
|
+
env_version: Optional[int] = None,
|
|
45
|
+
) -> "CliArtifact":
|
|
46
|
+
from agent_env.artifact.artifacts.file_artifact_universe import FileArtifactUniverse
|
|
47
|
+
from agent_env.artifact.store import get_artifact_store
|
|
48
|
+
|
|
49
|
+
if not cli_dir.is_dir():
|
|
50
|
+
raise ValueError(f"cli_dir {cli_dir} is not a directory")
|
|
51
|
+
entrypoint_path = cli_dir / entrypoint
|
|
52
|
+
if not entrypoint_path.is_file():
|
|
53
|
+
raise ValueError(f"entrypoint {entrypoint!r} not found at {entrypoint_path}")
|
|
54
|
+
|
|
55
|
+
files: dict[str, Path] = {}
|
|
56
|
+
for p in sorted(cli_dir.rglob("*")):
|
|
57
|
+
if p.is_file():
|
|
58
|
+
files[p.relative_to(cli_dir).as_posix()] = p
|
|
59
|
+
if not files:
|
|
60
|
+
raise ValueError(f"cli_dir {cli_dir} contains no files")
|
|
61
|
+
|
|
62
|
+
store = get_artifact_store()
|
|
63
|
+
version = store.next_version(id)
|
|
64
|
+
key = f"{get_config().get_artifact_key_prefix()}artifacts/cli/{key_segment(id)}/{version}/"
|
|
65
|
+
cli_s3_url = get_config().get_object_store_for(id).object_url(key)
|
|
66
|
+
|
|
67
|
+
universe = FileArtifactUniverse.put_bundled(id=derive_id(id, "files"), files=files, s3_url=cli_s3_url)
|
|
68
|
+
|
|
69
|
+
instance = cls(
|
|
70
|
+
id=id,
|
|
71
|
+
version=version,
|
|
72
|
+
cli_files_id=universe.id,
|
|
73
|
+
cli_s3_url=cli_s3_url,
|
|
74
|
+
entrypoint=entrypoint,
|
|
75
|
+
command_name=command_name,
|
|
76
|
+
env_id=env_id,
|
|
77
|
+
env_version=env_version,
|
|
78
|
+
)
|
|
79
|
+
return store.put_document(instance)
|
|
80
|
+
|
|
81
|
+
def get_cli_files(self) -> "FileArtifactUniverse":
|
|
82
|
+
from agent_env.artifact.artifacts.file_artifact_universe import FileArtifactUniverse
|
|
83
|
+
return FileArtifactUniverse.get(self.cli_files_id)
|
|
@@ -0,0 +1,457 @@
|
|
|
1
|
+
"""Docker image artifact for storing Docker images in S3."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import gzip
|
|
7
|
+
import logging
|
|
8
|
+
import os
|
|
9
|
+
import posixpath
|
|
10
|
+
import re
|
|
11
|
+
import shlex
|
|
12
|
+
import subprocess
|
|
13
|
+
import tarfile
|
|
14
|
+
import tempfile
|
|
15
|
+
import uuid
|
|
16
|
+
from dataclasses import dataclass
|
|
17
|
+
from importlib.metadata import version as pkg_version
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
from typing import Callable, ClassVar, Literal
|
|
20
|
+
from urllib.parse import urlparse
|
|
21
|
+
|
|
22
|
+
from pydantic import Field
|
|
23
|
+
|
|
24
|
+
from agent_env.artifact.artifact import Artifact
|
|
25
|
+
from agent_env.store.ids import fs_safe, image_repository, is_local_id
|
|
26
|
+
|
|
27
|
+
logger = logging.getLogger(__name__)
|
|
28
|
+
|
|
29
|
+
ProgressCallback = Callable[[str, str, int], None]
|
|
30
|
+
|
|
31
|
+
_SAFE_GIT_NAME = re.compile(r"^[a-zA-Z0-9._-]+$")
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _validate_git_name(value: str, label: str) -> str:
|
|
35
|
+
if not _SAFE_GIT_NAME.match(value):
|
|
36
|
+
raise ValueError(f"Invalid {label}: {value!r}")
|
|
37
|
+
return value
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _git_clone_commands(owner: str, repo: str, ref: str | None, token: str | None) -> list[str]:
|
|
42
|
+
"""Shell commands that clone ``owner/repo`` at ``ref`` into /tmp/repo. A token rides a
|
|
43
|
+
throwaway GIT_ASKPASS helper so it never appears on a command line, and the helper is
|
|
44
|
+
removed whether or not the clone succeeds; without a token the clone is anonymous and
|
|
45
|
+
fails fast on a private repository instead of prompting."""
|
|
46
|
+
ref_flag = f"--branch {ref}" if ref else ""
|
|
47
|
+
clone_url = f"https://github.com/{owner}/{repo}.git"
|
|
48
|
+
if not token:
|
|
49
|
+
return [f"GIT_TERMINAL_PROMPT=0 git clone --depth 1 {ref_flag} {clone_url} /tmp/repo"]
|
|
50
|
+
askpass_script = (
|
|
51
|
+
'#!/bin/sh\n'
|
|
52
|
+
'case "$1" in\n'
|
|
53
|
+
'*sername*) echo x-access-token;;\n'
|
|
54
|
+
f'*) echo "{token}";;\n'
|
|
55
|
+
'esac\n'
|
|
56
|
+
)
|
|
57
|
+
return [
|
|
58
|
+
f"cat > /tmp/git-askpass.sh << 'ASKEOF'\n{askpass_script}ASKEOF",
|
|
59
|
+
"chmod +x /tmp/git-askpass.sh",
|
|
60
|
+
f"trap 'rm -f /tmp/git-askpass.sh' EXIT; GIT_ASKPASS=/tmp/git-askpass.sh git clone --depth 1 {ref_flag} {clone_url} /tmp/repo",
|
|
61
|
+
]
|
|
62
|
+
|
|
63
|
+
class DockerImageArtifact(Artifact):
|
|
64
|
+
"""A Docker image artifact stored as tar.gz in S3."""
|
|
65
|
+
|
|
66
|
+
DOCKER_SAVE_TIMEOUT_SECONDS: ClassVar[int] = 900
|
|
67
|
+
|
|
68
|
+
type: Literal["docker_image"] = "docker_image"
|
|
69
|
+
description: str = Field(description="Description of the Docker image")
|
|
70
|
+
image_name: str = Field(description="Docker image name/tag")
|
|
71
|
+
tar_gz_object_url: str = Field(alias="tar_gz_s3_url", description="Object-store locator of the tar.gz file")
|
|
72
|
+
build_context_object_url: str | None = Field(default=None, alias="build_context_s3_url", description="Object-store locator of the build context tar.gz")
|
|
73
|
+
|
|
74
|
+
@classmethod
|
|
75
|
+
def put(
|
|
76
|
+
cls,
|
|
77
|
+
id: str,
|
|
78
|
+
*,
|
|
79
|
+
description: str,
|
|
80
|
+
image_name: str,
|
|
81
|
+
build_context_path: str | None = None,
|
|
82
|
+
dockerfile_path: str | None = None,
|
|
83
|
+
) -> "DockerImageArtifact":
|
|
84
|
+
from agent_env.artifact.store import get_artifact_store
|
|
85
|
+
from agent_env.config import get_config
|
|
86
|
+
|
|
87
|
+
store = get_artifact_store()
|
|
88
|
+
version = store.next_version(id)
|
|
89
|
+
|
|
90
|
+
config = get_config()
|
|
91
|
+
image_store = config.get_image_store_for(id)
|
|
92
|
+
repository = image_repository(id)
|
|
93
|
+
image_ref = image_store.image_ref(repository, f"v{version}")
|
|
94
|
+
image_store.ensure_repository(repository)
|
|
95
|
+
_push_local_image(image_name, image_ref, image_store)
|
|
96
|
+
|
|
97
|
+
with tempfile.NamedTemporaryFile(suffix=".tar.gz", delete=False) as tmp:
|
|
98
|
+
tmp_path = Path(tmp.name)
|
|
99
|
+
|
|
100
|
+
try:
|
|
101
|
+
save_proc = subprocess.Popen(
|
|
102
|
+
["docker", "save", image_ref],
|
|
103
|
+
stdout=subprocess.PIPE,
|
|
104
|
+
stderr=subprocess.PIPE,
|
|
105
|
+
)
|
|
106
|
+
with gzip.open(tmp_path, "wb") as gz_file:
|
|
107
|
+
while chunk := save_proc.stdout.read(8192):
|
|
108
|
+
gz_file.write(chunk)
|
|
109
|
+
try:
|
|
110
|
+
save_proc.wait(timeout=cls.DOCKER_SAVE_TIMEOUT_SECONDS)
|
|
111
|
+
except subprocess.TimeoutExpired:
|
|
112
|
+
save_proc.kill()
|
|
113
|
+
raise RuntimeError(f"docker save {image_ref} timed out after {cls.DOCKER_SAVE_TIMEOUT_SECONDS} seconds")
|
|
114
|
+
if save_proc.returncode != 0:
|
|
115
|
+
stderr = save_proc.stderr.read().decode() if save_proc.stderr else ""
|
|
116
|
+
raise RuntimeError(f"docker save {image_ref} failed: {stderr}")
|
|
117
|
+
|
|
118
|
+
tar_gz_s3_url = store.put_object_file(
|
|
119
|
+
artifact_type="docker_image",
|
|
120
|
+
id=id,
|
|
121
|
+
version=version,
|
|
122
|
+
object_name=f"{fs_safe(id)}-v{version}.tar.gz",
|
|
123
|
+
file_path=str(tmp_path),
|
|
124
|
+
content_type="application/gzip",
|
|
125
|
+
)
|
|
126
|
+
finally:
|
|
127
|
+
if tmp_path.exists():
|
|
128
|
+
tmp_path.unlink()
|
|
129
|
+
|
|
130
|
+
build_context_s3_url = None
|
|
131
|
+
if build_context_path:
|
|
132
|
+
context_dir = Path(build_context_path)
|
|
133
|
+
paths_to_include = _get_dockerfile_copy_sources(context_dir, dockerfile_path)
|
|
134
|
+
with tempfile.NamedTemporaryFile(suffix=".tar.gz", delete=False) as ctx_tmp:
|
|
135
|
+
ctx_tmp_path = Path(ctx_tmp.name)
|
|
136
|
+
try:
|
|
137
|
+
with tarfile.open(ctx_tmp_path, "w:gz") as tar:
|
|
138
|
+
for rel_path in paths_to_include:
|
|
139
|
+
full_path = context_dir / rel_path
|
|
140
|
+
if full_path.exists():
|
|
141
|
+
tar.add(str(full_path), arcname=rel_path)
|
|
142
|
+
build_context_s3_url = store.put_object_file(
|
|
143
|
+
artifact_type="docker_image",
|
|
144
|
+
id=id,
|
|
145
|
+
version=version,
|
|
146
|
+
object_name="build-context.tar.gz",
|
|
147
|
+
file_path=str(ctx_tmp_path),
|
|
148
|
+
content_type="application/gzip",
|
|
149
|
+
)
|
|
150
|
+
finally:
|
|
151
|
+
if ctx_tmp_path.exists():
|
|
152
|
+
ctx_tmp_path.unlink()
|
|
153
|
+
|
|
154
|
+
return cls.put_tar(
|
|
155
|
+
id,
|
|
156
|
+
description=description,
|
|
157
|
+
image_name=image_ref,
|
|
158
|
+
tar_gz_s3_url=tar_gz_s3_url,
|
|
159
|
+
build_context_s3_url=build_context_s3_url,
|
|
160
|
+
)
|
|
161
|
+
|
|
162
|
+
@classmethod
|
|
163
|
+
def put_tar(
|
|
164
|
+
cls,
|
|
165
|
+
id: str,
|
|
166
|
+
*,
|
|
167
|
+
description: str,
|
|
168
|
+
image_name: str,
|
|
169
|
+
tar_gz_s3_url: str,
|
|
170
|
+
build_context_s3_url: str | None = None,
|
|
171
|
+
) -> "DockerImageArtifact":
|
|
172
|
+
from agent_env.artifact.store import get_artifact_store
|
|
173
|
+
from agent_env.config import get_config
|
|
174
|
+
|
|
175
|
+
for url in (tar_gz_s3_url, build_context_s3_url):
|
|
176
|
+
if url:
|
|
177
|
+
get_config().check_object_url(id, url)
|
|
178
|
+
store = get_artifact_store()
|
|
179
|
+
version = store.next_version(id)
|
|
180
|
+
instance = cls(
|
|
181
|
+
id=id,
|
|
182
|
+
version=version,
|
|
183
|
+
description=description,
|
|
184
|
+
image_name=image_name,
|
|
185
|
+
tar_gz_s3_url=tar_gz_s3_url,
|
|
186
|
+
build_context_s3_url=build_context_s3_url,
|
|
187
|
+
)
|
|
188
|
+
return store.put_document(instance)
|
|
189
|
+
|
|
190
|
+
def load(self) -> bytes:
|
|
191
|
+
from agent_env.artifact.store import get_artifact_store
|
|
192
|
+
return get_artifact_store().get_object(self.tar_gz_object_url)
|
|
193
|
+
|
|
194
|
+
@classmethod
|
|
195
|
+
async def put_from_github(
|
|
196
|
+
cls,
|
|
197
|
+
id: str,
|
|
198
|
+
dockerfile_github_url: str,
|
|
199
|
+
docker_context_github_url: str | None = None,
|
|
200
|
+
on_progress: ProgressCallback | None = None,
|
|
201
|
+
github_token: str | None = None,
|
|
202
|
+
) -> GitHubBuildResult:
|
|
203
|
+
"""Build a Docker image from a GitHub repo on a temporary VM.
|
|
204
|
+
|
|
205
|
+
Clones the repo (with ``github_token`` when the repository is private), runs docker
|
|
206
|
+
build, saves the image, uploads to S3, and creates a DockerImageArtifact. Returns a GitHubBuildResult with
|
|
207
|
+
the artifact and git metadata.
|
|
208
|
+
|
|
209
|
+
Note: Branch names containing slashes (e.g. feature/fix-bug) are not
|
|
210
|
+
supported in GitHub URLs due to path ambiguity. Use branches/tags
|
|
211
|
+
without slashes, or the default branch.
|
|
212
|
+
"""
|
|
213
|
+
from agent_env.providers import get_sandbox_provider
|
|
214
|
+
from agent_env.config import get_config
|
|
215
|
+
|
|
216
|
+
refuse_local_github_build(id)
|
|
217
|
+
log = on_progress or (lambda step, msg, pct: None)
|
|
218
|
+
|
|
219
|
+
log("validate", "Parsing GitHub URLs...", 5)
|
|
220
|
+
df_parts = cls._parse_github_url(dockerfile_github_url)
|
|
221
|
+
if df_parts.path is None:
|
|
222
|
+
raise ValueError(f"Dockerfile URL must include a path to the Dockerfile: {dockerfile_github_url}")
|
|
223
|
+
|
|
224
|
+
if docker_context_github_url:
|
|
225
|
+
ctx_parts = cls._parse_github_url(docker_context_github_url)
|
|
226
|
+
if (ctx_parts.owner, ctx_parts.repo) != (df_parts.owner, df_parts.repo):
|
|
227
|
+
raise ValueError("Dockerfile and context URLs must reference the same repository")
|
|
228
|
+
if ctx_parts.ref and df_parts.ref and ctx_parts.ref != df_parts.ref:
|
|
229
|
+
raise ValueError("Dockerfile and context URLs must reference the same ref")
|
|
230
|
+
context_repo_path = ctx_parts.path or ""
|
|
231
|
+
else:
|
|
232
|
+
context_repo_path = posixpath.dirname(df_parts.path)
|
|
233
|
+
|
|
234
|
+
owner, repo, ref = df_parts.owner, df_parts.repo, df_parts.ref
|
|
235
|
+
dockerfile_repo_path = df_parts.path
|
|
236
|
+
config = get_config()
|
|
237
|
+
suffix = uuid.uuid4().hex[:8]
|
|
238
|
+
image_tag = f"{id}-{suffix}"
|
|
239
|
+
|
|
240
|
+
from agent_env.artifact.store import get_artifact_store
|
|
241
|
+
store = get_artifact_store()
|
|
242
|
+
version = store.next_version(id)
|
|
243
|
+
image_store = config.get_image_store()
|
|
244
|
+
repository = image_repository(id)
|
|
245
|
+
image_ref = image_store.image_ref(repository, f"v{version}")
|
|
246
|
+
await asyncio.to_thread(image_store.ensure_repository, repository)
|
|
247
|
+
auth = await asyncio.to_thread(image_store.auth, image_ref)
|
|
248
|
+
|
|
249
|
+
log("create_vm", "Creating VM...", 10)
|
|
250
|
+
logger.info(f"put_from_github: cloning {owner}/{repo} ref={ref} dockerfile={dockerfile_repo_path} context={context_repo_path}")
|
|
251
|
+
sandbox = await get_sandbox_provider().create_vm(
|
|
252
|
+
disk_size_gb=20, timeout=1800, exposed_ports=[],
|
|
253
|
+
attribution={"project_id": "agent-env-image-builder"},
|
|
254
|
+
)
|
|
255
|
+
try:
|
|
256
|
+
log("install_git", "Installing git...", 20)
|
|
257
|
+
await sandbox.exec_script("apt-get update -qq && apt-get install -y -qq git >/dev/null 2>&1")
|
|
258
|
+
|
|
259
|
+
log("clone", f"Cloning {owner}/{repo}...", 30)
|
|
260
|
+
_validate_git_name(owner, "owner")
|
|
261
|
+
_validate_git_name(repo, "repo")
|
|
262
|
+
if ref:
|
|
263
|
+
_validate_git_name(ref, "ref")
|
|
264
|
+
for command in _git_clone_commands(owner, repo, ref, github_token):
|
|
265
|
+
await sandbox.exec_script(command)
|
|
266
|
+
|
|
267
|
+
commit_hash = (await sandbox.exec_script("git -C /tmp/repo rev-parse HEAD")).strip()
|
|
268
|
+
|
|
269
|
+
log("build", "Building Docker image...", 50)
|
|
270
|
+
dockerfile_abs = shlex.quote(f"/tmp/repo/{dockerfile_repo_path}")
|
|
271
|
+
context_abs = shlex.quote(f"/tmp/repo/{context_repo_path}" if context_repo_path else "/tmp/repo")
|
|
272
|
+
await sandbox.exec_script(f"test -f {dockerfile_abs}")
|
|
273
|
+
await sandbox.exec_script(f"docker build --platform linux/amd64 -f {dockerfile_abs} -t {shlex.quote(image_tag)} {context_abs}")
|
|
274
|
+
|
|
275
|
+
log("push", "Pushing image...", 60)
|
|
276
|
+
login_cmd = ""
|
|
277
|
+
if auth is not None:
|
|
278
|
+
login_cmd = (
|
|
279
|
+
f"echo {shlex.quote(auth.password)} | docker login "
|
|
280
|
+
f"--username {shlex.quote(auth.username)} --password-stdin {shlex.quote(auth.registry)} && "
|
|
281
|
+
)
|
|
282
|
+
await sandbox.exec_script(
|
|
283
|
+
f"docker tag {shlex.quote(image_tag)} {shlex.quote(image_ref)} && "
|
|
284
|
+
f"{login_cmd}"
|
|
285
|
+
f"docker push {shlex.quote(image_ref)}"
|
|
286
|
+
)
|
|
287
|
+
|
|
288
|
+
log("save", "Saving Docker image...", 65)
|
|
289
|
+
await sandbox.exec_script(f"docker save {shlex.quote(image_ref)} | gzip > /tmp/image.tar.gz")
|
|
290
|
+
|
|
291
|
+
log("upload", "Uploading to object store...", 75)
|
|
292
|
+
object_store = config.get_object_store()
|
|
293
|
+
|
|
294
|
+
def _signed_put(key: str) -> tuple[str, str]:
|
|
295
|
+
url = object_store.object_url(f"{config.get_artifact_key_prefix()}{key}")
|
|
296
|
+
put = object_store.signed_put_url(url)
|
|
297
|
+
if put is None:
|
|
298
|
+
raise RuntimeError(
|
|
299
|
+
f"{type(object_store).__name__} can't presign uploads; GitHub image builds need a signable object store."
|
|
300
|
+
)
|
|
301
|
+
return url, put
|
|
302
|
+
|
|
303
|
+
tar_gz_s3_url, image_put_url = await asyncio.to_thread(_signed_put, f"github-builds/{id}/{image_tag}.tar.gz")
|
|
304
|
+
await sandbox.exec_script(f'curl -fsSL -X PUT --upload-file /tmp/image.tar.gz "{image_put_url}"')
|
|
305
|
+
|
|
306
|
+
log("upload_context", "Uploading build context...", 80)
|
|
307
|
+
dockerfile_content = await sandbox.exec_script(f"cat {dockerfile_abs}")
|
|
308
|
+
df_rel = posixpath.relpath(dockerfile_repo_path, context_repo_path) if context_repo_path else dockerfile_repo_path
|
|
309
|
+
copy_sources = _parse_copy_sources(dockerfile_content, df_rel)
|
|
310
|
+
tar_paths = " ".join(shlex.quote(p) for p in copy_sources)
|
|
311
|
+
await sandbox.exec_script(f"tar czf /tmp/build-context.tar.gz -C {context_abs} {tar_paths}")
|
|
312
|
+
build_context_s3_url, context_put_url = await asyncio.to_thread(
|
|
313
|
+
_signed_put, f"github-builds/{id}/{image_tag}-context.tar.gz"
|
|
314
|
+
)
|
|
315
|
+
await sandbox.exec_script(f'curl -fsSL -X PUT --upload-file /tmp/build-context.tar.gz "{context_put_url}"')
|
|
316
|
+
finally:
|
|
317
|
+
log("cleanup", "Terminating build VM...", 85)
|
|
318
|
+
await sandbox.terminate()
|
|
319
|
+
|
|
320
|
+
log("store", "Storing artifact...", 90)
|
|
321
|
+
artifact = cls.put_tar(
|
|
322
|
+
id=id,
|
|
323
|
+
description=f"Built from GitHub: {dockerfile_github_url}",
|
|
324
|
+
image_name=image_ref,
|
|
325
|
+
tar_gz_s3_url=tar_gz_s3_url,
|
|
326
|
+
build_context_s3_url=build_context_s3_url,
|
|
327
|
+
)
|
|
328
|
+
logger.info(f"put_from_github: created artifact id={artifact.id} version={artifact.version}")
|
|
329
|
+
|
|
330
|
+
result = GitHubBuildResult(
|
|
331
|
+
artifact=artifact,
|
|
332
|
+
github_owner=owner,
|
|
333
|
+
github_repo=repo,
|
|
334
|
+
github_ref=ref,
|
|
335
|
+
github_commit=commit_hash,
|
|
336
|
+
dockerfile_github_url=dockerfile_github_url,
|
|
337
|
+
docker_context_github_url=docker_context_github_url,
|
|
338
|
+
)
|
|
339
|
+
log("done", f"Created artifact {id}", 100)
|
|
340
|
+
return result
|
|
341
|
+
|
|
342
|
+
@dataclass
|
|
343
|
+
class _GitHubURLParts:
|
|
344
|
+
owner: str
|
|
345
|
+
repo: str
|
|
346
|
+
ref: str | None
|
|
347
|
+
path: str | None
|
|
348
|
+
|
|
349
|
+
@staticmethod
|
|
350
|
+
def _parse_github_url(url: str) -> DockerImageArtifact._GitHubURLParts:
|
|
351
|
+
"""Parse a GitHub browser URL into components.
|
|
352
|
+
|
|
353
|
+
Accepts:
|
|
354
|
+
https://github.com/{owner}/{repo}
|
|
355
|
+
https://github.com/{owner}/{repo}/tree/{ref}
|
|
356
|
+
https://github.com/{owner}/{repo}/tree/{ref}/{path}
|
|
357
|
+
https://github.com/{owner}/{repo}/blob/{ref}/{path}
|
|
358
|
+
"""
|
|
359
|
+
parsed = urlparse(url)
|
|
360
|
+
if parsed.hostname != "github.com":
|
|
361
|
+
raise ValueError(f"Expected github.com URL, got: {parsed.hostname}")
|
|
362
|
+
segments = [s for s in parsed.path.strip("/").split("/") if s]
|
|
363
|
+
if len(segments) < 2:
|
|
364
|
+
raise ValueError(f"GitHub URL must include owner and repo: {url}")
|
|
365
|
+
owner, repo = segments[0], segments[1]
|
|
366
|
+
if len(segments) == 2:
|
|
367
|
+
return DockerImageArtifact._GitHubURLParts(owner=owner, repo=repo, ref=None, path=None)
|
|
368
|
+
if segments[2] not in ("tree", "blob"):
|
|
369
|
+
raise ValueError(f"Expected /tree/ or /blob/ in GitHub URL, got /{segments[2]}/: {url}")
|
|
370
|
+
if len(segments) < 4:
|
|
371
|
+
raise ValueError(f"GitHub URL must include a ref after /tree/: {url}")
|
|
372
|
+
ref = segments[3]
|
|
373
|
+
path = "/".join(segments[4:]) if len(segments) > 4 else None
|
|
374
|
+
return DockerImageArtifact._GitHubURLParts(owner=owner, repo=repo, ref=ref, path=path)
|
|
375
|
+
|
|
376
|
+
|
|
377
|
+
def refuse_local_github_build(entity_id: str) -> None:
|
|
378
|
+
"""A GitHub build publishes to the configured registry, so it takes a registry id; an ``@local``
|
|
379
|
+
image is built from its own Dockerfile."""
|
|
380
|
+
if is_local_id(entity_id):
|
|
381
|
+
raise ValueError(
|
|
382
|
+
f"{entity_id!r} is an @local id, but a GitHub build publishes to the configured registry; give it a registry id"
|
|
383
|
+
)
|
|
384
|
+
|
|
385
|
+
|
|
386
|
+
@dataclass
|
|
387
|
+
class GitHubBuildResult:
|
|
388
|
+
artifact: DockerImageArtifact
|
|
389
|
+
github_owner: str
|
|
390
|
+
github_repo: str
|
|
391
|
+
github_ref: str | None
|
|
392
|
+
github_commit: str
|
|
393
|
+
dockerfile_github_url: str
|
|
394
|
+
docker_context_github_url: str | None
|
|
395
|
+
|
|
396
|
+
|
|
397
|
+
def _push_local_image(local_tag: str, image_ref: str, image_store) -> None:
|
|
398
|
+
auth = image_store.auth(image_ref)
|
|
399
|
+
if auth is not None:
|
|
400
|
+
login = subprocess.run(
|
|
401
|
+
["docker", "login", "--username", auth.username, "--password-stdin", auth.registry],
|
|
402
|
+
input=auth.password, text=True, capture_output=True,
|
|
403
|
+
)
|
|
404
|
+
if login.returncode != 0:
|
|
405
|
+
raise RuntimeError(f"docker login failed: {login.stderr}")
|
|
406
|
+
|
|
407
|
+
for cmd, label in [
|
|
408
|
+
(["docker", "tag", local_tag, image_ref], "tag"),
|
|
409
|
+
(["docker", "push", image_ref], "push"),
|
|
410
|
+
]:
|
|
411
|
+
result = subprocess.run(cmd, capture_output=True, text=True)
|
|
412
|
+
if result.returncode != 0:
|
|
413
|
+
raise RuntimeError(f"docker {label} failed: {result.stderr}")
|
|
414
|
+
logger.info(f"Pushed {local_tag} -> {image_ref}")
|
|
415
|
+
|
|
416
|
+
|
|
417
|
+
def _parse_copy_sources(dockerfile_text: str, dockerfile_rel_path: str | None = None) -> list[str]:
|
|
418
|
+
"""Parse COPY/ADD source paths from Dockerfile text. Returns top-level directories to include in context tar."""
|
|
419
|
+
paths: list[str] = []
|
|
420
|
+
if dockerfile_rel_path:
|
|
421
|
+
paths.append(dockerfile_rel_path)
|
|
422
|
+
|
|
423
|
+
for line in dockerfile_text.splitlines():
|
|
424
|
+
stripped = line.strip()
|
|
425
|
+
if not stripped.upper().startswith("COPY ") and not stripped.upper().startswith("ADD "):
|
|
426
|
+
continue
|
|
427
|
+
parts = stripped.split()
|
|
428
|
+
args = [p for p in parts[1:] if not p.startswith("--")]
|
|
429
|
+
if len(args) < 2:
|
|
430
|
+
continue
|
|
431
|
+
for src in args[:-1]:
|
|
432
|
+
if src.startswith("/") or "://" in src:
|
|
433
|
+
continue
|
|
434
|
+
src_clean = src.rstrip("/")
|
|
435
|
+
top_dir = src_clean.split("/")[0]
|
|
436
|
+
if top_dir and top_dir != "." and top_dir not in paths:
|
|
437
|
+
paths.append(top_dir)
|
|
438
|
+
|
|
439
|
+
return paths if paths else ["."]
|
|
440
|
+
|
|
441
|
+
|
|
442
|
+
def _get_dockerfile_copy_sources(context_dir: Path, dockerfile_path: str | None) -> list[str]:
|
|
443
|
+
"""Parse COPY source paths from a local Dockerfile and return relative paths to include in the build context tar."""
|
|
444
|
+
if not dockerfile_path:
|
|
445
|
+
return ["."]
|
|
446
|
+
|
|
447
|
+
df = Path(dockerfile_path)
|
|
448
|
+
if not df.is_absolute() and not df.exists():
|
|
449
|
+
df = context_dir / df
|
|
450
|
+
|
|
451
|
+
try:
|
|
452
|
+
df_rel = str(df.relative_to(context_dir))
|
|
453
|
+
except ValueError:
|
|
454
|
+
df_rel = None
|
|
455
|
+
|
|
456
|
+
paths = _parse_copy_sources(df.read_text(), df_rel)
|
|
457
|
+
return [p for p in paths if (context_dir / p).exists()] or ["."]
|