agentenv-framework 0.9.1301__tar.gz → 0.9.1302__tar.gz

{agentenv_framework-0.9.1301 → agentenv_framework-0.9.1302}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: agentenv-framework
-Version: 0.9.1301
+Version: 0.9.1302
 Summary: SDK and CLI for building, deploying and running agent environments and the tasks that grade agents in them.
 Project-URL: Homepage, https://www.agentenvframework.com
 Project-URL: Documentation, https://www.agentenvframework.com/docs
@@ -29,7 +29,7 @@ Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
 Classifier: Typing :: Typed
 Requires-Python: >=3.11
 Requires-Dist: a2a-sdk==0.3.26
-Requires-Dist: agentenv-framework-protocol==0.1.316
+Requires-Dist: agentenv-framework-protocol==0.1.317
 Requires-Dist: certifi>=2024.2.2
 Requires-Dist: click>=8.0.0
 Requires-Dist: cryptography>=42

{agentenv_framework-0.9.1301 → agentenv_framework-0.9.1302}/packages/agentenv-protocol/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "agentenv-framework-protocol"
-version = "0.1.316"
+version = "0.1.317"
 description = "Open data-plane protocol and server SDK for agent environments."
 readme = "README.md"
 license = "Apache-2.0"

{agentenv_framework-0.9.1301 → agentenv_framework-0.9.1302}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "agentenv-framework"
-version = "0.9.1301"
+version = "0.9.1302"
 description = "SDK and CLI for building, deploying and running agent environments and the tasks that grade agents in them."
 readme = "README.md"
 license = "Apache-2.0"
@@ -29,7 +29,7 @@ classifiers = [
     "Typing :: Typed",
 ]
 dependencies = [
-    "agentenv-framework-protocol==0.1.316",
+    "agentenv-framework-protocol==0.1.317",
     "certifi>=2024.2.2",  # the public roots the local transfer CA's trust bundle carries
     "click>=8.0.0",
     "cryptography>=42",  # the local transfer CA

{agentenv_framework-0.9.1301 → agentenv_framework-0.9.1302}/src/agent_env/artifact/artifacts/docker_image.py

@@ -10,7 +10,6 @@ import posixpath
 import re
 import shlex
 import subprocess
-import tarfile
 import tempfile
 import uuid
 from dataclasses import dataclass
@@ -22,10 +21,12 @@ from urllib.parse import urlparse
 from pydantic import ConfigDict, Field, model_serializer
 
 from agent_env.artifact.artifact import Artifact, _write_twin
-from agent_env.store.ids import fs_safe, image_repository, is_local_id
+from agent_env.store.ids import fs_safe, image_repository, is_local_id, local_image_repository
 from agent_env.store.image_store.local_registry_image_store import LocalRegistryImageStore
 from agent_env.store.image_store.oci_registry_credentials import is_loopback_host, names_registry, registry_host_from_ref
 from agent_env.store.image_store.registry_api import pin_digest
+from agent_env.utils.build_context import BuildContext
+from agent_env.utils.docker_build import DEFAULT_BUILD_PLATFORM
 
 logger = logging.getLogger(__name__)
 
@@ -64,8 +65,8 @@ def _git_clone_commands(owner: str, repo: str, ref: str | None, token: str | Non
     ]
 
 class DockerImageArtifact(Artifact):
-    """A Docker image artifact: a tar.gz of the image in the object store, or, without one, a registry reference a
-    sandbox pulls."""
+    """A Docker image artifact: a tar.gz of the image in the object store; without one, a registry reference a
+    sandbox pulls; or, with neither, only its build context."""
 
     model_config = ConfigDict(populate_by_name=True)
 
@@ -78,6 +79,11 @@ class DockerImageArtifact(Artifact):
         default=None, alias="tar_gz_s3_url", description="Object-store locator of the tar.gz file, if it has one"
     )
     build_context_object_url: str | None = Field(default=None, alias="build_context_s3_url", description="Object-store locator of the build context tar.gz")
+    dockerfile_path: str | None = Field(default=None, description="The Dockerfile's path in the build context, POSIX")
+    platform: str | None = Field(default=None, description="The platform the image is built for, as `docker build --platform` names it")
+    source_digest: str | None = Field(
+        default=None, description="sha256 over the build context's files and modes, the Dockerfile's path and the platform"
+    )
 
     # No return annotation: pydantic builds the serialization schema from one, and a dict drops the fields.
     @model_serializer(mode="wrap")
@@ -97,10 +103,15 @@ class DockerImageArtifact(Artifact):
         image_name: str,
         build_context_path: str | None = None,
         dockerfile_path: str | None = None,
+        platform: str | None = None,
     ) -> "DockerImageArtifact":
         from agent_env.artifact.store import get_artifact_store
         from agent_env.config import get_config
 
+        context = None
+        if build_context_path:
+            context_dir = Path(build_context_path)
+            context = BuildContext.of(context_dir, _resolve_dockerfile(context_dir, dockerfile_path))
         store = get_artifact_store()
         version = store.next_version(id)
         # Objects are written once, so each attempt writes under a prefix of its own: one that stopped
@@ -114,6 +125,7 @@ class DockerImageArtifact(Artifact):
         image_ref = image_store.image_ref(repository, f"v{version}")
         image_store.ensure_repository(repository)
         _push_local_image(image_name, image_ref, image_store)
+        platform = platform or _image_platform(image_ref)
 
         with tempfile.NamedTemporaryFile(suffix=".tar.gz", delete=False) as tmp:
             tmp_path = Path(tmp.name)
@@ -143,31 +155,15 @@ class DockerImageArtifact(Artifact):
             if tmp_path.exists():
                 tmp_path.unlink()
 
-        build_context_object_url = None
-        if build_context_path:
-            context_dir = Path(build_context_path)
-            paths_to_include = _get_dockerfile_copy_sources(context_dir, dockerfile_path)
-            with tempfile.NamedTemporaryFile(suffix=".tar.gz", delete=False) as ctx_tmp:
-                ctx_tmp_path = Path(ctx_tmp.name)
-            try:
-                with tarfile.open(ctx_tmp_path, "w:gz") as tar:
-                    for rel_path in paths_to_include:
-                        full_path = context_dir / rel_path
-                        if full_path.exists():
-                            tar.add(str(full_path), arcname=rel_path)
-                build_context_object_url = objects.put_file_at(
-                    f"{prefix}build-context.tar.gz", str(ctx_tmp_path), "application/gzip"
-                )
-            finally:
-                if ctx_tmp_path.exists():
-                    ctx_tmp_path.unlink()
-
         return cls.put_tar(
             id,
             description=description,
             image_name=image_ref,
             tar_gz_object_url=tar_gz_object_url,
-            build_context_object_url=build_context_object_url,
+            build_context_object_url=_upload_context(objects, prefix, context) if context else None,
+            dockerfile_path=context.dockerfile if context else None,
+            platform=platform,
+            source_digest=context.source_digest(platform) if context else None,
         )
 
     @classmethod
@@ -179,6 +175,9 @@ class DockerImageArtifact(Artifact):
         image_name: str,
         tar_gz_object_url: str,
         build_context_object_url: str | None = None,
+        dockerfile_path: str | None = None,
+        platform: str | None = None,
+        source_digest: str | None = None,
     ) -> "DockerImageArtifact":
         from agent_env.artifact.store import get_artifact_store
         from agent_env.config import get_config
@@ -198,6 +197,9 @@ class DockerImageArtifact(Artifact):
             image_name=image_name,
             tar_gz_object_url=tar_gz_object_url,
             build_context_object_url=build_context_object_url,
+            dockerfile_path=dockerfile_path,
+            platform=platform,
+            source_digest=source_digest,
         )
         return store.put_document(instance)
 
@@ -226,17 +228,53 @@ class DockerImageArtifact(Artifact):
         version = store.next_version(id)
         return store.put_document(cls(id=id, version=version, description=description, image_name=pinned))
 
+    @classmethod
+    def put_context(cls, id: str, *, description: str, context_path: str, dockerfile_path: str,
+                    platform: str = DEFAULT_BUILD_PLATFORM) -> "DockerImageArtifact":
+        """Register an image by its build context alone, to be built where it runs: the context is uploaded and
+        nothing is built, pushed or saved, so no Docker is needed here. Its ``image_name``,
+        ``local/<slug>-<hash>:v<version>``, names no registry: it's the tag the build gives the image.
+        ``source_digest`` is equal for equal sources, so one build can serve both."""
+        from agent_env.artifact.store import get_artifact_store
+        from agent_env.config import get_config
+
+        root = Path(context_path)
+        dockerfile = _resolve_dockerfile(root, dockerfile_path)
+        if not dockerfile.is_file():
+            raise ValueError(f"no Dockerfile at {dockerfile}")
+        context = BuildContext.of(root, dockerfile)
+        if context.dockerfile is None:
+            raise ValueError(f"{dockerfile} is outside the build context {root}; the image is built from the context "
+                             "alone, so the Dockerfile must be in it")
+        store = get_artifact_store()
+        version = store.next_version(id)
+        prefix = store.attempt_prefix("docker_image", id)
+        build_context_object_url = _upload_context(get_config().get_object_store_to_write(prefix, id), prefix, context)
+        return store.put_document(cls(
+            id=id,
+            version=version,
+            description=description,
+            image_name=f"{local_image_repository(id)}:v{version}",
+            build_context_object_url=build_context_object_url,
+            dockerfile_path=context.dockerfile,
+            platform=platform,
+            source_digest=context.source_digest(platform),
+        ))
+
     def load(self) -> bytes:
         from agent_env.artifact.store import get_artifact_store
         if not self.tar_gz_object_url:
-            raise ValueError(f"{self.id!r} v{self.version} has no tar.gz; its image is pulled from {self.image_name}")
+            raise ValueError(self.load_problem() or f"{self.id!r} v{self.version} has no tar.gz; its image is pulled "
+                             f"from {self.image_name}")
         return get_artifact_store().get_object(self.tar_gz_object_url)
 
     def load_problem(self) -> str | None:
         """Why no sandbox can get this image, or None when one can: a tar.gz is loaded, and with none, ``image_name``
-        is pulled, so it must name its registry."""
+        is pulled, so it must name its registry. One registered by its build context alone isn't built yet."""
         if self.tar_gz_object_url or names_registry(self.image_name):
             return None
+        if self.build_context_object_url:
+            return f"{self.id!r} v{self.version} is only a build context, and sandboxes don't build images from one yet"
         return (f"{self.id!r} v{self.version} has no tar.gz, and its image name {self.image_name!r} doesn't name a "
                 "registry to pull it from")
 
@@ -482,19 +520,29 @@ def _parse_copy_sources(dockerfile_text: str, dockerfile_rel_path: str | None =
     return paths if paths else ["."]
 
 
-def _get_dockerfile_copy_sources(context_dir: Path, dockerfile_path: str | None) -> list[str]:
-    """Parse COPY source paths from a local Dockerfile and return relative paths to include in the build context tar."""
-    if not dockerfile_path:
-        return ["."]
+def _image_platform(image_ref: str) -> str | None:
+    """The platform of the local image ``image_ref`` as ``docker build --platform`` names it, or None when docker
+    can't say."""
+    result = subprocess.run(["docker", "image", "inspect", "--format", "{{.Os}}/{{.Architecture}}/{{.Variant}}", image_ref],
+                            capture_output=True, text=True)
+    if result.returncode != 0 or result.stdout.count("/") != 2:
+        return None
+    os_name, architecture, variant = result.stdout.strip().split("/")
+    # arm64 has one variant, so it goes unnamed as in `--platform linux/arm64`; 32-bit arm's v6 and v7 differ.
+    return f"{os_name}/{architecture}/{variant}" if architecture == "arm" and variant else f"{os_name}/{architecture}"
+
 
-    df = Path(dockerfile_path)
-    if not df.is_absolute() and not df.exists():
-        df = context_dir / df
+def _resolve_dockerfile(context_dir: Path, dockerfile_path: str | None) -> Path | None:
+    """``dockerfile_path`` as given, or under ``context_dir`` when it's relative and not found from here."""
+    if not dockerfile_path:
+        return None
+    dockerfile = Path(dockerfile_path)
+    return dockerfile if dockerfile.is_absolute() or dockerfile.exists() else context_dir / dockerfile
 
-    try:
-        df_rel = str(df.relative_to(context_dir))
-    except ValueError:
-        df_rel = None
 
-    paths = _parse_copy_sources(df.read_text(), df_rel)
-    return [p for p in paths if (context_dir / p).exists()] or ["."]
+def _upload_context(objects, prefix: str, context: BuildContext) -> str:
+    """Write ``context`` under ``prefix`` as a tar.gz and return its object URL."""
+    with tempfile.TemporaryDirectory(prefix="agent-env-context-") as tmp:
+        path = Path(tmp) / "build-context.tar.gz"
+        context.write(path)
+        return objects.put_file_at(f"{prefix}build-context.tar.gz", str(path), "application/gzip")

{agentenv_framework-0.9.1301 → agentenv_framework-0.9.1302}/src/agent_env/bundle/materialize.py

@@ -158,7 +158,7 @@ def _write_built_image(plan: Plan, write: Write) -> int:
         try:
             return DockerImageArtifact.put(
                 id=write.id, description=f"built from {_path(plan, write)}/{image.dockerfile}", image_name=tag,
-                build_context_path=str(context),
+                build_context_path=str(context), dockerfile_path=str(context / image.dockerfile),
             ).version
         except RuntimeError as e:  # the image store, docker push or docker save
             raise BundleError([f"{_path(plan, write)}: {e}"]) from None

{agentenv_framework-0.9.1301 → agentenv_framework-0.9.1302}/src/agent_env/bundle/parse.py

@@ -24,6 +24,7 @@ from typing import Any, NoReturn
 from agent_env.bundle._fs import Unreadable, fold, home, on_disk, os_reason, read_regular, show
 from agent_env.entity_refs import EntityKind
 from agent_env.store.ids import LOCAL_PREFIX, MAX_AUTHORED_LOCAL_ID_BYTES, validate_local_id
+from agent_env.utils.paths import LEAVINGS
 
 BUNDLE_TOML = "bundle.toml"
 DOCKERFILE = "Dockerfile"
@@ -98,11 +99,6 @@ class BundleError(ValueError):
         return self.problems[0] + (f" (and {more} more)" if more else "")
 
 
-# What the OS or Python writes into a folder on its own. In an artifact or skill folder, everything
-# else is content, dot files included.
-LEAVINGS = frozenset({".DS_Store", "Thumbs.db", "desktop.ini", "__pycache__"})
-
-
 def is_ignored(name: str) -> bool:
     """Dot entries and ``__pycache__`` (pip compiles the .py files of an installed bundle) are never
     entities."""

{agentenv_framework-0.9.1301 → agentenv_framework-0.9.1302}/src/agent_env/cli/a2a_agent/put.py

@@ -73,6 +73,7 @@ def put(agent_id: str, dockerfile: str | None, context_path: str | None, image_r
             image_name=image_tag,
             build_context_path=str(context),
             dockerfile_path=str(dockerfile_path),
+            platform=build_platform,
         )
         click.echo(f"Created artifact: id={artifact.id} version={artifact.version}")
         metadata = detect_env_metadata(dockerfile_path, context)

{agentenv_framework-0.9.1301 → agentenv_framework-0.9.1302}/src/agent_env/cli/env/mcp_server.py

@@ -219,6 +219,7 @@ def put(env_id: str, dockerfile: str | None, context_path: str | None, dockerfil
             image_name=image_tag,
             build_context_path=str(context),
             dockerfile_path=str(dockerfile_path),
+            platform=build_platform,
         )
         click.echo(f"Created artifact: id={artifact.id} version={artifact.version}")
         metadata = detect_env_metadata(dockerfile_path, context)

{agentenv_framework-0.9.1301 → agentenv_framework-0.9.1302}/src/agent_env/store/document_store/document_store.py

@@ -369,7 +369,7 @@ class DocumentStore(ABC):
         unique: bool = False,
         ttl_seconds: Optional[int] = None,
     ) -> None:
-        """Register an index over ``fields`` (idempotent); ``unique=True`` is enforced by ``insert`` and upserts via ``DuplicateKeyError``. ``ttl_seconds`` requests server-side expiry off the single indexed datetime field (backends without TTL may ignore it)."""
+        """Register an index over ``fields`` (idempotent); ``unique=True`` is enforced by ``insert`` and upserts via ``DuplicateKeyError``. ``ttl_seconds`` requests server-side expiry off the single indexed datetime field (backends without TTL may ignore it). A backend may return while a long build is still running; the index applies once built."""
 
     def check_id(self, entity_id: str) -> None:
         """Raise ValueError unless an entity with ``entity_id`` may be written here. No store takes an id

{agentenv_framework-0.9.1301 → agentenv_framework-0.9.1302}/src/agent_env/store/document_store/firestore_mongo_document_store.py

@@ -200,7 +200,7 @@ class FirestoreMongoDocumentStore(MongoDocumentStore):
             if key in _built_indexes:
                 return
             with pymongo.timeout(_INDEX_BUILD_SECONDS):
-                super().ensure_index(collection, fields, unique=unique, ttl_seconds=ttl_seconds)
+                self._create_index(collection, fields, unique=unique, ttl_seconds=ttl_seconds)
             _built_indexes.add(key)
 
     def _stamped(self, update: UpdateSpec) -> tuple[dict, str]:

{agentenv_framework-0.9.1301 → agentenv_framework-0.9.1302}/src/agent_env/store/document_store/mongo_document_store.py

@@ -8,15 +8,19 @@ currently spread across the individual ``*/store.py`` modules.
 from __future__ import annotations
 
 import logging
-from collections.abc import Iterator
-from contextlib import contextmanager
+import threading
+import time
+from collections.abc import Callable, Iterator
+from concurrent.futures import Future, wait
+from contextlib import contextmanager, suppress
+from functools import partial
 from typing import Optional
 
 from pymongo import ASCENDING, DESCENDING, MongoClient, ReturnDocument
 from pymongo.collection import Collection
 from pymongo.database import Database
 from pymongo.errors import DuplicateKeyError as PyMongoDuplicateKeyError
-from pymongo.errors import OperationFailure
+from pymongo.errors import ConnectionFailure, OperationFailure
 
 from agent_env.store.document_store.document_store import (
     AbsentOrNull,
@@ -37,6 +41,10 @@ from agent_env.store.document_store.document_store import (
 
 logger = logging.getLogger(__name__)
 
+# Not pymongo.timeout: that sends maxTimeMS, and MongoDB aborts a build whose createIndexes runs out of it.
+_INDEX_BUILD_WAIT_SECONDS = 5
+_INDEX_RETRY_SECONDS = 1
+
 
 def _field_cond(mongo: dict, field: str) -> dict:
     if field in mongo and not isinstance(mongo[field], dict):
@@ -116,6 +124,41 @@ def _unique_violations() -> Iterator[None]:
         raise DuplicateKeyError(str(e)) from e
 
 
+def _index_spec(fields: list[str], unique: bool) -> tuple[list[tuple[str, int]], str]:
+    return [(f, ASCENDING) for f in fields], "_".join(fields) + ("_unique" if unique else "_index")
+
+
+def _build_index(build: Future[bool], create: Callable[[], None], listed: Callable[[], bool]) -> None:
+    """Resolve ``build`` to True once MongoDB has built the index, or to False once a request cut off by a connection
+    failure has left a build that MongoDB lists. A request that left none is sent again, so an index the caller
+    stopped waiting for is never left missing."""
+    try:
+        while True:
+            try:
+                create()
+            except ConnectionFailure:
+                with suppress(ConnectionFailure):
+                    if listed():
+                        build.set_result(False)
+                        return
+                time.sleep(_INDEX_RETRY_SECONDS)
+            else:
+                build.set_result(True)
+                return
+    except BaseException as exc:
+        build.set_exception(exc)
+
+
+def _report_late_build(collection: str, name: str, build: Future[bool]) -> None:
+    error = build.exception()
+    if error is not None:
+        logger.error("MongoDB did not build index %s on %s: %s", name, collection, error)
+    elif build.result():
+        logger.info("MongoDB finished building index %s on %s", name, collection)
+    else:
+        logger.info("Lost the connection while MongoDB builds index %s on %s; it finishes the build", name, collection)
+
+
 def _strip_id(doc: Optional[dict]) -> Optional[dict]:
     if doc is not None:
         doc.pop("_id", None)
@@ -248,16 +291,42 @@ class MongoDocumentStore(DocumentStore):
         unique: bool = False,
         ttl_seconds: Optional[int] = None,
     ) -> None:
+        """Wait at most ``_INDEX_BUILD_WAIT_SECONDS`` for a missing plain index, then go on while a background thread
+        sees the build through: over a large collection it takes minutes, and holding up the write that asked for it
+        fails that write. A unique index is awaited in full, since writes rely on its constraint."""
+        if unique:
+            self._create_index(collection, fields, unique, ttl_seconds)
+            return
+        keys, name = _index_spec(fields, unique)
+        if self._listed(collection, keys, name):
+            return
+        build: Future[bool] = Future()
+        create = partial(self._create_index, collection, fields, unique, ttl_seconds)
+        listed = partial(self._listed, collection, keys, name)
+        thread_name = f"ensure-index-{collection}.{name}"
+        threading.Thread(target=_build_index, args=(build, create, listed), name=thread_name, daemon=True).start()
+        if wait([build], timeout=_INDEX_BUILD_WAIT_SECONDS).done and build.result():
+            return
+        logger.warning("Index %s on %s is not built yet; going on without waiting for it", name, collection)
+        build.add_done_callback(partial(_report_late_build, collection, name))
+
+    def _listed(self, collection: str, keys: list[tuple[str, int]], name: str) -> bool:
+        existing = self._c(collection).index_information()
+        return name in existing or any(spec["key"] == keys for spec in existing.values())
+
+    def _create_index(
+        self,
+        collection: str,
+        fields: list[str],
+        unique: bool = False,
+        ttl_seconds: Optional[int] = None,
+    ) -> None:
+        keys, name = _index_spec(fields, unique)
         kwargs: dict = {}
         if ttl_seconds is not None:
             kwargs["expireAfterSeconds"] = ttl_seconds
         try:
-            self._c(collection).create_index(
-                [(f, ASCENDING) for f in fields],
-                unique=unique,
-                name="_".join(fields) + ("_unique" if unique else "_index"),
-                **kwargs,
-            )
+            self._c(collection).create_index(keys, unique=unique, name=name, **kwargs)
         except OperationFailure as e:
             # 85=IndexOptionsConflict, 86=IndexKeySpecsConflict: an equivalent index
             # already exists under a different name/spec (pre-refactor). Tolerate it.

{agentenv_framework-0.9.1301 → agentenv_framework-0.9.1302}/src/agent_env/store/ids.py

@@ -114,6 +114,12 @@ def image_repository(entity_id: str) -> str:
     return key_segment(entity_id)
 
 
+def local_image_repository(entity_id: str) -> str:
+    """A repository for ``entity_id`` that is a valid OCI repository path and names no registry, whatever the id:
+    ``local/<slug>-<hash>``, the name an image built where it runs is tagged with."""
+    return _local_segment(entity_id)
+
+
 def fs_safe(entity_id: str) -> str:
     """A filename for an ``@local`` id: its key segment with no ``/``. Any other id is returned
     unchanged, so it is only as filename-safe as it already was."""