coder-eval 0.8.5__tar.gz → 0.8.7rc10__tar.gz

{coder_eval-0.8.5 → coder_eval-0.8.7rc10}/.github/workflows/release.yml

@@ -13,6 +13,13 @@ name: Release
 # and push.
 # (Continuous :latest / :sha- agent images still publish on every main push via
 # docker-publish.yml -- only the versioned release artifacts gate on this run.)
+#
+# PRERELEASE mode (dispatched from a NON-main branch): instead of bumping and
+# pushing main, stamp a throwaway `<next-patch>rc<run#>` version, then build and
+# publish the wheel to public PyPI + a `:<version>` GHCR image. It does NOT move
+# `:latest`, tag, commit, or push to main. This lets a branch be dry-run on the
+# ADO nightly infra before merge (pinned via the pipeline's `coderEvalVersion`).
+# The `bump` input is ignored off main. On `main` the behavior is unchanged.
 
 on:
   workflow_dispatch:
@@ -47,9 +54,9 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 15
     outputs:
-      # Exposed so the downstream publish-pypi job can gate on a real release
-      # having been cut (empty => no version bumped => skip PyPI publish).
-      version: ${{ steps.release.outputs.version }}
+      # Exposed so the downstream publish-pypi job gates on a version having been
+      # produced (real release on main, or a stamped prerelease on a branch).
+      version: ${{ steps.ver.outputs.version }}
     env:
       # The self-hosted `uipath-ubuntu-latest` runners enforce a minimum
       # package-age safe-chain check on uv installs; on GitHub-hosted runners
@@ -58,8 +65,13 @@ jobs:
       SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS: "openai-codex-cli-bin,openai-codex"
 
     steps:
+      # Only a real release (main) needs the app token: semantic-release pushes the
+      # bump commit + tag to the ruleset-protected main branch, and only this app
+      # has the bypass. A prerelease from a branch never commits or pushes, so it
+      # skips the token and checks out with the default GITHUB_TOKEN.
       - name: Mint release app token
         id: app-token
+        if: github.ref == 'refs/heads/main'
         uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1  # v3.2.0
         with:
           app-id: ${{ secrets.RELEASE_APP_ID }}
@@ -70,8 +82,9 @@ jobs:
         with:
           fetch-depth: 0  # semantic-release needs full history for tags + changelog
           # Persisted in .git config so semantic-release's push to main is
-          # authenticated as the app (which bypasses the branch ruleset).
-          token: ${{ steps.app-token.outputs.token }}
+          # authenticated as the app (which bypasses the branch ruleset). Falls back
+          # to the default token for a prerelease (no push, read-only checkout).
+          token: ${{ steps.app-token.outputs.token || github.token }}
 
       - name: Set up Python 3.13
         uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6.2.0
@@ -83,11 +96,38 @@ jobs:
         with:
           enable-cache: true
 
+      # Release mode is chosen by the dispatched ref: main => real release (the
+      # semantic-release path below); any other branch => prerelease (stamp a
+      # throwaway rc version, publish, never touch main). Keying off the ref (not a
+      # new input) keeps this dispatchable from a branch — a new input would have to
+      # exist on the default branch first to be accepted.
+      - name: Determine release mode
+        id: mode
+        env:
+          REF: ${{ github.ref }}
+          RUN_NUMBER: ${{ github.run_number }}
+        run: |
+          set -euo pipefail
+          if [ "$REF" = "refs/heads/main" ]; then
+            echo "prerelease=false" >> "$GITHUB_OUTPUT"
+            echo "Mode: RELEASE (main)"
+          else
+            # Next patch of the current version, suffixed with the run number so
+            # repeated dispatches never collide on PyPI. An exact `==` pin installs
+            # it even though pip/uv skip prereleases by default.
+            NEXT=$(python3 -c 'import re,tomllib; v=tomllib.load(open("pyproject.toml","rb"))["project"]["version"]; m=re.match(r"(\d+)\.(\d+)\.(\d+)",v); print("{}.{}.{}".format(int(m[1]),int(m[2]),int(m[3])+1))')
+            PRE="${NEXT}rc${RUN_NUMBER}"
+            echo "prerelease=true" >> "$GITHUB_OUTPUT"
+            echo "version=${PRE}" >> "$GITHUB_OUTPUT"
+            echo "Mode: PRERELEASE ${PRE}"
+          fi
+
       - name: Install build + release tools
         run: uv tool install python-semantic-release && uv tool install twine
 
       - name: Run semantic-release (bump + tag, no push yet)
         id: release
+        if: steps.mode.outputs.prerelease != 'true'
         env:
           GH_TOKEN: ${{ steps.app-token.outputs.token }}
           # Passed via env (not interpolated into the script) per GitHub's
@@ -105,8 +145,45 @@ jobs:
           # A dispatch always cuts a release; report the just-published version.
           echo "version=$($PSR version --print)" >> "$GITHUB_OUTPUT"
 
+      # PRERELEASE: stamp the rc version into the two canonical spots (mirrors
+      # publish-testpypi.yml), then refresh uv.lock so the Dockerfile's
+      # `uv export --frozen` accepts the bumped project version. No commit is made —
+      # the working tree is what `uv build` and the image build below consume.
+      - name: Stamp prerelease version
+        if: steps.mode.outputs.prerelease == 'true'
+        env:
+          PRE_VERSION: ${{ steps.mode.outputs.version }}
+        run: |
+          set -euo pipefail
+          python3 - <<'PY'
+          import os, re, pathlib
+          version = os.environ["PRE_VERSION"]
+          for path, key in (("pyproject.toml", "version"), ("src/coder_eval/__init__.py", "__version__")):
+              p = pathlib.Path(path)
+              new, n = re.subn(rf'(?m)^{key}\s*=\s*".+?"$', f'{key} = "{version}"', p.read_text(), count=1)
+              if n != 1:
+                  raise SystemExit(f"version pattern did not match {path} (matched {n})")
+              p.write_text(new)
+          print(f"Stamped prerelease version: {version}")
+          PY
+          uv lock
+
+      # Single source of truth for the steps below: the real release version (main)
+      # or the stamped prerelease version (branch).
+      - name: Resolve published version
+        id: ver
+        env:
+          REL: ${{ steps.release.outputs.version }}
+          PRE: ${{ steps.mode.outputs.version }}
+        run: |
+          set -euo pipefail
+          V="${REL:-$PRE}"
+          if [ -z "$V" ]; then echo "no version resolved" >&2; exit 1; fi
+          echo "version=$V" >> "$GITHUB_OUTPUT"
+          echo "Publishing version: $V"
+
       - name: Regenerate uv.lock and amend release commit
-        if: steps.release.outputs.version != ''
+        if: steps.mode.outputs.prerelease != 'true' && steps.release.outputs.version != ''
         run: |
           uv lock
           if ! git diff --quiet uv.lock; then
@@ -119,43 +196,64 @@ jobs:
           fi
 
       - name: Push release commit and tags
-        if: steps.release.outputs.version != ''
+        if: steps.mode.outputs.prerelease != 'true' && steps.release.outputs.version != ''
         run: git push origin main "v${{ steps.release.outputs.version }}"
 
       - name: Build wheel + sdist
-        if: steps.release.outputs.version != ''
+        if: steps.ver.outputs.version != ''
         run: uv build
 
       # Hand the exact built artifacts to the publish-pypi job. Publishing to
       # public PyPI runs in its own environment-gated job (OIDC), so it must
       # consume these files rather than rebuild them.
       - name: Upload dist for PyPI publish
-        if: steps.release.outputs.version != ''
+        if: steps.ver.outputs.version != ''
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02  # v4.6.2
         with:
           name: release-dist
           path: dist/
           if-no-files-found: error
 
-      # Build + push the agent image HERE, in the same job that cut the release,
-      # so the `:<version>` tag is built from the BUMPED pyproject (the version
-      # only exists after the semantic-release step above). docker-publish.yml
-      # runs on the triggering commit, BEFORE the bump, so it can never tag the
-      # release version -- this is the authoritative versioned image. It also
-      # repoints `:latest` to the exact release artifact.
+      # Build + push the agent image HERE, in the same job that produced the
+      # version, so the `:<version>` tag is built from the correct pyproject (bumped
+      # by semantic-release on main, or the stamped rc on a prerelease).
+      # docker-publish.yml runs on the triggering commit, BEFORE a main bump, so it
+      # can never tag the release version -- this is the authoritative versioned
+      # image. A real release also repoints `:latest`; a prerelease publishes only
+      # its `:<version>` tag (see Compute image tags).
       - name: Lowercase owner for GHCR
-        if: steps.release.outputs.version != ''
+        if: steps.ver.outputs.version != ''
         id: img
         run: echo "owner_lc=$(echo '${{ github.repository_owner }}' | tr '[:upper:]' '[:lower:]')" >> "$GITHUB_OUTPUT"
 
+      # `:<version>` always; `:latest` only for a real release. A prerelease must
+      # not move `:latest`, which the nightly treats as tip-of-main.
+      - name: Compute image tags
+        if: steps.ver.outputs.version != ''
+        id: tags
+        env:
+          OWNER: ${{ steps.img.outputs.owner_lc }}
+          VERSION: ${{ steps.ver.outputs.version }}
+          IS_PRERELEASE: ${{ steps.mode.outputs.prerelease }}
+        run: |
+          set -euo pipefail
+          {
+            echo "tags<<EOF"
+            echo "ghcr.io/${OWNER}/coder-eval-agent:${VERSION}"
+            if [ "$IS_PRERELEASE" != "true" ]; then
+              echo "ghcr.io/${OWNER}/coder-eval-agent:latest"
+            fi
+            echo "EOF"
+          } >> "$GITHUB_OUTPUT"
+
       - name: Set up Docker Buildx
         continue-on-error: true  # GHCR image is internal/best-effort; don't block PyPI
-        if: steps.release.outputs.version != ''
+        if: steps.ver.outputs.version != ''
         uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f  # v3.12.0
 
       - name: Log in to GHCR
         continue-on-error: true  # GHCR image is internal/best-effort; don't block PyPI
-        if: steps.release.outputs.version != ''
+        if: steps.ver.outputs.version != ''
         uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9  # v3.7.0
         with:
           registry: ghcr.io
@@ -164,17 +262,15 @@ jobs:
 
       - name: Build and push versioned agent image
         continue-on-error: true  # GHCR image is internal/best-effort; don't block PyPI
-        if: steps.release.outputs.version != ''
+        if: steps.ver.outputs.version != ''
         uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8  # v6.19.2
         with:
           context: .
           file: docker/Dockerfile
           push: true
-          tags: |
-            ghcr.io/${{ steps.img.outputs.owner_lc }}/coder-eval-agent:${{ steps.release.outputs.version }}
-            ghcr.io/${{ steps.img.outputs.owner_lc }}/coder-eval-agent:latest
+          tags: ${{ steps.tags.outputs.tags }}
           build-args: |
-            CODER_EVAL_VERSION=${{ steps.release.outputs.version }}
+            CODER_EVAL_VERSION=${{ steps.ver.outputs.version }}
           secrets: |
             "uv_index_username=${{ secrets.UV_INDEX_UIPATH_USERNAME }}"
             "uv_index_password=${{ secrets.UV_INDEX_UIPATH_PASSWORD }}"

{coder_eval-0.8.5 → coder_eval-0.8.7rc10}/CHANGELOG.md

@@ -2,6 +2,15 @@
 
 <!-- version list -->
 
+## v0.8.6 (2026-07-20)
+
+### Features
+
+- Add Sonnet 5 + GPT-5.6 pricing; default antigravity to gemini-3.5-flash
+  ([#31](https://github.com/UiPath/coder_eval/pull/31),
+  [`4c40acd`](https://github.com/UiPath/coder_eval/commit/4c40acd64482e9ffcec66b61ac2aab57b5c89f6f))
+
+
 ## v0.8.5 (2026-07-20)
 
 ### Bug Fixes

{coder_eval-0.8.5 → coder_eval-0.8.7rc10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: coder-eval
-Version: 0.8.5
+Version: 0.8.7rc10
 Summary: Evaluate, benchmark, and A/B-test AI coding agents (Claude Code, Codex, Gemini/Antigravity) with sandboxed, reproducible YAML task suites.
 Project-URL: Homepage, https://github.com/UiPath/coder_eval
 Project-URL: Repository, https://github.com/UiPath/coder_eval
@@ -26,7 +26,7 @@ Requires-Python: >=3.13
 Requires-Dist: anthropic>=0.86.0
 Requires-Dist: anyio>=4.13.0
 Requires-Dist: azure-monitor-opentelemetry-exporter<1.1.0,>=1.0.0b30
-Requires-Dist: claude-agent-sdk>=0.2.82
+Requires-Dist: claude-agent-sdk>=0.2.124
 Requires-Dist: click>=8.3.3
 Requires-Dist: jmespath>=1.1.0
 Requires-Dist: jsonschema>=4.26.0
@@ -41,9 +41,9 @@ Requires-Dist: starlette>=1.3.1
 Requires-Dist: tqdm>=4.67.3
 Requires-Dist: typer>=0.24.1
 Provides-Extra: antigravity
-Requires-Dist: google-antigravity==0.1.5; extra == 'antigravity'
+Requires-Dist: google-antigravity==0.1.7; extra == 'antigravity'
 Provides-Extra: codex
-Requires-Dist: openai-codex>=0.1.0b3; extra == 'codex'
+Requires-Dist: openai-codex>=0.144.4; extra == 'codex'
 Provides-Extra: dev
 Requires-Dist: bandit[toml]>=1.9.4; extra == 'dev'
 Requires-Dist: mcp>=1.28.1; extra == 'dev'

{coder_eval-0.8.5 → coder_eval-0.8.7rc10}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "coder-eval"
-version = "0.8.5"
+version = "0.8.7rc10"
 description = "Evaluate, benchmark, and A/B-test AI coding agents (Claude Code, Codex, Gemini/Antigravity) with sandboxed, reproducible YAML task suites."
 readme = "README.md"
 license = "Apache-2.0"
@@ -34,7 +34,7 @@ dependencies = [
     "rich>=14.3.3",
     "python-dotenv>=1.2.2",
     "anthropic>=0.86.0",
-    "claude-agent-sdk>=0.2.82",
+    "claude-agent-sdk>=0.2.124",
     "anyio>=4.13.0",
     "radon>=6.0.1",
     "tqdm>=4.67.3",
@@ -85,19 +85,24 @@ uipath = [
 # Without this extra, the framework still installs and runs; Codex-dependent
 # code paths fail at dispatch with a clear hint pointing back here.
 codex = [
-    "openai-codex>=0.1.0b3",
+    "openai-codex>=0.144.4",
 ]
 # Optional extra that enables Antigravity agent support:
 #   - AntigravityAgent implementation using Google's official google-antigravity SDK
 #     (drives the bundled `localharness` binary, authenticated via GEMINI_API_KEY).
 # Pinned exactly: the harness binary is a dominant non-model driver of eval results,
 # so it travels with the coder_eval release tag and is bumped deliberately (mirrors
-# the claude-code CLI / codex cli-bin pins). The wheel ships platform-specific
-# (manylinux x86_64 + aarch64, macOS, Windows), so CI/Linux/Docker installs work.
+# the claude-code CLI / codex cli-bin pins).
+# glibc note — verify before bumping: the wheel's manylinux_2_17 tag is NOT a
+# reliable floor; the bundled `localharness` ELF can require a newer glibc than the
+# tag advertises. Eval agents run on Ubuntu 22.04 (glibc 2.35), so the pinned build
+# must load there. 0.1.7 links without DT_RELR (max required symbol GLIBC_2.26) and
+# loads on 2.35+; confirm any future pin does too (readelf -d | grep RELR must be
+# empty / max GLIBC symbol <= the oldest target's glibc).
 # Without this extra the framework still installs and runs; Antigravity-dependent
 # code paths fail at start() with a clear hint pointing back here.
 antigravity = [
-    "google-antigravity==0.1.5",
+    "google-antigravity==0.1.7",
 ]
 
 [project.scripts]
@@ -129,12 +134,14 @@ packages = ["src/coder_eval"]
 allow-direct-references = true
 
 [tool.uv]
-# openai-codex 0.1.0b3 hardpins `openai-codex-cli-bin==0.137.0a4`, a
-# PRE-RELEASE. Because that pin is transitive (not referenced directly by us),
-# uv won't select it without an explicit opt-in. Naming it here as an override
-# both enables the pre-release and documents the pinned cli-bin build; 0.137.0a4
-# publishes manylinux wheels (x86_64 + aarch64), so CI/Linux installs work.
-override-dependencies = ["openai-codex-cli-bin==0.137.0a4"]
+# openai-codex 0.144.4 hardpins `openai-codex-cli-bin==0.144.4` (a stable
+# release; the SDK version now tracks the codex CLI version line). Naming it
+# here documents and holds the pinned cli-bin build — the harness binary is a
+# dominant non-model driver of eval results, so it travels with the coder_eval
+# release tag and is bumped deliberately (mirrors the antigravity localharness
+# and claude-code CLI pins). 0.144.4 publishes manylinux wheels (x86_64 +
+# aarch64), so CI/Linux installs work.
+override-dependencies = ["openai-codex-cli-bin==0.144.4"]
 
 constraint-dependencies = [
     # Fix known CVEs in transitive dependencies

{coder_eval-0.8.5 → coder_eval-0.8.7rc10}/src/coder_eval/__init__.py

@@ -1,3 +1,3 @@
 """coder_eval - A framework for evaluating AI coding agents."""
 
-__version__ = "0.8.5"
+__version__ = "0.8.7rc10"

{coder_eval-0.8.5 → coder_eval-0.8.7rc10}/src/coder_eval/agent.py

@@ -167,6 +167,7 @@ class Agent[ConfigT: BaseAgentConfig](ABC):
         *,
         env_path_prepend: list[str] | None = None,
         plugin_tools_dir: str | None = None,
+        sandbox_managed: bool = False,
     ) -> None:
         """Initialize and start the agent.
 
@@ -180,6 +181,10 @@ class Agent[ConfigT: BaseAgentConfig](ABC):
                 instead of walking up from CWD. An external ``PLUGIN_TOOLS_DIR`` in
                 the process environment still wins. Implementations that don't shell
                 out may ignore this argument.
+            sandbox_managed: True when coder_eval itself provides this run's isolation
+                boundary (the ephemeral host tempdir it created for the task). Agents
+                that run their own in-process OS sandbox may use this to skip it as
+                redundant; agents without one may ignore this argument.
         """
         pass
 

{coder_eval-0.8.5 → coder_eval-0.8.7rc10}/src/coder_eval/agents/antigravity_agent.py

@@ -93,10 +93,10 @@ def _harness_spawn_lock() -> asyncio.Lock:
 
 
 # Recommended Gemini coding model when a task pins no ``agent.model`` and neither
-# ``--model`` nor ``ANTIGRAVITY_MODEL`` is set. Gemini 3.1 Pro is the current
-# flagship agentic-coding model (the earlier ``gemini-3-pro-preview`` is retired);
-# ``medium`` thinking is its recommended daily-driver default.
-_DEFAULT_MODEL = "gemini-3.1-pro-preview"
+# ``--model`` nor ``ANTIGRAVITY_MODEL`` is set. Gemini 3.5 Flash is Antigravity 2.0's
+# default coding model (2026-05) — it outperforms the older Gemini 3.1 Pro on coding /
+# agentic benchmarks while running faster; ``medium`` thinking is its daily-driver default.
+_DEFAULT_MODEL = "gemini-3.5-flash"
 
 # Antigravity builtin tool name -> canonical Claude-ish tool name, so cross-agent
 # success criteria (command_executed / commands_efficiency / skill_triggered) and
@@ -290,9 +290,14 @@ class AntigravityAgent(Agent[AntigravityAgentConfig]):
         *,
         env_path_prepend: list[str] | None = None,
         plugin_tools_dir: str | None = None,
+        sandbox_managed: bool = False,
     ) -> None:
         """Initialize and start the Antigravity agent's local harness session.
 
+        (``sandbox_managed`` is accepted for interface parity with the base
+        ``Agent.start`` and ignored: the Antigravity harness has no in-process
+        OS sandbox to drop.)
+
         Args:
             working_directory: Path to the sandbox working directory. The primary
                 ``workspace`` so file writes (and run_command) operate there —

{coder_eval-0.8.5 → coder_eval-0.8.7rc10}/src/coder_eval/agents/claude_code_agent.py

@@ -692,6 +692,7 @@ class ClaudeCodeAgent(Agent[ClaudeCodeAgentConfig]):
         *,
         env_path_prepend: list[str] | None = None,
         plugin_tools_dir: str | None = None,
+        sandbox_managed: bool = False,
     ) -> None:
         """Initialize and start the Claude Code agent.
 
@@ -701,6 +702,9 @@ class ClaudeCodeAgent(Agent[ClaudeCodeAgentConfig]):
                 subprocess (typically the resolved ``SandboxConfig.mock_path_dirs``).
             plugin_tools_dir: Canonical ``node_modules/@uipath`` to export as
                 ``PLUGIN_TOOLS_DIR``. An external env-var pin still wins.
+            sandbox_managed: Accepted for interface parity with the base
+                ``Agent.start`` and ignored — Claude Code runs the tempdir path
+                with no in-agent OS sandbox to drop.
         """
         self.working_directory = Path(working_directory)
         self._env_path_prepend = list(env_path_prepend or [])