coder-eval 0.8.6__tar.gz → 0.8.7__tar.gz

coder_eval-0.8.7/.github/workflows/docs.yml

@@ -0,0 +1,48 @@
+name: Docs
+
+# Build the MkDocs Material site and publish it to the `gh-pages` branch via
+# `mkdocs gh-deploy` — the same mechanism UiPath/uipath-python uses. This workflow
+# only pushes a branch (needs `contents: write`); it never calls the Pages API, so
+# it succeeds even before Pages is switched on.
+#
+# One-time setup (org owner): the UiPath org blocks Pages *creation*, so enable
+# Pages for this repo once — Settings → Pages → Build and deployment →
+# Source: "Deploy from a branch" → branch `gh-pages` / `/ (root)`. That's the same
+# (legacy, branch-based) configuration uipath-python already runs on.
+on:
+  push:
+    branches: [main]
+    paths:
+      - "docs/**"
+      - "mkdocs.yml"
+      - ".github/workflows/docs.yml"
+  workflow_dispatch:
+
+permissions:
+  contents: write
+
+concurrency:
+  group: docs-gh-pages
+  cancel-in-progress: true
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+      - name: Install social-card system libraries
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends \
+            libcairo2 libfreetype6 libjpeg-turbo8 libpng16-16 pngquant
+      - name: Install MkDocs Material
+        run: pip install "mkdocs-material[imaging]>=9.5,<10"
+      - name: Configure git identity for gh-pages commits
+        run: |
+          git config user.name "coder-eval"
+          git config user.email "coder-eval@uipath.com"
+      - name: Build strictly and publish to gh-pages
+        run: mkdocs gh-deploy --force --strict

{coder_eval-0.8.6 → coder_eval-0.8.7}/.github/workflows/release.yml

@@ -13,12 +13,19 @@ name: Release
 # and push.
 # (Continuous :latest / :sha- agent images still publish on every main push via
 # docker-publish.yml -- only the versioned release artifacts gate on this run.)
+#
+# PRERELEASE mode (dispatched from a NON-main branch): instead of bumping and
+# pushing main, stamp a throwaway `<next-patch>rc<run#>` version, then build and
+# publish the wheel to public PyPI + a `:<version>` GHCR image. It does NOT move
+# `:latest`, tag, commit, or push to main. This lets a branch be dry-run on the
+# ADO nightly infra before merge (pinned via the pipeline's `coderEvalVersion`).
+# The `bump` input is ignored off main. On `main` the behavior is unchanged.
 
 on:
   workflow_dispatch:
     inputs:
       bump:
-        description: 'Version bump to release'
+        description: 'Version bump to release (ignored on non-main / prerelease dispatch, which always stamps a next-patch rc)'
         type: choice
         default: patch
         options:
@@ -47,9 +54,9 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 15
     outputs:
-      # Exposed so the downstream publish-pypi job can gate on a real release
-      # having been cut (empty => no version bumped => skip PyPI publish).
-      version: ${{ steps.release.outputs.version }}
+      # Exposed so the downstream publish-pypi job gates on a version having been
+      # produced (real release on main, or a stamped prerelease on a branch).
+      version: ${{ steps.ver.outputs.version }}
     env:
       # The self-hosted `uipath-ubuntu-latest` runners enforce a minimum
       # package-age safe-chain check on uv installs; on GitHub-hosted runners
@@ -58,8 +65,13 @@ jobs:
       SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS: "openai-codex-cli-bin,openai-codex"
 
     steps:
+      # Only a real release (main) needs the app token: semantic-release pushes the
+      # bump commit + tag to the ruleset-protected main branch, and only this app
+      # has the bypass. A prerelease from a branch never commits or pushes, so it
+      # skips the token and checks out with the default GITHUB_TOKEN.
       - name: Mint release app token
         id: app-token
+        if: github.ref == 'refs/heads/main'
         uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1  # v3.2.0
         with:
           app-id: ${{ secrets.RELEASE_APP_ID }}
@@ -70,8 +82,9 @@ jobs:
         with:
           fetch-depth: 0  # semantic-release needs full history for tags + changelog
           # Persisted in .git config so semantic-release's push to main is
-          # authenticated as the app (which bypasses the branch ruleset).
-          token: ${{ steps.app-token.outputs.token }}
+          # authenticated as the app (which bypasses the branch ruleset). Falls back
+          # to the default token for a prerelease (no push, read-only checkout).
+          token: ${{ steps.app-token.outputs.token || github.token }}
 
       - name: Set up Python 3.13
         uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6.2.0
@@ -83,11 +96,38 @@ jobs:
         with:
           enable-cache: true
 
+      # Release mode is chosen by the dispatched ref: main => real release (the
+      # semantic-release path below); any other branch => prerelease (stamp a
+      # throwaway rc version, publish, never touch main). Keying off the ref (not a
+      # new input) keeps this dispatchable from a branch — a new input would have to
+      # exist on the default branch first to be accepted.
+      - name: Determine release mode
+        id: mode
+        env:
+          REF: ${{ github.ref }}
+          RUN_NUMBER: ${{ github.run_number }}
+        run: |
+          set -euo pipefail
+          if [ "$REF" = "refs/heads/main" ]; then
+            echo "prerelease=false" >> "$GITHUB_OUTPUT"
+            echo "Mode: RELEASE (main)"
+          else
+            # Next patch of the current version, suffixed with the run number so
+            # repeated dispatches never collide on PyPI. An exact `==` pin installs
+            # it even though pip/uv skip prereleases by default.
+            NEXT=$(python3 -c 'import re,tomllib; v=tomllib.load(open("pyproject.toml","rb"))["project"]["version"]; m=re.match(r"(\d+)\.(\d+)\.(\d+)",v); print("{}.{}.{}".format(int(m[1]),int(m[2]),int(m[3])+1))')
+            PRE="${NEXT}rc${RUN_NUMBER}"
+            echo "prerelease=true" >> "$GITHUB_OUTPUT"
+            echo "version=${PRE}" >> "$GITHUB_OUTPUT"
+            echo "Mode: PRERELEASE ${PRE}"
+          fi
+
       - name: Install build + release tools
         run: uv tool install python-semantic-release && uv tool install twine
 
       - name: Run semantic-release (bump + tag, no push yet)
         id: release
+        if: steps.mode.outputs.prerelease != 'true'
         env:
           GH_TOKEN: ${{ steps.app-token.outputs.token }}
           # Passed via env (not interpolated into the script) per GitHub's
@@ -105,8 +145,45 @@ jobs:
           # A dispatch always cuts a release; report the just-published version.
           echo "version=$($PSR version --print)" >> "$GITHUB_OUTPUT"
 
+      # PRERELEASE: stamp the rc version into the two canonical spots (mirrors
+      # publish-testpypi.yml), then refresh uv.lock so the Dockerfile's
+      # `uv export --frozen` accepts the bumped project version. No commit is made —
+      # the working tree is what `uv build` and the image build below consume.
+      - name: Stamp prerelease version
+        if: steps.mode.outputs.prerelease == 'true'
+        env:
+          PRE_VERSION: ${{ steps.mode.outputs.version }}
+        run: |
+          set -euo pipefail
+          python3 - <<'PY'
+          import os, re, pathlib
+          version = os.environ["PRE_VERSION"]
+          for path, key in (("pyproject.toml", "version"), ("src/coder_eval/__init__.py", "__version__")):
+              p = pathlib.Path(path)
+              new, n = re.subn(rf'(?m)^{key}\s*=\s*".+?"$', f'{key} = "{version}"', p.read_text(), count=1)
+              if n != 1:
+                  raise SystemExit(f"version pattern did not match {path} (matched {n})")
+              p.write_text(new)
+          print(f"Stamped prerelease version: {version}")
+          PY
+          uv lock
+
+      # Single source of truth for the steps below: the real release version (main)
+      # or the stamped prerelease version (branch).
+      - name: Resolve published version
+        id: ver
+        env:
+          REL: ${{ steps.release.outputs.version }}
+          PRE: ${{ steps.mode.outputs.version }}
+        run: |
+          set -euo pipefail
+          V="${REL:-$PRE}"
+          if [ -z "$V" ]; then echo "no version resolved" >&2; exit 1; fi
+          echo "version=$V" >> "$GITHUB_OUTPUT"
+          echo "Publishing version: $V"
+
       - name: Regenerate uv.lock and amend release commit
-        if: steps.release.outputs.version != ''
+        if: steps.mode.outputs.prerelease != 'true' && steps.release.outputs.version != ''
         run: |
           uv lock
           if ! git diff --quiet uv.lock; then
@@ -119,43 +196,64 @@ jobs:
           fi
 
       - name: Push release commit and tags
-        if: steps.release.outputs.version != ''
+        if: steps.mode.outputs.prerelease != 'true' && steps.release.outputs.version != ''
         run: git push origin main "v${{ steps.release.outputs.version }}"
 
       - name: Build wheel + sdist
-        if: steps.release.outputs.version != ''
+        if: steps.ver.outputs.version != ''
         run: uv build
 
       # Hand the exact built artifacts to the publish-pypi job. Publishing to
       # public PyPI runs in its own environment-gated job (OIDC), so it must
       # consume these files rather than rebuild them.
       - name: Upload dist for PyPI publish
-        if: steps.release.outputs.version != ''
+        if: steps.ver.outputs.version != ''
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02  # v4.6.2
         with:
           name: release-dist
           path: dist/
           if-no-files-found: error
 
-      # Build + push the agent image HERE, in the same job that cut the release,
-      # so the `:<version>` tag is built from the BUMPED pyproject (the version
-      # only exists after the semantic-release step above). docker-publish.yml
-      # runs on the triggering commit, BEFORE the bump, so it can never tag the
-      # release version -- this is the authoritative versioned image. It also
-      # repoints `:latest` to the exact release artifact.
+      # Build + push the agent image HERE, in the same job that produced the
+      # version, so the `:<version>` tag is built from the correct pyproject (bumped
+      # by semantic-release on main, or the stamped rc on a prerelease).
+      # docker-publish.yml runs on the triggering commit, BEFORE a main bump, so it
+      # can never tag the release version -- this is the authoritative versioned
+      # image. A real release also repoints `:latest`; a prerelease publishes only
+      # its `:<version>` tag (see Compute image tags).
       - name: Lowercase owner for GHCR
-        if: steps.release.outputs.version != ''
+        if: steps.ver.outputs.version != ''
         id: img
         run: echo "owner_lc=$(echo '${{ github.repository_owner }}' | tr '[:upper:]' '[:lower:]')" >> "$GITHUB_OUTPUT"
 
+      # `:<version>` always; `:latest` only for a real release. A prerelease must
+      # not move `:latest`, which the nightly treats as tip-of-main.
+      - name: Compute image tags
+        if: steps.ver.outputs.version != ''
+        id: tags
+        env:
+          OWNER: ${{ steps.img.outputs.owner_lc }}
+          VERSION: ${{ steps.ver.outputs.version }}
+          IS_PRERELEASE: ${{ steps.mode.outputs.prerelease }}
+        run: |
+          set -euo pipefail
+          {
+            echo "tags<<EOF"
+            echo "ghcr.io/${OWNER}/coder-eval-agent:${VERSION}"
+            if [ "$IS_PRERELEASE" != "true" ]; then
+              echo "ghcr.io/${OWNER}/coder-eval-agent:latest"
+            fi
+            echo "EOF"
+          } >> "$GITHUB_OUTPUT"
+
       - name: Set up Docker Buildx
         continue-on-error: true  # GHCR image is internal/best-effort; don't block PyPI
-        if: steps.release.outputs.version != ''
+        if: steps.ver.outputs.version != ''
         uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f  # v3.12.0
 
       - name: Log in to GHCR
         continue-on-error: true  # GHCR image is internal/best-effort; don't block PyPI
-        if: steps.release.outputs.version != ''
+        if: steps.ver.outputs.version != ''
         uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9  # v3.7.0
         with:
           registry: ghcr.io
@@ -164,17 +262,15 @@ jobs:
 
       - name: Build and push versioned agent image
         continue-on-error: true  # GHCR image is internal/best-effort; don't block PyPI
-        if: steps.release.outputs.version != ''
+        if: steps.ver.outputs.version != ''
         uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8  # v6.19.2
         with:
           context: .
           file: docker/Dockerfile
           push: true
-          tags: |
-            ghcr.io/${{ steps.img.outputs.owner_lc }}/coder-eval-agent:${{ steps.release.outputs.version }}
-            ghcr.io/${{ steps.img.outputs.owner_lc }}/coder-eval-agent:latest
+          tags: ${{ steps.tags.outputs.tags }}
           build-args: |
-            CODER_EVAL_VERSION=${{ steps.release.outputs.version }}
+            CODER_EVAL_VERSION=${{ steps.ver.outputs.version }}
           secrets: |
             "uv_index_username=${{ secrets.UV_INDEX_UIPATH_USERNAME }}"
             "uv_index_password=${{ secrets.UV_INDEX_UIPATH_PASSWORD }}"

{coder_eval-0.8.6 → coder_eval-0.8.7}/.gitignore

@@ -1,3 +1,7 @@
+# MkDocs build output
+site/
+.cache/
+
 # Python-generated files
 __pycache__/
 *.py[oc]

{coder_eval-0.8.6 → coder_eval-0.8.7}/CHANGELOG.md

@@ -2,6 +2,100 @@
 
 <!-- version list -->
 
+## v0.8.7 (2026-07-22)
+
+### Bug Fixes
+
+- Detect Windows skill paths in telemetry ([#24](https://github.com/UiPath/coder_eval/pull/24),
+  [`c57a6b0`](https://github.com/UiPath/coder_eval/commit/c57a6b0408ef116177b15bc9b09a768096aaa33f))
+
+- **agents**: Run codex + antigravity harnesses on the tempdir/host path
+  ([#33](https://github.com/UiPath/coder_eval/pull/33),
+  [`04498a9`](https://github.com/UiPath/coder_eval/commit/04498a96f6b9ac0cc5c73f36e73a361f3f3fa0ae))
+
+- **antigravity**: Pin google-antigravity 0.1.7 to load on glibc 2.35
+  ([#33](https://github.com/UiPath/coder_eval/pull/33),
+  [`04498a9`](https://github.com/UiPath/coder_eval/commit/04498a96f6b9ac0cc5c73f36e73a361f3f3fa0ae))
+
+- **codex**: Always run full-access; drop the in-process OS sandbox
+  ([#33](https://github.com/UiPath/coder_eval/pull/33),
+  [`04498a9`](https://github.com/UiPath/coder_eval/commit/04498a96f6b9ac0cc5c73f36e73a361f3f3fa0ae))
+
+- **codex**: Cover zsh login shells (macOS default) in the mock-PATH home shim
+  ([#26](https://github.com/UiPath/coder_eval/pull/26),
+  [`73f0db2`](https://github.com/UiPath/coder_eval/commit/73f0db24df0b67f82d46e3bf15c85a21837237ef))
+
+- **codex**: Keep mock CLIs shadowed in bash login shells
+  ([#26](https://github.com/UiPath/coder_eval/pull/26),
+  [`73f0db2`](https://github.com/UiPath/coder_eval/commit/73f0db24df0b67f82d46e3bf15c85a21837237ef))
+
+- **codex**: Make login-shell temp-home lifecycle exception-safe incl. kill_sync
+  ([#26](https://github.com/UiPath/coder_eval/pull/26),
+  [`73f0db2`](https://github.com/UiPath/coder_eval/commit/73f0db24df0b67f82d46e3bf15c85a21837237ef))
+
+- **codex**: Restore mock-CLI PATH prepend in login shells via per-task HOME profile
+  ([#26](https://github.com/UiPath/coder_eval/pull/26),
+  [`73f0db2`](https://github.com/UiPath/coder_eval/commit/73f0db24df0b67f82d46e3bf15c85a21837237ef))
+
+- **codex**: Restore original HOME inside generated login-shell profile
+  ([#26](https://github.com/UiPath/coder_eval/pull/26),
+  [`73f0db2`](https://github.com/UiPath/coder_eval/commit/73f0db24df0b67f82d46e3bf15c85a21837237ef))
+
+- **codex**: Use full-access sandbox on coder_eval-managed tempdir path
+  ([#33](https://github.com/UiPath/coder_eval/pull/33),
+  [`04498a9`](https://github.com/UiPath/coder_eval/commit/04498a96f6b9ac0cc5c73f36e73a361f3f3fa0ae))
+
+- **deps**: Bump pyasn1 to 0.6.4 for GHSA-8ppf-4f7h-5ppj / GHSA-hm4w-wwcw-mr6r
+  ([#33](https://github.com/UiPath/coder_eval/pull/33),
+  [`04498a9`](https://github.com/UiPath/coder_eval/commit/04498a96f6b9ac0cc5c73f36e73a361f3f3fa0ae))
+
+### Chores
+
+- **deps**: Bump pyasn1 0.6.3 -> 0.6.4 (GHSA-8ppf-4f7h-5ppj, GHSA-hm4w-wwcw-mr6r)
+  ([#32](https://github.com/UiPath/coder_eval/pull/32),
+  [`f1f4f9f`](https://github.com/UiPath/coder_eval/commit/f1f4f9fba2bd034a271e50fc1a95bcc74c2ac19b))
+
+- **deps**: Upgrade agent SDKs to latest (claude 0.2.124, codex 0.144.4)
+  ([#33](https://github.com/UiPath/coder_eval/pull/33),
+  [`04498a9`](https://github.com/UiPath/coder_eval/commit/04498a96f6b9ac0cc5c73f36e73a361f3f3fa0ae))
+
+### Continuous Integration
+
+- **release**: Publish a prerelease from a non-main branch
+  ([#33](https://github.com/UiPath/coder_eval/pull/33),
+  [`04498a9`](https://github.com/UiPath/coder_eval/commit/04498a96f6b9ac0cc5c73f36e73a361f3f3fa0ae))
+
+### Documentation
+
+- Add MkDocs docs site, comparison page, and SEO metadata
+  ([#32](https://github.com/UiPath/coder_eval/pull/32),
+  [`f1f4f9f`](https://github.com/UiPath/coder_eval/commit/f1f4f9fba2bd034a271e50fc1a95bcc74c2ac19b))
+
+- Address PR review — Coder Eval naming, cleaner table, gh-deploy workflow
+  ([#32](https://github.com/UiPath/coder_eval/pull/32),
+  [`f1f4f9f`](https://github.com/UiPath/coder_eval/commit/f1f4f9fba2bd034a271e50fc1a95bcc74c2ac19b))
+
+- Address review — git identity for gh-pages, single strict deploy, table glyphs, uv tool install in
+  Tutorial 01 ([#32](https://github.com/UiPath/coder_eval/pull/32),
+  [`f1f4f9f`](https://github.com/UiPath/coder_eval/commit/f1f4f9fba2bd034a271e50fc1a95bcc74c2ac19b))
+
+### Refactoring
+
+- **codex**: Drop dead sandbox branch + honest full-access messaging
+  ([#33](https://github.com/UiPath/coder_eval/pull/33),
+  [`04498a9`](https://github.com/UiPath/coder_eval/commit/04498a96f6b9ac0cc5c73f36e73a361f3f3fa0ae))
+
+### Testing
+
+- Accept sandbox_managed kwarg in agent test doubles
+  ([#33](https://github.com/UiPath/coder_eval/pull/33),
+  [`04498a9`](https://github.com/UiPath/coder_eval/commit/04498a96f6b9ac0cc5c73f36e73a361f3f3fa0ae))
+
+- **codex**: Pin start-to-CodexConfig env composition and tidy test imports
+  ([#26](https://github.com/UiPath/coder_eval/pull/26),
+  [`73f0db2`](https://github.com/UiPath/coder_eval/commit/73f0db24df0b67f82d46e3bf15c85a21837237ef))
+
+
 ## v0.8.6 (2026-07-20)
 
 ### Features

{coder_eval-0.8.6 → coder_eval-0.8.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: coder-eval
-Version: 0.8.6
+Version: 0.8.7
 Summary: Evaluate, benchmark, and A/B-test AI coding agents (Claude Code, Codex, Gemini/Antigravity) with sandboxed, reproducible YAML task suites.
 Project-URL: Homepage, https://github.com/UiPath/coder_eval
 Project-URL: Repository, https://github.com/UiPath/coder_eval
@@ -11,7 +11,7 @@ Author-email: UiPath <coder-eval@uipath.com>
 License-Expression: Apache-2.0
 License-File: LICENSE
 License-File: NOTICE
-Keywords: agent,agent-evaluation,agent-skills,ai,anthropic,antigravity,benchmark,claude,claude-code,claude-skills,code-generation,codex,coding-agent,eval,evals,evaluation,gemini,llm,llm-evaluation,sandbox,skills-evaluation,skillsbench,swe-bench
+Keywords: agent,agent-evaluation,agent-skills,agent-testing,ai,ai-evaluation,anthropic,antigravity,benchmark,claude,claude-code,claude-code-skills,claude-skills,code-generation,codex,coding-agent,eval,evals,evaluation,gemini,llm,llm-eval,llm-evaluation,llmops,sandbox,skills-evaluation,skillsbench,swe-bench
 Classifier: Development Status :: 4 - Beta
 Classifier: Environment :: Console
 Classifier: Intended Audience :: Developers
@@ -26,7 +26,7 @@ Requires-Python: >=3.13
 Requires-Dist: anthropic>=0.86.0
 Requires-Dist: anyio>=4.13.0
 Requires-Dist: azure-monitor-opentelemetry-exporter<1.1.0,>=1.0.0b30
-Requires-Dist: claude-agent-sdk>=0.2.82
+Requires-Dist: claude-agent-sdk>=0.2.124
 Requires-Dist: click>=8.3.3
 Requires-Dist: jmespath>=1.1.0
 Requires-Dist: jsonschema>=4.26.0
@@ -41,9 +41,9 @@ Requires-Dist: starlette>=1.3.1
 Requires-Dist: tqdm>=4.67.3
 Requires-Dist: typer>=0.24.1
 Provides-Extra: antigravity
-Requires-Dist: google-antigravity==0.1.5; extra == 'antigravity'
+Requires-Dist: google-antigravity==0.1.7; extra == 'antigravity'
 Provides-Extra: codex
-Requires-Dist: openai-codex>=0.1.0b3; extra == 'codex'
+Requires-Dist: openai-codex>=0.144.4; extra == 'codex'
 Provides-Extra: dev
 Requires-Dist: bandit[toml]>=1.9.4; extra == 'dev'
 Requires-Dist: mcp>=1.28.1; extra == 'dev'
@@ -60,28 +60,36 @@ Provides-Extra: uipath
 Requires-Dist: uipath>=2.10.31; extra == 'uipath'
 Description-Content-Type: text/markdown
 
-# coder_eval — evaluate AI coding agents & their skills
+# Coder Eval — evaluate & benchmark AI coding agents and Claude Code skills
 
 [![PyPI](https://img.shields.io/pypi/v/coder-eval.svg)](https://pypi.org/project/coder-eval/)
+[![Docs](https://img.shields.io/badge/docs-uipath.github.io%2Fcoder__eval-1f6feb.svg)](https://uipath.github.io/coder_eval/)
 [![License: Apache 2.0](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](LICENSE)
 [![Python 3.13+](https://img.shields.io/badge/python-3.13%2B-blue.svg)](https://www.python.org/downloads/)
 [![CI](https://github.com/UiPath/coder_eval/actions/workflows/pr-checks.yml/badge.svg)](https://github.com/UiPath/coder_eval/actions/workflows/pr-checks.yml)
 [![Code style: Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
 
-A framework for evaluating AI coding agents **and their skills** — built for CLI
+**Coder Eval** (`pip install coder-eval` / `uv tool install coder-eval`) is an open-source framework for
+**evaluating and benchmarking AI coding agents and their skills** — built for CLI
 and skill builders — with sandboxing, reproducibility, and data-driven analysis.
-Not an "agentic coding" benchmark: it measures how effective your CLI and skills
-are when used by coding agents.
+It runs a real agent (**Claude Code**, **Codex**, or **Google Antigravity /
+Gemini**) in a sandbox against declarative YAML tasks, then scores the files and
+commands it actually produced. Not an "agentic coding" benchmark: it measures how
+effective your CLI and skills are when used by coding agents.
+
+Reach for it when you want to **test whether a Claude Code skill triggers**,
+**A/B-test Claude Code vs. Codex vs. Gemini** (or model vs. model, prompt vs.
+prompt), or **gate CI on coding-agent quality**. Unlike fixed datasets (SWE-bench,
+SkillsBench) that rank models on a shared leaderboard, Coder Eval evaluates the
+tasks, skills, and workflows *you* ship — with weighted 0.0–1.0 criteria, a
+`skill_triggered` activation check, an A/B experiment layer, and per-tool cost
+telemetry. See [How it compares](https://uipath.github.io/coder_eval/comparison/).
+📚 **Full docs:** **[uipath.github.io/coder_eval](https://uipath.github.io/coder_eval/)**.
 
 <p align="center">
-  <img src="docs/assets/hero.gif" alt="coder_eval running the hello_date task: a sandboxed agent writes and runs a script from a YAML task, then the scored result is browsed in evalboard" width="100%">
+  <img src="docs/assets/hero.gif" alt="Coder Eval running the hello_date task: a sandboxed agent writes and runs a script from a YAML task, then the scored result is browsed in evalboard" width="100%">
 </p>
 
-> **The Coding Agents Gym.** A sandboxed, reproducible framework to evaluate,
-> benchmark, and A/B-test AI coding agents — Claude Code, Codex, and Google
-> Antigravity (Gemini) today, any agent via a plugin SPI — with declarative
-> YAML tasks and weighted scoring.
-
 - **Declarative YAML tasks** with pinned dependencies and clear success criteria
 - **Sandboxed execution** in isolated environments with resource limits
 - **Weighted, continuous scoring** (0.0–1.0) with fractional credit and thresholds
@@ -92,14 +100,14 @@ are when used by coding agents.
 
 ## What you can do with it
 
-- **Benchmark coding agents** — score an agent across a suite of tasks with weighted, pass/fail thresholds
+- **Benchmark coding agents** — score an agent across a suite of tasks with weighted scoring and pass/fail thresholds
 - **Compare models & configs** — A/B-test Claude vs. Codex vs. Gemini, model vs. model, tool-on vs. tool-off, prompt vs. prompt
 - **Evaluate skills** — verify an agent actually engages a target skill (`skill_triggered`) and score skill-driven suites (SkillsBench-style)
 - **Keep skills up to date in CI** — re-validate your skills on every change or on a schedule; catch silent regressions when models, prompts, or the skills themselves drift
 - **Gate CI on agent quality** — run the suite in GitHub Actions and fail the build on regressions
 - **Bring your own dataset** — fan one task out over many rows for larger benchmark suites
 
-> **Keeping skills fresh?** Run coder_eval as a scheduled GitHub Actions job so your
+> **Keeping skills fresh?** Run Coder Eval as a scheduled GitHub Actions job so your
 > skills are continuously re-evaluated against the latest model — a skill that quietly
 > stops triggering surfaces as a failing criterion before your users hit it. See
 > **[Tutorial 02 — Running coder_eval in CI](docs/tutorials/02-ci-pipeline.md)**.
@@ -115,7 +123,9 @@ git clone https://github.com/UiPath/coder_eval.git
 cd coder_eval
 
 uv sync --extra dev          # install core + dev tools
-cp .env.example .env         # then set ANTHROPIC_API_KEY
+cp .env.example .env         # then set ANTHROPIC_API_KEY — or skip that: an
+                             # existing Claude Code login (`claude login`) is
+                             # picked up automatically
 
 uv run coder-eval plan tasks/hello_date.yaml   # validate (no tokens spent)
 uv run coder-eval run  tasks/hello_date.yaml   # run your first evaluation
@@ -129,11 +139,23 @@ The optional `[uipath]` extra (`uv sync --extra dev --extra uipath`) adds the in
 required). Without it the framework runs end-to-end; uipath-dependent features fail
 at dispatch with a clear hint.
 
-> **Using coder_eval in CI or another project?** Install the published package:
-> `pip install coder-eval` (or `uv add coder-eval`; extras install the same way —
-> `pip install "coder-eval[codex,antigravity]"`). In a real CI gate, pin to a
-> specific released version so a harness upgrade can't silently move your results.
-> See [Tutorial 02 — Running coder_eval in CI](docs/tutorials/02-ci-pipeline.md) for the full setup.
+**Using Coder Eval in CI or another project?** Install the published package
+instead of cloning:
+
+```bash
+uv tool install coder-eval    # puts the `coder-eval` CLI on your PATH,
+                              # in its own isolated environment
+
+uv tool install "coder-eval[codex,antigravity]"   # same, with agent extras
+coder-eval --version                              # verify the install
+```
+
+To add it as a project dependency instead: `uv add coder-eval` or
+`pip install coder-eval`. In a real CI gate, pin to a specific released version
+so a harness upgrade can't silently move your results. (The example `tasks/`
+live in this repo — clone it or point the CLI at your own task files.) See
+[Tutorial 02 — Running coder_eval in CI](docs/tutorials/02-ci-pipeline.md) for
+the full setup.
 
 ## Telemetry
 
@@ -161,17 +183,18 @@ at dispatch with a clear hint.
 
 ## How it compares
 
-- **vs. SWE-bench and fixed benchmarks** — SWE-bench is a fixed dataset of GitHub
-  issues; coder_eval is a *framework* for authoring your own tasks in declarative
-  YAML, so you evaluate the skills and workflows you care about (and can still wrap
-  a fixed dataset via [Bring Your Own Dataset](docs/BYOD.md)).
-- **vs. LLM-output eval harnesses (e.g. OpenAI Evals)** — those grade a model's text;
-  coder_eval runs a full **agent** in a **sandbox** with real tool use and multi-turn
-  dialog, then scores the files and commands it actually produced (continuous
-  0.0–1.0) — not just a judge over a string.
+- **vs. fixed benchmarks (SWE-bench, SkillsBench)** — they score a canonical dataset;
+  Coder Eval scores *your* tasks with continuous 0.0–1.0 weighted criteria (and can
+  still wrap a fixed dataset via [Bring Your Own Dataset](docs/BYOD.md)).
+- **vs. large-scale / RL harnesses (Harbor)** — Harbor targets scale and RL rollouts;
+  Coder Eval targets weighted, skill-aware suites gated in CI.
+- **vs. model-output eval tools (OpenAI Evals)** — they grade model text; Coder Eval
+  runs a full agent in a sandbox and scores the files and commands it produced.
 - **vs. hand-rolled scripts** — reproducible sandboxes, weighted criteria,
   cost/token telemetry, A/B experiments, and CI-ready pass/fail gates out of the box.
 
+See the full [comparison — with sources](https://uipath.github.io/coder_eval/comparison/).
+
 ## Task Definition
 
 A task is a YAML file: a prompt, the agent config, a sandbox, and success criteria.
@@ -221,12 +244,12 @@ extension points (new criteria, new agents).
 
 ## Known limits & non-goals
 
-- **Not a fixed benchmark or leaderboard** — coder_eval scores *your* tasks and ships
+- **Not a fixed benchmark or leaderboard** — Coder Eval scores *your* tasks and ships
   example tasks, not a canonical scored dataset.
 - **Tasks execute real code** — run untrusted tasks only under the container driver
   (see [Docker Isolation](docs/DOCKER_ISOLATION.md)); the `tempdir` driver is not a
   security boundary.
-- **Bring your own model credentials** — Anthropic, Bedrock, or Gemini keys; coder_eval
+- **Bring your own model credentials** — Anthropic, Bedrock, or Gemini keys; Coder Eval
   does not proxy or supply model access.
 - **Python 3.13+ only.**