benchrail 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. benchrail-0.1.0/.github/CODEOWNERS +1 -0
  2. benchrail-0.1.0/.github/workflows/ci.yml +104 -0
  3. benchrail-0.1.0/.github/workflows/publish.yml +80 -0
  4. benchrail-0.1.0/.github/workflows/release.yml +103 -0
  5. benchrail-0.1.0/.gitignore +109 -0
  6. benchrail-0.1.0/AGENTS.md +398 -0
  7. benchrail-0.1.0/CODEOWNERS +1 -0
  8. benchrail-0.1.0/LICENSES/LICENSE +21 -0
  9. benchrail-0.1.0/LICENSES/THIRD_PARTY.md +31 -0
  10. benchrail-0.1.0/Makefile +33 -0
  11. benchrail-0.1.0/PKG-INFO +395 -0
  12. benchrail-0.1.0/README.md +368 -0
  13. benchrail-0.1.0/benchrail/__init__.py +3 -0
  14. benchrail-0.1.0/benchrail/adapters/__init__.py +3 -0
  15. benchrail-0.1.0/benchrail/adapters/base.py +47 -0
  16. benchrail-0.1.0/benchrail/adapters/claude_code.py +95 -0
  17. benchrail-0.1.0/benchrail/adapters/codex.py +138 -0
  18. benchrail-0.1.0/benchrail/cli.py +22 -0
  19. benchrail-0.1.0/benchrail/commands/__init__.py +0 -0
  20. benchrail-0.1.0/benchrail/commands/run.py +97 -0
  21. benchrail-0.1.0/benchrail/commands/validate.py +104 -0
  22. benchrail-0.1.0/benchrail/commands/version.py +10 -0
  23. benchrail-0.1.0/benchrail/dto/__init__.py +16 -0
  24. benchrail-0.1.0/benchrail/dto/config.py +163 -0
  25. benchrail-0.1.0/benchrail/dto/manifest.py +34 -0
  26. benchrail-0.1.0/benchrail/dto/result.py +140 -0
  27. benchrail-0.1.0/benchrail/pricing.py +118 -0
  28. benchrail-0.1.0/benchrail/registry.py +40 -0
  29. benchrail-0.1.0/benchrail/runner/__init__.py +0 -0
  30. benchrail-0.1.0/benchrail/runner/docker.py +472 -0
  31. benchrail-0.1.0/benchrail/runner/local.py +277 -0
  32. benchrail-0.1.0/benchrail/runner/logging_util.py +264 -0
  33. benchrail-0.1.0/benchrail/runner/orchestrator.py +445 -0
  34. benchrail-0.1.0/benchrail/runner/worker.py +1150 -0
  35. benchrail-0.1.0/docker/universal/Dockerfile +364 -0
  36. benchrail-0.1.0/docker/universal/README.md +159 -0
  37. benchrail-0.1.0/docker/universal/entrypoint.sh +16 -0
  38. benchrail-0.1.0/docker/universal/setup_agents.sh +48 -0
  39. benchrail-0.1.0/docker/universal/setup_universal.sh +91 -0
  40. benchrail-0.1.0/docker/universal/verify.sh +116 -0
  41. benchrail-0.1.0/examples/multi-swe-bench-universal-smoke/README.md +48 -0
  42. benchrail-0.1.0/examples/multi-swe-bench-universal-smoke/caddyserver__caddy-6411/config.json +35 -0
  43. benchrail-0.1.0/examples/multi-swe-bench-universal-smoke/caddyserver__caddy-6411/environment/run-gold-tests.sh +4 -0
  44. benchrail-0.1.0/examples/multi-swe-bench-universal-smoke/caddyserver__caddy-6411/environment/run-selected-tests.sh +4 -0
  45. benchrail-0.1.0/examples/multi-swe-bench-universal-smoke/caddyserver__caddy-6411/environment/setup.sh +4 -0
  46. benchrail-0.1.0/examples/multi-swe-bench-universal-smoke/caddyserver__caddy-6411/patches/original_swe_test.patch +48 -0
  47. benchrail-0.1.0/examples/multi-swe-bench-universal-smoke/caddyserver__caddy-6411/patches/test.patch +62 -0
  48. benchrail-0.1.0/examples/multi-swe-bench-universal-smoke/fmtlib__fmt-3901/config.json +33 -0
  49. benchrail-0.1.0/examples/multi-swe-bench-universal-smoke/fmtlib__fmt-3901/environment/run-gold-tests.sh +6 -0
  50. benchrail-0.1.0/examples/multi-swe-bench-universal-smoke/fmtlib__fmt-3901/environment/run-selected-tests.sh +4 -0
  51. benchrail-0.1.0/examples/multi-swe-bench-universal-smoke/fmtlib__fmt-3901/environment/setup.sh +5 -0
  52. benchrail-0.1.0/examples/multi-swe-bench-universal-smoke/fmtlib__fmt-3901/patches/original_swe_test.patch +14 -0
  53. benchrail-0.1.0/examples/multi-swe-bench-universal-smoke/fmtlib__fmt-3901/patches/test.patch +28 -0
  54. benchrail-0.1.0/examples/multi-swe-bench-universal-smoke/manifest.json +10 -0
  55. benchrail-0.1.0/examples/multi-swe-bench-universal-smoke/pydicom__pydicom-1694/config.json +35 -0
  56. benchrail-0.1.0/examples/multi-swe-bench-universal-smoke/pydicom__pydicom-1694/environment/run-gold-tests.sh +5 -0
  57. benchrail-0.1.0/examples/multi-swe-bench-universal-smoke/pydicom__pydicom-1694/environment/run-selected-tests.sh +7 -0
  58. benchrail-0.1.0/examples/multi-swe-bench-universal-smoke/pydicom__pydicom-1694/environment/setup.sh +13 -0
  59. benchrail-0.1.0/examples/multi-swe-bench-universal-smoke/pydicom__pydicom-1694/patches/original_swe_test.patch +37 -0
  60. benchrail-0.1.0/examples/multi-swe-bench-universal-smoke/pydicom__pydicom-1694/patches/test.patch +25 -0
  61. benchrail-0.1.0/examples/multi-swe-bench-universal-smoke/tokio-rs__tokio-7139/config.json +35 -0
  62. benchrail-0.1.0/examples/multi-swe-bench-universal-smoke/tokio-rs__tokio-7139/environment/run-gold-tests.sh +4 -0
  63. benchrail-0.1.0/examples/multi-swe-bench-universal-smoke/tokio-rs__tokio-7139/environment/run-selected-tests.sh +4 -0
  64. benchrail-0.1.0/examples/multi-swe-bench-universal-smoke/tokio-rs__tokio-7139/environment/setup.sh +4 -0
  65. benchrail-0.1.0/examples/multi-swe-bench-universal-smoke/tokio-rs__tokio-7139/patches/original_swe_test.patch +37 -0
  66. benchrail-0.1.0/examples/multi-swe-bench-universal-smoke/tokio-rs__tokio-7139/patches/test.patch +36 -0
  67. benchrail-0.1.0/examples/multi-swe-bench-universal-smoke/vuejs__core-11915/config.json +35 -0
  68. benchrail-0.1.0/examples/multi-swe-bench-universal-smoke/vuejs__core-11915/environment/run-gold-tests.sh +5 -0
  69. benchrail-0.1.0/examples/multi-swe-bench-universal-smoke/vuejs__core-11915/environment/run-selected-tests.sh +5 -0
  70. benchrail-0.1.0/examples/multi-swe-bench-universal-smoke/vuejs__core-11915/environment/setup.sh +7 -0
  71. benchrail-0.1.0/examples/multi-swe-bench-universal-smoke/vuejs__core-11915/patches/original_swe_test.patch +26 -0
  72. benchrail-0.1.0/examples/multi-swe-bench-universal-smoke/vuejs__core-11915/patches/test.patch +29 -0
  73. benchrail-0.1.0/pyproject.toml +86 -0
  74. benchrail-0.1.0/tests/__init__.py +0 -0
  75. benchrail-0.1.0/tests/integration/__init__.py +0 -0
  76. benchrail-0.1.0/tests/unit/__init__.py +0 -0
  77. benchrail-0.1.0/tests/unit/test_adapters.py +188 -0
  78. benchrail-0.1.0/tests/unit/test_cli.py +15 -0
  79. benchrail-0.1.0/tests/unit/test_docker.py +253 -0
  80. benchrail-0.1.0/tests/unit/test_dto.py +291 -0
  81. benchrail-0.1.0/tests/unit/test_local.py +128 -0
  82. benchrail-0.1.0/tests/unit/test_pricing.py +122 -0
  83. benchrail-0.1.0/tests/unit/test_worker.py +381 -0
  84. benchrail-0.1.0/uv.lock +1260 -0
@@ -0,0 +1 @@
1
+ * @tripcher
@@ -0,0 +1,104 @@
1
+ name: CI
2
+
3
+ on:
4
+ push:
5
+ branches: [main]
6
+ pull_request:
7
+
8
+ jobs:
9
+ lint:
10
+ runs-on: ubuntu-latest
11
+ steps:
12
+ - name: Checkout
13
+ uses: actions/checkout@v4
14
+
15
+ - name: Install uv
16
+ uses: astral-sh/setup-uv@v6
17
+ with:
18
+ version: "latest"
19
+
20
+ - name: Set up Python
21
+ run: uv python install 3.10
22
+
23
+ - name: Install dependencies
24
+ run: uv sync --dev
25
+
26
+ - name: Lint
27
+ run: uv run ruff check benchrail/ tests/
28
+
29
+ - name: Check formatting
30
+ run: uv run ruff format --check benchrail/ tests/
31
+
32
+ type-check:
33
+ runs-on: ubuntu-latest
34
+ steps:
35
+ - name: Checkout
36
+ uses: actions/checkout@v4
37
+
38
+ - name: Install uv
39
+ uses: astral-sh/setup-uv@v6
40
+ with:
41
+ version: "latest"
42
+
43
+ - name: Set up Python
44
+ run: uv python install 3.10
45
+
46
+ - name: Install dependencies
47
+ run: uv sync --dev
48
+
49
+ - name: Type check
50
+ run: uv run mypy benchrail tests
51
+
52
+ test:
53
+ runs-on: ubuntu-latest
54
+ strategy:
55
+ fail-fast: false
56
+ matrix:
57
+ python-version: ["3.10", "3.11", "3.12", "3.13"]
58
+ steps:
59
+ - name: Checkout
60
+ uses: actions/checkout@v4
61
+
62
+ - name: Install uv
63
+ uses: astral-sh/setup-uv@v6
64
+ with:
65
+ version: "latest"
66
+
67
+ - name: Set up Python ${{ matrix.python-version }}
68
+ run: uv python install ${{ matrix.python-version }}
69
+
70
+ - name: Install dependencies
71
+ run: uv sync --dev --python ${{ matrix.python-version }}
72
+
73
+ - name: Run unit tests
74
+ run: uv run --python ${{ matrix.python-version }} pytest tests/unit/ -v
75
+
76
+ build:
77
+ runs-on: ubuntu-latest
78
+ needs: [lint, type-check, test]
79
+ steps:
80
+ - name: Checkout
81
+ uses: actions/checkout@v4
82
+
83
+ - name: Install uv
84
+ uses: astral-sh/setup-uv@v6
85
+ with:
86
+ version: "latest"
87
+
88
+ - name: Set up Python
89
+ run: uv python install 3.10
90
+
91
+ - name: Install dependencies
92
+ run: uv sync --dev
93
+
94
+ - name: Build package
95
+ run: uv run python -m build
96
+
97
+ - name: Check distribution metadata
98
+ run: uv run twine check dist/*
99
+
100
+ - name: Upload artifacts
101
+ uses: actions/upload-artifact@v4
102
+ with:
103
+ name: dist
104
+ path: dist/
@@ -0,0 +1,80 @@
1
+ name: Publish to PyPI
2
+
3
+ on:
4
+ release:
5
+ types: [published]
6
+
7
+ permissions:
8
+ id-token: write
9
+ contents: read
10
+ packages: write
11
+
12
+ jobs:
13
+ publish-pypi:
14
+ runs-on: ubuntu-latest
15
+ environment: pypi
16
+ steps:
17
+ - name: Checkout
18
+ uses: actions/checkout@v4
19
+
20
+ - name: Install uv
21
+ uses: astral-sh/setup-uv@v6
22
+ with:
23
+ version: "latest"
24
+
25
+ - name: Set up Python
26
+ run: uv python install 3.10
27
+
28
+ - name: Install dependencies
29
+ run: uv sync --dev
30
+
31
+ - name: Build package
32
+ run: uv run python -m build
33
+
34
+ - name: Check distribution metadata
35
+ run: uv run twine check dist/*
36
+
37
+ - name: Upload build artifacts
38
+ uses: actions/upload-artifact@v4
39
+ with:
40
+ name: dist
41
+ path: dist/
42
+
43
+ - name: Publish to PyPI
44
+ uses: pypa/gh-action-pypi-publish@release/v1
45
+
46
+ publish-docker:
47
+ needs: publish-pypi
48
+ runs-on: ubuntu-latest
49
+ steps:
50
+ - name: Checkout
51
+ uses: actions/checkout@v4
52
+
53
+ - name: Log in to GHCR
54
+ uses: docker/login-action@v3
55
+ with:
56
+ registry: ghcr.io
57
+ username: ${{ github.actor }}
58
+ password: ${{ secrets.GITHUB_TOKEN }}
59
+
60
+ - name: Extract Docker metadata
61
+ id: meta
62
+ uses: docker/metadata-action@v5
63
+ with:
64
+ images: ghcr.io/${{ github.repository_owner }}/benchrail-universal
65
+ tags: |
66
+ type=raw,value=latest
67
+ type=ref,event=tag
68
+
69
+ - name: Set up Docker Buildx
70
+ uses: docker/setup-buildx-action@v3
71
+
72
+ - name: Build and push image
73
+ uses: docker/build-push-action@v6
74
+ with:
75
+ context: .
76
+ file: docker/universal/Dockerfile
77
+ platforms: linux/amd64,linux/arm64
78
+ push: true
79
+ tags: ${{ steps.meta.outputs.tags }}
80
+ labels: ${{ steps.meta.outputs.labels }}
@@ -0,0 +1,103 @@
1
+ name: Create Release
2
+
3
+ on:
4
+ workflow_dispatch:
5
+ inputs:
6
+ bump:
7
+ description: "Semantic version bump type"
8
+ required: true
9
+ default: patch
10
+ type: choice
11
+ options:
12
+ - patch
13
+ - minor
14
+ - major
15
+
16
+ permissions:
17
+ contents: write
18
+
19
+ concurrency:
20
+ group: release
21
+ cancel-in-progress: false
22
+
23
+ jobs:
24
+ release:
25
+ runs-on: ubuntu-latest
26
+ steps:
27
+ - name: Require main branch
28
+ if: github.ref_name != 'main'
29
+ run: |
30
+ echo "Run this workflow from the main branch."
31
+ exit 1
32
+
33
+ - name: Checkout
34
+ uses: actions/checkout@v4
35
+ with:
36
+ fetch-depth: 0
37
+
38
+ - name: Set up Python
39
+ uses: actions/setup-python@v5
40
+ with:
41
+ python-version: "3.10"
42
+
43
+ - name: Bump version
44
+ id: bump
45
+ run: |
46
+ python - <<'PY'
47
+ import re
48
+ from pathlib import Path
49
+
50
+ bump = "${{ inputs.bump }}"
51
+ path = Path("benchrail/__init__.py")
52
+ text = path.read_text()
53
+ match = re.search(r'__version__ = "(\d+)\.(\d+)\.(\d+)"', text)
54
+ if match is None:
55
+ raise SystemExit("Unable to locate __version__ in benchrail/__init__.py")
56
+
57
+ major, minor, patch = (int(part) for part in match.groups())
58
+ if bump == "major":
59
+ major, minor, patch = major + 1, 0, 0
60
+ elif bump == "minor":
61
+ minor, patch = minor + 1, 0
62
+ elif bump == "patch":
63
+ patch += 1
64
+ else:
65
+ raise SystemExit(f"Unsupported bump type: {bump}")
66
+
67
+ version = f"{major}.{minor}.{patch}"
68
+ path.write_text(
69
+ re.sub(
70
+ r'__version__ = "\d+\.\d+\.\d+"',
71
+ f'__version__ = "{version}"',
72
+ text,
73
+ )
74
+ )
75
+ Path("/tmp/benchrail_version").write_text(version)
76
+ PY
77
+
78
+ version="$(cat /tmp/benchrail_version)"
79
+ echo "version=$version" >> "$GITHUB_OUTPUT"
80
+ echo "tag=v$version" >> "$GITHUB_OUTPUT"
81
+
82
+ - name: Commit version bump
83
+ run: |
84
+ git config user.name "github-actions[bot]"
85
+ git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
86
+ git add benchrail/__init__.py pyproject.toml
87
+ git commit -m "Release ${{ steps.bump.outputs.tag }}"
88
+
89
+ - name: Create git tag
90
+ run: git tag "${{ steps.bump.outputs.tag }}"
91
+
92
+ - name: Push commit and tag
93
+ run: |
94
+ git push origin HEAD:main
95
+ git push origin "${{ steps.bump.outputs.tag }}"
96
+
97
+ - name: Create GitHub release
98
+ env:
99
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
100
+ run: |
101
+ gh release create "${{ steps.bump.outputs.tag }}" \
102
+ --title "${{ steps.bump.outputs.tag }}" \
103
+ --notes "Release ${{ steps.bump.outputs.tag }}"
@@ -0,0 +1,109 @@
1
+ # Byte-compiled / optimized / DLL files
2
+ __pycache__/
3
+ *.py[codz]
4
+ *$py.class
5
+
6
+ # C extensions
7
+ *.so
8
+
9
+ # Distribution / packaging
10
+ .Python
11
+ build/
12
+ develop-eggs/
13
+ dist/
14
+ downloads/
15
+ eggs/
16
+ .eggs/
17
+ lib/
18
+ lib64/
19
+ parts/
20
+ sdist/
21
+ var/
22
+ wheels/
23
+ share/python-wheels/
24
+ *.egg-info/
25
+ .installed.cfg
26
+ *.egg
27
+ MANIFEST
28
+
29
+ # PyInstaller
30
+ # Usually these files are written by a python script from a template
31
+ # before PyInstaller builds the exe, so as to inject date/other infos into it.
32
+ *.manifest
33
+ *.spec
34
+
35
+ # Installer logs
36
+ pip-log.txt
37
+ pip-delete-this-directory.txt
38
+
39
+ # Unit test / coverage reports
40
+ htmlcov/
41
+ .tox/
42
+ .nox/
43
+ .coverage
44
+ .coverage.*
45
+ .cache
46
+ nosetests.xml
47
+ coverage.xml
48
+ *.cover
49
+ *.py.cover
50
+ *.lcov
51
+ .hypothesis/
52
+ .pytest_cache/
53
+ cover/
54
+
55
+ # PyBuilder
56
+ .pybuilder/
57
+ target/
58
+
59
+ # UV
60
+ .uv-cache
61
+
62
+ # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
63
+ __pypackages__/
64
+
65
+ # Environments
66
+ .env
67
+ .envrc
68
+ .venv
69
+ env/
70
+ venv/
71
+ ENV/
72
+ env.bak/
73
+ venv.bak/
74
+
75
+ # mypy
76
+ .mypy_cache/
77
+ .dmypy.json
78
+ dmypy.json
79
+
80
+ # Pyre type checker
81
+ .pyre/
82
+
83
+ # pytype static type analyzer
84
+ .pytype/
85
+
86
+ # Cython debug symbols
87
+ cython_debug/
88
+
89
+ # PyCharm
90
+ .idea/
91
+ .DS_Store
92
+
93
+ # Visual Studio Code
94
+ # Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore
95
+ # that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
96
+ # and can be added to the global gitignore or merged into this file. However, if you prefer,
97
+ # you could uncomment the following to ignore the entire vscode folder
98
+ # .vscode/
99
+ # Temporary file for partial code execution
100
+ tempCodeRunnerFile.py
101
+
102
+ # Ruff stuff:
103
+ .ruff_cache/
104
+
105
+ # PyPI configuration file
106
+ .pypirc
107
+
108
+ # Tmp
109
+ tmp/