evalcore 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. evalcore-0.1.0/.github/workflows/ci.yml +35 -0
  2. evalcore-0.1.0/.github/workflows/publish.yml +31 -0
  3. evalcore-0.1.0/.gitignore +24 -0
  4. evalcore-0.1.0/.pre-commit-config.yaml +20 -0
  5. evalcore-0.1.0/LICENSE +28 -0
  6. evalcore-0.1.0/PKG-INFO +828 -0
  7. evalcore-0.1.0/README.md +803 -0
  8. evalcore-0.1.0/docs/design.md +157 -0
  9. evalcore-0.1.0/examples/__init__.py +0 -0
  10. evalcore-0.1.0/examples/quickstart/README.md +61 -0
  11. evalcore-0.1.0/examples/quickstart/__init__.py +0 -0
  12. evalcore-0.1.0/examples/quickstart/adapter.py +75 -0
  13. evalcore-0.1.0/examples/quickstart/datasets/support_reply/v1/cases/angry_complaint.yaml +7 -0
  14. evalcore-0.1.0/examples/quickstart/datasets/support_reply/v1/cases/billing_question.yaml +7 -0
  15. evalcore-0.1.0/examples/quickstart/datasets/support_reply/v1/cases/double_charge.yaml +7 -0
  16. evalcore-0.1.0/examples/quickstart/datasets/support_reply/v1/cases/refund_request.yaml +7 -0
  17. evalcore-0.1.0/examples/quickstart/fixtures/support_reply_judge.yaml +23 -0
  18. evalcore-0.1.0/examples/quickstart/fixtures/support_reply_pairwise.yaml +20 -0
  19. evalcore-0.1.0/examples/quickstart/fixtures/support_reply_replay.yaml +38 -0
  20. evalcore-0.1.0/examples/quickstart/graders.py +93 -0
  21. evalcore-0.1.0/examples/quickstart/run_eval.py +112 -0
  22. evalcore-0.1.0/examples/quickstart/suite.yaml +102 -0
  23. evalcore-0.1.0/examples/quickstart/tests/__init__.py +0 -0
  24. evalcore-0.1.0/examples/quickstart/tests/test_quickstart.py +100 -0
  25. evalcore-0.1.0/justfile +34 -0
  26. evalcore-0.1.0/pyproject.toml +110 -0
  27. evalcore-0.1.0/pyrightconfig.json +4 -0
  28. evalcore-0.1.0/src/evalkit/__init__.py +39 -0
  29. evalcore-0.1.0/src/evalkit/adapters/__init__.py +15 -0
  30. evalcore-0.1.0/src/evalkit/adapters/_env.py +24 -0
  31. evalcore-0.1.0/src/evalkit/adapters/base.py +40 -0
  32. evalcore-0.1.0/src/evalkit/adapters/http.py +96 -0
  33. evalcore-0.1.0/src/evalkit/adapters/replay.py +41 -0
  34. evalcore-0.1.0/src/evalkit/cli.py +655 -0
  35. evalcore-0.1.0/src/evalkit/compare.py +137 -0
  36. evalcore-0.1.0/src/evalkit/graders/__init__.py +18 -0
  37. evalcore-0.1.0/src/evalkit/graders/base.py +77 -0
  38. evalcore-0.1.0/src/evalkit/graders/classification.py +107 -0
  39. evalcore-0.1.0/src/evalkit/graders/deterministic.py +127 -0
  40. evalcore-0.1.0/src/evalkit/graders/judge.py +635 -0
  41. evalcore-0.1.0/src/evalkit/graders/numeric.py +91 -0
  42. evalcore-0.1.0/src/evalkit/loader.py +129 -0
  43. evalcore-0.1.0/src/evalkit/models.py +390 -0
  44. evalcore-0.1.0/src/evalkit/pairwise.py +324 -0
  45. evalcore-0.1.0/src/evalkit/py.typed +0 -0
  46. evalcore-0.1.0/src/evalkit/rating.py +1323 -0
  47. evalcore-0.1.0/src/evalkit/refs.py +71 -0
  48. evalcore-0.1.0/src/evalkit/report.py +186 -0
  49. evalcore-0.1.0/src/evalkit/reporters/__init__.py +33 -0
  50. evalcore-0.1.0/src/evalkit/reporters/base.py +159 -0
  51. evalcore-0.1.0/src/evalkit/reporters/html.py +426 -0
  52. evalcore-0.1.0/src/evalkit/reporters/markdown.py +74 -0
  53. evalcore-0.1.0/src/evalkit/retry.py +85 -0
  54. evalcore-0.1.0/src/evalkit/runner.py +283 -0
  55. evalcore-0.1.0/src/evalkit/store.py +286 -0
  56. evalcore-0.1.0/src/evalkit/sweep.py +93 -0
  57. evalcore-0.1.0/tests/__init__.py +0 -0
  58. evalcore-0.1.0/tests/test_adapters.py +199 -0
  59. evalcore-0.1.0/tests/test_cli.py +701 -0
  60. evalcore-0.1.0/tests/test_edge_cases.py +230 -0
  61. evalcore-0.1.0/tests/test_judge.py +347 -0
  62. evalcore-0.1.0/tests/test_judge_extra.py +106 -0
  63. evalcore-0.1.0/tests/test_live_clients.py +173 -0
  64. evalcore-0.1.0/tests/test_pairwise_extra.py +68 -0
  65. evalcore-0.1.0/tests/test_rating.py +317 -0
  66. evalcore-0.1.0/tests/test_rating_server.py +422 -0
  67. evalcore-0.1.0/tests/test_reporters.py +465 -0
  68. evalcore-0.1.0/tests/test_retry.py +115 -0
  69. evalcore-0.1.0/tests/test_runner.py +335 -0
  70. evalcore-0.1.0/tests/test_store.py +173 -0
  71. evalcore-0.1.0/tests/test_sweep_pairwise.py +126 -0
  72. evalcore-0.1.0/tests/test_unit.py +239 -0
  73. evalcore-0.1.0/uv.lock +510 -0
  74. evalcore-0.1.0/uv.toml +6 -0
@@ -0,0 +1,35 @@
1
+ name: CI
2
+
3
+ on:
4
+ push:
5
+ branches: [main]
6
+ pull_request:
7
+ branches: [main]
8
+ workflow_dispatch: {}
9
+
10
+ jobs:
11
+ test:
12
+ runs-on: ubuntu-latest
13
+ steps:
14
+ - uses: actions/checkout@v4
15
+
16
+ - name: Install uv
17
+ uses: astral-sh/setup-uv@v5
18
+ with:
19
+ python-version: "3.14"
20
+
21
+ - name: Sync dependencies
22
+ run: uv sync --all-extras
23
+
24
+ - name: Lint (ruff format + check)
25
+ run: |
26
+ uv run ruff format --check
27
+ uv run ruff check
28
+
29
+ - name: Tests + coverage (gate at 90%)
30
+ run: |
31
+ uv run coverage run
32
+ uv run coverage report
33
+
34
+ - name: Build wheel + sdist
35
+ run: uv build
@@ -0,0 +1,31 @@
1
+ name: Publish to PyPI
2
+
3
+ # Publishes to PyPI via Trusted Publishing (OIDC) when a GitHub Release is
4
+ # published - no API tokens stored anywhere. Configure a matching "pending
5
+ # publisher" on PyPI first (see README): project evalcore, this repo, this
6
+ # workflow file (publish.yml), environment "pypi".
7
+
8
+ on:
9
+ release:
10
+ types: [published]
11
+ workflow_dispatch: {}
12
+
13
+ jobs:
14
+ publish:
15
+ runs-on: ubuntu-latest
16
+ environment: pypi
17
+ permissions:
18
+ id-token: write # required for Trusted Publishing (OIDC)
19
+ steps:
20
+ - uses: actions/checkout@v4
21
+
22
+ - name: Install uv
23
+ uses: astral-sh/setup-uv@v5
24
+ with:
25
+ python-version: "3.14"
26
+
27
+ - name: Build sdist + wheel
28
+ run: uv build
29
+
30
+ - name: Publish to PyPI
31
+ uses: pypa/gh-action-pypi-publish@release/v1
@@ -0,0 +1,24 @@
1
+ .*
2
+ !/.gitignore
3
+ !/.github
4
+ !/.pre-commit-config.yaml
5
+
6
+ /build
7
+ /dist
8
+ *.egg-info/
9
+ **/__pycache__/*
10
+
11
+ # evalkit run artifacts
12
+ *.scorecard.json
13
+ *.run.json
14
+ *.run.ckpt
15
+ outbox.jsonl
16
+ scores.jsonl
17
+ ratings.jsonl
18
+ preferences.jsonl
19
+ examples/*/results/
20
+ examples/*/results-*/
21
+
22
+ # ad-hoc report / screenshot artifacts from manual runs at the repo root
23
+ /*.html
24
+ /*.png
@@ -0,0 +1,20 @@
1
+ repos:
2
+ - repo: https://github.com/pre-commit/pre-commit-hooks
3
+ rev: v6.0.0
4
+ hooks:
5
+ - id: check-ast
6
+ - id: check-toml
7
+ - id: check-yaml
8
+ args: [--allow-multiple-documents]
9
+ - id: debug-statements
10
+ - id: end-of-file-fixer
11
+ - id: check-merge-conflict
12
+ - id: mixed-line-ending
13
+ - id: trailing-whitespace
14
+
15
+ - repo: https://github.com/astral-sh/ruff-pre-commit
16
+ rev: v0.15.11
17
+ hooks:
18
+ - id: ruff-format
19
+ - id: ruff
20
+ args: [--fix]
evalcore-0.1.0/LICENSE ADDED
@@ -0,0 +1,28 @@
1
+ BSD 3-Clause License
2
+
3
+ Copyright (c) 2026 AWeber Communications
4
+
5
+ Redistribution and use in source and binary forms, with or without
6
+ modification, are permitted provided that the following conditions are met:
7
+
8
+ 1. Redistributions of source code must retain the above copyright notice, this
9
+ list of conditions and the following disclaimer.
10
+
11
+ 2. Redistributions in binary form must reproduce the above copyright notice,
12
+ this list of conditions and the following disclaimer in the documentation
13
+ and/or other materials provided with the distribution.
14
+
15
+ 3. Neither the name of the copyright holder nor the names of its contributors
16
+ may be used to endorse or promote products derived from this software
17
+ without specific prior written permission.
18
+
19
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20
+ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21
+ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
22
+ DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
23
+ FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24
+ DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
25
+ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
26
+ CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
27
+ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.