pace-eval 0.2.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cme/__init__.py +11 -0
- cme/__main__.py +5 -0
- cme/analysis.py +223 -0
- cme/audit.py +145 -0
- cme/baseline.py +162 -0
- cme/cli.py +279 -0
- cme/client.py +165 -0
- cme/dynamics.py +212 -0
- cme/examples/crewai_integration.py +16 -0
- cme/examples/demo_pool.jsonl +360 -0
- cme/examples/demo_run.jsonl +480 -0
- cme/examples/langgraph_integration.py +27 -0
- cme/examples/make_demo.py +37 -0
- cme/examples/openai_quickstart.py +18 -0
- cme/experiment.py +555 -0
- cme/frontier.py +143 -0
- cme/hidden_profile.py +135 -0
- cme/proxy.py +106 -0
- cme/rebuttal.py +76 -0
- cme/report.py +197 -0
- cme/rollout_card.py +137 -0
- cme/tasks.py +266 -0
- cme/verified_debate.py +127 -0
- cme/verifier.py +92 -0
- mae_eval/__init__.py +11 -0
- mae_eval/__main__.py +5 -0
- mae_eval/auditor.py +112 -0
- mae_eval/classifier.py +62 -0
- mae_eval/cli.py +210 -0
- mae_eval/tracker.py +162 -0
- mae_eval/vector.py +91 -0
- pace/__init__.py +11 -0
- pace/__main__.py +5 -0
- pace/analysis.py +129 -0
- pace/audit.py +145 -0
- pace/baseline.py +162 -0
- pace/card.py +205 -0
- pace/cli.py +898 -0
- pace/client.py +165 -0
- pace/dynamics.py +372 -0
- pace/escape_benchmarks.py +162 -0
- pace/examples/crewai_integration.py +16 -0
- pace/examples/demo_pool.jsonl +360 -0
- pace/examples/demo_run.jsonl +480 -0
- pace/examples/langgraph_integration.py +27 -0
- pace/examples/make_demo.py +37 -0
- pace/examples/openai_quickstart.py +18 -0
- pace/experiment.py +555 -0
- pace/frontier.py +114 -0
- pace/hidden_profile.py +301 -0
- pace/importers.py +219 -0
- pace/inflate.py +213 -0
- pace/influence.py +231 -0
- pace/meta_engine.py +246 -0
- pace/middleware.py +230 -0
- pace/open_ended.py +114 -0
- pace/plotting.py +132 -0
- pace/pool_sizer.py +154 -0
- pace/protocol_v3.py +414 -0
- pace/proxy.py +106 -0
- pace/rebuttal.py +219 -0
- pace/report.py +197 -0
- pace/rollout_card.py +137 -0
- pace/schema.py +74 -0
- pace/simulator.py +199 -0
- pace/studio.html +1040 -0
- pace/tasks.py +266 -0
- pace/verified_debate.py +127 -0
- pace/verifier.py +92 -0
- pace/workbench.html +26 -0
- pace/workbench.py +119 -0
- pace_eval-0.2.2.dist-info/METADATA +205 -0
- pace_eval-0.2.2.dist-info/RECORD +83 -0
- pace_eval-0.2.2.dist-info/WHEEL +5 -0
- pace_eval-0.2.2.dist-info/entry_points.txt +4 -0
- pace_eval-0.2.2.dist-info/licenses/LICENSE +202 -0
- pace_eval-0.2.2.dist-info/top_level.txt +4 -0
- pace_lab/__init__.py +9 -0
- pace_lab/clients.py +568 -0
- pace_lab/concurrency_test.py +127 -0
- pace_lab/data.py +445 -0
- pace_lab/graders.py +626 -0
- pace_lab/strategies.py +345 -0
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: pace-eval
|
|
3
|
+
Version: 0.2.2
|
|
4
|
+
Summary: PACE: Parity-Aware Compute Evaluation for Multi-Agent LLMs (CLI, Resource Vector Auditor, Debate Dynamics & Browser Studio)
|
|
5
|
+
Author-email: Seyed Mahmoud Sajjadi Mohammadabadi <mahmoud.sajjadi@unr.edu>
|
|
6
|
+
License: Apache-2.0
|
|
7
|
+
Project-URL: Homepage, https://github.com/mahmoudsajjadi/pace
|
|
8
|
+
Project-URL: Paper, https://arxiv.org/abs/2611.17610
|
|
9
|
+
Project-URL: Studio, https://huggingface.co/spaces/sajjadi/pace-studio
|
|
10
|
+
Project-URL: Dataset, https://huggingface.co/datasets/sajjadi/pace-contrasts
|
|
11
|
+
Project-URL: Archive, https://doi.org/10.5281/zenodo.24286397
|
|
12
|
+
Keywords: llm,multi-agent,evaluation,test-time-compute,self-consistency,benchmarking,pace
|
|
13
|
+
Classifier: Development Status :: 4 - Beta
|
|
14
|
+
Classifier: Intended Audience :: Science/Research
|
|
15
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
21
|
+
Requires-Python: >=3.10
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
License-File: LICENSE
|
|
24
|
+
Requires-Dist: numpy>=1.24.0
|
|
25
|
+
Provides-Extra: plot
|
|
26
|
+
Requires-Dist: matplotlib>=3.7.0; extra == "plot"
|
|
27
|
+
Provides-Extra: dev
|
|
28
|
+
Requires-Dist: pytest>=7.4.0; extra == "dev"
|
|
29
|
+
Requires-Dist: matplotlib>=3.7.0; extra == "dev"
|
|
30
|
+
Dynamic: license-file
|
|
31
|
+
|
|
32
|
+
# PACE: Parity-Aware Compute Evaluation
|
|
33
|
+
|
|
34
|
+
**More Agents, More Samples, or More Thinking? Untangling Coordination, Ensembling, and Test-Time Compute in Multi-Agent LLMs**
|
|
35
|
+
|
|
36
|
+
**Author:** Seyed Mahmoud Sajjadi Mohammadabadi
|
|
37
|
+
**Affiliation:** Department of Computer Science and Engineering, University of Nevada, Reno
|
|
38
|
+
**Correspondence:** `mahmoud.sajjadi@unr.edu`
|
|
39
|
+
**Repository:** [https://github.com/mahmoudsajjadi/pace](https://github.com/mahmoudsajjadi/pace)
|
|
40
|
+
**arXiv Preprint:** [arXiv:2611.17610](https://arxiv.org/abs/2611.17610) ([HTML version](https://arxiv.org/html/2611.17610v1))
|
|
41
|
+
**Paper PDF:** Preprint: [`manuscript/main.pdf`](manuscript/main.pdf) | Visual Cheatsheet: [`docs/assets/pace_cheatsheet.pdf`](docs/assets/pace_cheatsheet.pdf) | Interactive Studio: [`dist/hf_space_pace_studio/`](dist/hf_space_pace_studio/)
|
|
42
|
+
|
|
43
|
+
[](https://arxiv.org/abs/2611.17610)
|
|
44
|
+
[](LICENSE)
|
|
45
|
+
[](zenodo_datasets.json)
|
|
46
|
+
[](https://www.python.org/downloads/)
|
|
47
|
+
[](tests/)
|
|
48
|
+
[](https://doi.org/10.5281/zenodo.24286397)
|
|
49
|
+
[](https://colab.research.google.com/github/mahmoudsajjadi/pace/blob/main/notebooks/audit_your_log.ipynb)
|
|
50
|
+
|
|
51
|
+
**Reproducible on one consumer GPU.** The controlled experiments (`llama3.1:8b` served by Ollama, about 4.9 GB) run on a single 12 GB card (developed on an RTX 4070): `bash scripts/reproduce_12gb_gpu.sh` runs the tests, the hidden-profile file check and a 170-call smoke run, and records its own wall time (the expected runtimes in the script header are derived from a measured batch rate, not from a timed run of the script).
|
|
52
|
+
|
|
53
|
+
Official replication repository, audited contrast ledger (`9` primary studies, `516` unique multi-agent configurations from `531` raw extracted rows, `834` contrast records), `pace` CLI and evaluation toolkit (legacy aliases: `cme`, `mae-eval`), and zero-dependency browser workbench (**PACE Studio**) for Parity-Aware Compute Evaluation.
|
|
54
|
+
|
|
55
|
+
**Licences.** Code (`code/`, `analysis/`, `scripts/`, `tests/`): Apache-2.0 (`LICENSE`). Dataset files (extracted contrast tables and the hidden-profile task, staged in `zenodo_datasets.json`): CC-BY-4.0.
|
|
56
|
+
|
|
57
|
+

|
|
58
|
+
|
|
59
|
+
---
|
|
60
|
+
|
|
61
|
+
## Abstract & Canonical Findings
|
|
62
|
+
|
|
63
|
+
Multi-agent language model evaluations often compare communicating systems against a single greedy pass, confounding inter-agent coordination with independent sampling and expanded test-time compute. We introduce **Parity-Aware Compute Evaluation (PACE)**, which classifies single-agent baselines on a four-category ladder (**Category A**: single greedy pass $k=1$; **Category B**: noncommunicating sampled baseline; **Category C**: resource-matched or shared-ceiling baseline; **Category D**: compute-cost Pareto frontier), audits a **resource vector with six dimensions**
|
|
64
|
+
|
|
65
|
+
$$\mathbf{R} = \langle R_{\text{reason}}, R_{\text{token}}, R_{\text{cost}}, R_{\text{lat}}, R_{\text{sample}}, R_{\text{sys}} \rangle,$$
|
|
66
|
+
|
|
67
|
+
enforces directional resource feasibility, and measures the frontier gap relative to single-agent self-consistency at parity in tokens.
|
|
68
|
+
|
|
69
|
+
### Extracted Corpus & Empirical Synthesis
|
|
70
|
+
|
|
71
|
+
- **Canonical Denominators:** From `12` studies considered across two stages (`5` initial studies and `7` screened candidates: `4` included, `3` excluded), the included corpus comprises **`9` primary studies**, **`516` unique multi-agent configurations** (`531` raw extracted rows minus `15` baseline-only rows), and **`834` baseline-specific contrast records** (`358` Category A, `360` Category B, `114` Category C, and `2` Category D Pareto contrasts; **`474` combined Category B/C contrasts** across `8` studies; **`156` shared-ceiling contrasts**; **`318` paired Category A/B contrasts** across **`72` unique baseline cells**).
|
|
72
|
+
- **Single-Pass vs. Compute-Controlled Categories:** Mean accuracy differences are **`-0.032`** against Category A (`358` contrasts; `-0.005` omitting Zhang et al., who supply `180` configurations), **`-0.051`** against Category B (`360` contrasts), and **`-0.012`** against Category C (`114` contrasts). Across combined Category B/C (`474` contrasts, `8` studies), the unweighted contrast mean is **`-0.042`**, with exploratory three-level REML log odds $\hat\beta = \mathbf{-0.195}$ (CR2 95% CI `[-0.323, -0.067]`; 95% study-mean prediction interval `[-0.515, +0.125]`).
|
|
73
|
+
- **Consistency Across Studies:** Multi-agent systems fail to lead in **`74%`** (`352` of `474`) of Category B/C contrasts, feasible baselines match or beat them in **`36%`** (`171` of `474` directionally feasible; `56` below `-1.96 SE`; `69%` or `356` of `516` unique configurations across Categories A/B/C), and **`8 of 9`** study means are nonpositive (sole positive exception: Wunderlich et al. at **`+0.020`** at stated configurations on MMLU-Pro, compute not matched).
|
|
74
|
+
- **Sampling Gap ($\Delta_{\text{gap}}$):** Across `5` dual-baseline studies (`318` pairs spanning `72` unique baseline cells, `64 of 72` negative), switching from greedy to sampled baselines lowers accuracy differences by **`-0.038`** weighted equally by study (**`-0.028`** across pairs; **`-0.036`** across the `72` unique cells; cell-level REML $\hat\beta = \mathbf{-0.178}$, CR2 95% CI `[-0.347, -0.009]`).
|
|
75
|
+
- **Controlled Open-Weight Experiment (`llama3.1:8b` via Ollama):**
|
|
76
|
+
- **Shared-evidence MMLU-Pro (`n=250`):** Vanilla debate (`−0.015` frontier gap, `0.567` vs. `0.581` frontier) and Mixture-of-Agents (`−0.072` frontier gap) do not clear the equal-token self-consistency frontier under Holm correction.
|
|
77
|
+
- **Complementary Hidden Profile split-view control (`n=200`):** Split-view debate exceeds the split-view independent-voting frontier (`+0.073` frontier gap, `0.182` vs. `0.109`, Holm-adjusted `p = 0.007`), while trailing content-free placebo prompts (`0.248`) and four-choice chance (`0.250`).
|
|
78
|
+
|
|
79
|
+
---
|
|
80
|
+
|
|
81
|
+
## Repository Structure
|
|
82
|
+
|
|
83
|
+
```text
|
|
84
|
+
├── .github/ # CI workflows & study-contrast issue template (add_study_contrast.yml)
|
|
85
|
+
├── analysis/ # Synthesis, REML pooling, plotting, and LaTeX build gates
|
|
86
|
+
│ ├── assemble_paper.py # Assembles & compiles manuscript/main.pdf
|
|
87
|
+
│ ├── run_baseline_tiers.py # Descriptive corpus synthesis, dual-baseline pairing & numbers.tex
|
|
88
|
+
│ └── run_k9_reml.py # Three-level REML + CR2 inference (including 72-cell Model D)
|
|
89
|
+
├── code/
|
|
90
|
+
│ ├── pace/ # Standalone PACE CLI, importers (incl. Inspect AI), & browser Studio
|
|
91
|
+
│ ├── cme/ # Legacy CLI/API aliases preserved for backwards compatibility
|
|
92
|
+
│ └── mae_eval/ # Legacy resource tracker/auditor package aliases
|
|
93
|
+
├── dist/
|
|
94
|
+
│ ├── assets/ # Explainer GIF & cheatsheet (PNG/PDF)
|
|
95
|
+
│ ├── github_release_staging/# Offline-verified GitHub release staging tree
|
|
96
|
+
│ └── hf_space_pace_studio/ # Hugging Face Space workbench + corpus_contrasts.json (832 rows)
|
|
97
|
+
├── docs/ # Project landing page (index.html), community playbook & visual assets
|
|
98
|
+
├── evidence/ # Verified extracted CSVs, screening ledger & audit reports
|
|
99
|
+
│ ├── contrasts_anchor5_core.csv
|
|
100
|
+
│ ├── contrasts_dual_baseline.csv
|
|
101
|
+
│ ├── extension_contrasts.csv
|
|
102
|
+
│ └── verification_worksheet.csv
|
|
103
|
+
├── figures/ # Publication vector PDF figures and PNG previews
|
|
104
|
+
├── manuscript/ # Modular LaTeX source of truth and generated macro files
|
|
105
|
+
├── notebooks/ # Interactive Colab/Jupyter log-auditing notebook (audit_your_log.ipynb)
|
|
106
|
+
├── outputs/ # Baseline contrast tables, REML JSON & experimental logs
|
|
107
|
+
├── paper/ # Synchronized manuscript build targets and figure/table mirrors
|
|
108
|
+
├── pyproject.toml # PEP 621 package metadata (`pace-eval`) & CLI entry points
|
|
109
|
+
├── review/ # Programmatic concordance ledgers & claim-verification tables
|
|
110
|
+
├── scripts/ # Reproducibility runner (12 GB GPU), corpus builders & exporters
|
|
111
|
+
├── supplement/ # Moderator synthesis, empirical trace harvest & author templates
|
|
112
|
+
└── tests/ # Automated unit, regression, and concordance test suite
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
---
|
|
116
|
+
|
|
117
|
+
## Quickstart: the `pace` CLI & PACE Studio
|
|
118
|
+
|
|
119
|
+
Install `pace-eval` from the repository root (`pyproject.toml`) or from `code/`:
|
|
120
|
+
|
|
121
|
+
```bash
|
|
122
|
+
pip install -e . # installs `pace` (plus legacy `cme` and `mae-eval` console aliases)
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
Core auditing, reporting, log import (including Inspect AI via `--format inspect`), and local GUI commands:
|
|
126
|
+
|
|
127
|
+
```bash
|
|
128
|
+
pace studio # launches the zero-dependency browser UI (PACE Studio)
|
|
129
|
+
pace classify --config sas.json
|
|
130
|
+
pace audit run_trace.jsonl
|
|
131
|
+
pace import-logs --format inspect --input eval_log.json --output trace.jsonl && pace audit trace.jsonl
|
|
132
|
+
pace report --pool pool.jsonl --logs run.jsonl --system debate --export-latex card.tex
|
|
133
|
+
pace dynamics debate_log.jsonl --null null.jsonl --json
|
|
134
|
+
pace card run_trace.jsonl --latex --output pace-card.tex
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
Constructive planning, simulation, meta-analysis, frontier, and rebuttal commands:
|
|
138
|
+
|
|
139
|
+
```bash
|
|
140
|
+
pace pool-size --agents 5 --rounds 3 --prompt-tokens 100 --msg-tokens 20 --json
|
|
141
|
+
pace simulate --mode debate --items 100 --agents 5 --rounds 3 --out debate.jsonl
|
|
142
|
+
pace meta --contrasts outputs/baseline_tier_contrasts.csv --scale log_or --json
|
|
143
|
+
pace power --p-base .55 --delta .10 --n-items 40 --json
|
|
144
|
+
pace benchmark-gen --route clue_grid --out clues.jsonl
|
|
145
|
+
pace aggregate-open --input candidates.jsonl --metric token_jaccard --out centroids.jsonl
|
|
146
|
+
pace plot --kind phase --out phase.svg
|
|
147
|
+
pace hidden-profile --n 100 --out hp.jsonl
|
|
148
|
+
pace frontier --samples pool.jsonl --logs run.jsonl --out frontier.pdf
|
|
149
|
+
pace rebuttal --contrasts data/baseline_tier_contrasts.csv --export-markdown rebuttal.md
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
End-to-end synthetic walkthrough, interactive Colab notebook, GitHub Action CI workflow template, and community study-submission playbook live in:
|
|
153
|
+
- [`notebooks/audit_your_log.ipynb`](notebooks/audit_your_log.ipynb)
|
|
154
|
+
- [`code/examples/end_to_end_walkthrough.py`](code/examples/end_to_end_walkthrough.py)
|
|
155
|
+
- [`code/examples/github_action_pace_audit.yml`](code/examples/github_action_pace_audit.yml)
|
|
156
|
+
- [`docs/index.html`](docs/index.html), [`docs/COMMUNITY_AND_PR_PLAYBOOK.md`](docs/COMMUNITY_AND_PR_PLAYBOOK.md), and [`.github/ISSUE_TEMPLATE/add_study_contrast.yml`](.github/ISSUE_TEMPLATE/add_study_contrast.yml)
|
|
157
|
+
- Visual assets: [`docs/assets/pace_explainer.gif`](docs/assets/pace_explainer.gif), [`docs/assets/pace_cheatsheet.png`](docs/assets/pace_cheatsheet.png), and [`dist/assets/`](dist/assets/)
|
|
158
|
+
|
|
159
|
+
Legacy imports (`from cme ...` and `from mae_eval ...`) remain supported alongside `pace`.
|
|
160
|
+
|
|
161
|
+
---
|
|
162
|
+
|
|
163
|
+
## Reporting Standard: Evaluation Card & Progressive Levels
|
|
164
|
+
|
|
165
|
+
Every completed comparison should report:
|
|
166
|
+
1. **The seven-field PACE Evaluation Card:** Baseline rung (`A`–`D`), resource vector $\mathbf{R}$, token accounting (prompt, completion, cached vs. uncached tokens), MAS/SAS resource ratios, frontier gap with 95% paired bootstrap CI, evidence/route structure (`shared` vs. `split`, verifier availability, model heterogeneity), and per-item resource distribution and stopping rules.
|
|
167
|
+
2. **Three progressive reporting levels:**
|
|
168
|
+
- **`PACE-Minimum`:** Category B/C baseline (e.g., nested initial-round self-consistency voting), full token/call counts, and paired bootstrap accuracy intervals.
|
|
169
|
+
- **`PACE-Recommended`:** Equal-token self-consistency frontier curve, directional feasibility across all six dimensions of $\mathbf{R}$, and placebo control or split-view checks.
|
|
170
|
+
- **`PACE-Extended`:** Per-item budget matching ($k_i$), cache-weighted billing accounting, adaptive stopping distributions, hybrid call-graph edge counts, and peer-influence transition dynamics ($\tau_{i\to c}$ vs. $\tau_{c\to i}$).
|
|
171
|
+
|
|
172
|
+
---
|
|
173
|
+
|
|
174
|
+
## Verification & Manuscript Compilation
|
|
175
|
+
|
|
176
|
+
Run the full automated verification suite (`865 passed, 8 skipped`):
|
|
177
|
+
|
|
178
|
+
```bash
|
|
179
|
+
pytest -q tests/
|
|
180
|
+
```
|
|
181
|
+
|
|
182
|
+
Assemble and verify all PDF targets (`manuscript/main.pdf`, `paper/main.pdf`) with Build Gates 1 and 2:
|
|
183
|
+
|
|
184
|
+
```bash
|
|
185
|
+
python -I analysis/assemble_paper.py
|
|
186
|
+
```
|
|
187
|
+
|
|
188
|
+
---
|
|
189
|
+
|
|
190
|
+
## Citation
|
|
191
|
+
|
|
192
|
+
```bibtex
|
|
193
|
+
@article{sajjadi2026more,
|
|
194
|
+
title = {More Agents, More Samples, or More Thinking? Untangling Coordination, Ensembling, and Test-Time Compute in Multi-Agent LLMs},
|
|
195
|
+
author = {Sajjadi Mohammadabadi, Seyed Mahmoud},
|
|
196
|
+
journal = {arXiv preprint arXiv:2611.17610},
|
|
197
|
+
year = {2026},
|
|
198
|
+
eprint = {2611.17610},
|
|
199
|
+
archivePrefix = {arXiv},
|
|
200
|
+
primaryClass = {cs.AI},
|
|
201
|
+
doi = {10.48550/arXiv.2611.17610},
|
|
202
|
+
url = {https://arxiv.org/abs/2611.17610},
|
|
203
|
+
note = {Replication repository at https://github.com/mahmoudsajjadi/pace}
|
|
204
|
+
}
|
|
205
|
+
```
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
cme/__init__.py,sha256=U3NT5OKjxhs3WDWPcA6E8HiucZKSbh83zAo7ae8uqno,805
|
|
2
|
+
cme/__main__.py,sha256=E6Gls0DNz8GQK2K-kOUIx8cYhgANW_CH54VKrfCfs14,52
|
|
3
|
+
cme/analysis.py,sha256=tmZTi_6aALNge6C-d5uZIoNClPCCoZUIJzoY3ELL33s,7841
|
|
4
|
+
cme/audit.py,sha256=S7W2crOuW04z7tEpthHYedeFsRQ8sgvX-Jp1y-F8h6c,7314
|
|
5
|
+
cme/baseline.py,sha256=INov6vntRcRhYynnwvpfuxtlj8zC7rV6djDOogYI3i8,7942
|
|
6
|
+
cme/cli.py,sha256=DkIQCGCZgB-TqUXs1wLmCFk91-R22Pj8tedWPaucS8Q,15114
|
|
7
|
+
cme/client.py,sha256=ONMcWWN4SLtghwiUiaWkZ-AJ-HiCA2LF6Z2zTqdvRZ0,7566
|
|
8
|
+
cme/dynamics.py,sha256=iebIB-myujYxDC_v9T4Cvj_s0BYJFKjQpENgSdxiGsA,12360
|
|
9
|
+
cme/experiment.py,sha256=4_XsnReDmzjL604cUoyGD7QopiBx0zcGGWvvFIW674w,32396
|
|
10
|
+
cme/frontier.py,sha256=bMDnwU0uwZ8mRCPwUjEUgqfiOWKK1AsFPmXsijWgzqE,6018
|
|
11
|
+
cme/hidden_profile.py,sha256=zfYqvXDmYopP0suGib9DRXF_FORF4_Pyv1s4t1O3tq8,7590
|
|
12
|
+
cme/proxy.py,sha256=eOcuAVehUEuVgwdeEOBaU4URlr3r5k0RvEUQsFT7l9g,5594
|
|
13
|
+
cme/rebuttal.py,sha256=bsQ_CbW6vtomLJxbTdOQseHRtJt8QzQE7nWgZlkDOKg,4097
|
|
14
|
+
cme/report.py,sha256=QAfquw1y3pPHqftJ9e_A4zWnxasZ_gdUUlnEtK-bWNE,11470
|
|
15
|
+
cme/rollout_card.py,sha256=XyM-nzVif6AlYk66XdFzuGQ77JEiZH9rm9F9lkseNHE,9833
|
|
16
|
+
cme/tasks.py,sha256=b77FOWislFKmg7fQSw2erIvx16w5Df60xS6TydvetQk,13749
|
|
17
|
+
cme/verified_debate.py,sha256=6wzm1C3KUrYobPdHvnxrPhcybbd1hyDyOpH0PaTHZLc,7441
|
|
18
|
+
cme/verifier.py,sha256=sQ4OyU95aCZhGbTGqMYCCrs6yIF1yeMhWmt32V4dYPg,4664
|
|
19
|
+
cme/examples/crewai_integration.py,sha256=rcuwRwj4LJJ9u15jVDUgrwjNIRPxDbRQuWwoO9ZWOuQ,1230
|
|
20
|
+
cme/examples/demo_pool.jsonl,sha256=UZ-S4Yl-p8h9oAs_W-EX2jX7lPSaws5wNdpNcWs1DZc,78204
|
|
21
|
+
cme/examples/demo_run.jsonl,sha256=o1eYosMJqYlVXB0sg1hmHK0O9uGP9K70AmIQ57aOvgQ,59397
|
|
22
|
+
cme/examples/langgraph_integration.py,sha256=qoYlCLbOKNa8OFKmbgs_l1jULKknrNPtBRCXzFITAYs,1274
|
|
23
|
+
cme/examples/make_demo.py,sha256=7IXi3DcJEKhbT1l9r8xrJHGtaNrxbyKTUR-WFKkClx8,2389
|
|
24
|
+
cme/examples/openai_quickstart.py,sha256=k0Jg074aQHTvCT25mPRpIUqXOgI0vcRW2fDo18Gv1PM,1242
|
|
25
|
+
mae_eval/__init__.py,sha256=mV3CVf9eRk4J8QXBWA24umHy-0swiod5t_WhyK9_KiU,648
|
|
26
|
+
mae_eval/__main__.py,sha256=E6Gls0DNz8GQK2K-kOUIx8cYhgANW_CH54VKrfCfs14,52
|
|
27
|
+
mae_eval/auditor.py,sha256=iNgWBK3QmL62z0WYXwCJ7dplEDQHb3BKc-eJII5bYUM,4996
|
|
28
|
+
mae_eval/classifier.py,sha256=vUhq8roZ2bLMZp-awQbayNWXK9tHnczVwHOLpxvhj5M,4038
|
|
29
|
+
mae_eval/cli.py,sha256=U6R2exN_FbO4rbvHYG0_Grjtsh1PGTzSdk-_J0Mhs50,10843
|
|
30
|
+
mae_eval/tracker.py,sha256=BiyFlpUr4QkVS8sR8h0uwHj-ZnU-2A7TyXB-DOi05Fk,8067
|
|
31
|
+
mae_eval/vector.py,sha256=Tw33Wv9WRTDRyQDqqYNncR4qBfDTfY79NK7AogN2K3o,4423
|
|
32
|
+
pace/__init__.py,sha256=aHYP2-tt1LWJ4D_u8yAy434RkLjplxi4-dX_7o-5bCc,802
|
|
33
|
+
pace/__main__.py,sha256=E6Gls0DNz8GQK2K-kOUIx8cYhgANW_CH54VKrfCfs14,52
|
|
34
|
+
pace/analysis.py,sha256=J6M2ZZf2Qd0HbHg1cE4oDuOKkPzutIvcGV9PjFbmbTM,6820
|
|
35
|
+
pace/audit.py,sha256=lm-P8LDMyCvC-T9Lf53aFHoaw3LOuP3mbpcUkAQdzFY,7175
|
|
36
|
+
pace/baseline.py,sha256=SC2CE4EcNpBtD5QcWbZ_APsbVocoZhp3YtZEYtSSKoI,7943
|
|
37
|
+
pace/card.py,sha256=eos40m_OPe9ofpaiKFiSzYeUL4XT8GFdWOZCVwjY9xY,8126
|
|
38
|
+
pace/cli.py,sha256=NWp5zQz-leQ-TfIi299ovMBNmaXHg5XwbGTrFPVqe1c,32609
|
|
39
|
+
pace/client.py,sha256=_r4fORsfbdH3rxtOsqmwhNnJ9ewxhhI_uwVjw8iDCYM,7578
|
|
40
|
+
pace/dynamics.py,sha256=d7L_ll0zLhrrFu3ZBbfsQ8OqPen774sg_NUUPVXvBks,14800
|
|
41
|
+
pace/escape_benchmarks.py,sha256=zZeveHwFVRRcTQrsLaUw_1q2Pe5kHm5pWPgDZSMIjp0,5911
|
|
42
|
+
pace/experiment.py,sha256=w-hodvllMom4XofVUZ1aNn-jtLzAEyzzSd_unL--dEM,31845
|
|
43
|
+
pace/frontier.py,sha256=ag6Bi4hQ9tQcdu5p9XxhXJfWS4uLHlJWN0kpxwJu9RA,5750
|
|
44
|
+
pace/hidden_profile.py,sha256=gz0ndmbysmLCpeMJSD-7HDJHMHwdlgMvcONAqyltV00,11967
|
|
45
|
+
pace/importers.py,sha256=ISkBmWvwSQiNjANV4ImzqQTg1be4xJ8Iege4UIy8KY8,8833
|
|
46
|
+
pace/inflate.py,sha256=1sqqAzoY--_56M-qaaUu9A-PxdeZxMipnLyra_Sa9cA,9531
|
|
47
|
+
pace/influence.py,sha256=v4zGVOgMuZwMbI0Wa6sythDOWXB6RPptiw0FZe6sQeM,9057
|
|
48
|
+
pace/meta_engine.py,sha256=I8gdLufM-mCe4eKp3bni_0l4VyGcxdkmg6iGQY-E3XY,7890
|
|
49
|
+
pace/middleware.py,sha256=WQkdQ0H2pjH1zH5PCgWfCUBsV3nWYVjaN3VDHdLBIJo,7412
|
|
50
|
+
pace/open_ended.py,sha256=lV6er72kNT7wzVbdD8Ti2mfobq4G3uhyokdi82gelXA,4038
|
|
51
|
+
pace/plotting.py,sha256=nFbiTsrMbwukh34yym9uVM49OfE3vxrzWCaUUCrIAP0,5454
|
|
52
|
+
pace/pool_sizer.py,sha256=bI8NVYo4m5VczqJo0b06_wrVGj-0EkBdsLCfayTE4dk,5258
|
|
53
|
+
pace/protocol_v3.py,sha256=uJo0uaTdJiN-BjmBBPhgiJPigfHSniJleDlq6O5M0co,14252
|
|
54
|
+
pace/proxy.py,sha256=F3MxYq219mrE44JyXS3nsYje_o5MVnogDh-2ABiLqF0,5500
|
|
55
|
+
pace/rebuttal.py,sha256=lPISe6nnCHhbtHn-dfFUhhRjTqjIxbTNg1tPORoK6RM,7485
|
|
56
|
+
pace/report.py,sha256=XL_75euTgVdsFy5PZTfwM6qH1UG5Ktv8cmyLn5-BXNw,11331
|
|
57
|
+
pace/rollout_card.py,sha256=ToQYFAy72tj3QEWpCyrwgvofOhymETfhLWdtd9lTc18,9707
|
|
58
|
+
pace/schema.py,sha256=7kjFAB5DEXbWq1OD7zijIQWcBs1t24tceTUSonZHYVw,2420
|
|
59
|
+
pace/simulator.py,sha256=ow-iEUweQM_MvwBQGoDUUxGLEwU89hwNg4yvOh11qA0,7603
|
|
60
|
+
pace/studio.html,sha256=j-TcIFz1Mb4IbAum7N-ObXVOFjEcET5rzhWjdvoSTcU,48697
|
|
61
|
+
pace/tasks.py,sha256=OZGslU569BWSLGnrIPetOEHWWdLL-npeZXV0ZStLj70,13483
|
|
62
|
+
pace/verified_debate.py,sha256=yfPFKJIfwmk5D7EyTyjxOxLGzftsAO3B6oFNyZndcPw,7317
|
|
63
|
+
pace/verifier.py,sha256=2EJ6dPDnTLf5pzfPMlR3muYkLCwW1q9parwy94PEdsI,4572
|
|
64
|
+
pace/workbench.html,sha256=28FRizLBSUcj7J0bRwSIQ9G7ttJp5gXT_TEl5iQhi2w,9274
|
|
65
|
+
pace/workbench.py,sha256=F6uqS-u8cinjvwrPTXU9qpHWzSdTk5h8n83Ga6qe2ts,4157
|
|
66
|
+
pace/examples/crewai_integration.py,sha256=qW9J0enD8gh3YwWbpQuRVMqh9VLHNKRCCwOSkdi7QHw,1236
|
|
67
|
+
pace/examples/demo_pool.jsonl,sha256=UZ-S4Yl-p8h9oAs_W-EX2jX7lPSaws5wNdpNcWs1DZc,78204
|
|
68
|
+
pace/examples/demo_run.jsonl,sha256=o1eYosMJqYlVXB0sg1hmHK0O9uGP9K70AmIQ57aOvgQ,59397
|
|
69
|
+
pace/examples/langgraph_integration.py,sha256=8oCdiPPAVfODLfISEdo6UtgU15E3C8f4poJL1Qq6LFY,1279
|
|
70
|
+
pace/examples/make_demo.py,sha256=7IXi3DcJEKhbT1l9r8xrJHGtaNrxbyKTUR-WFKkClx8,2389
|
|
71
|
+
pace/examples/openai_quickstart.py,sha256=rpnDqi396q7lwGEhFfJ4ZonjIaZpq27pgTMILDK0DxA,1251
|
|
72
|
+
pace_eval-0.2.2.dist-info/licenses/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
|
|
73
|
+
pace_lab/__init__.py,sha256=WeuEhgJ7nHWsA16I9fIsH011qhEUXYEZUyR0jOq8-8M,634
|
|
74
|
+
pace_lab/clients.py,sha256=7GLv7RYPqAKhieA_I3b4a7wsKKDoSNraEIl30SRcg8E,28337
|
|
75
|
+
pace_lab/concurrency_test.py,sha256=6QwbCdcH34of5piymXXRV2XKunZpfXlvUEZBVExisHo,4532
|
|
76
|
+
pace_lab/data.py,sha256=ykS8tVcW8X3Qr64LxUMVP8-owDUfbZp_5aYWbSuRQ0M,22347
|
|
77
|
+
pace_lab/graders.py,sha256=uK-cRvFS7z69bK7v6GwS8zmAufy3DIjhrdv3d03TZSo,26278
|
|
78
|
+
pace_lab/strategies.py,sha256=KsgqxQskmHhSTFj2pur5Z-zfj4ccgQ0aNybMk2V8S68,11612
|
|
79
|
+
pace_eval-0.2.2.dist-info/METADATA,sha256=06ZQKJHzhtshZ4NTNBePfhODRJmsgXIH33Peb93MORo,14960
|
|
80
|
+
pace_eval-0.2.2.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
|
|
81
|
+
pace_eval-0.2.2.dist-info/entry_points.txt,sha256=PoA3YGmfc6egJ28LrUyf128fLU5-TV7q8zzsv-eZyog,87
|
|
82
|
+
pace_eval-0.2.2.dist-info/top_level.txt,sha256=_p4HbSvlXyVMUp7q3M4uRvUanEz717JnbZ77-TuZWrM,27
|
|
83
|
+
pace_eval-0.2.2.dist-info/RECORD,,
|
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
|
|
2
|
+
Apache License
|
|
3
|
+
Version 2.0, January 2004
|
|
4
|
+
http://www.apache.org/licenses/
|
|
5
|
+
|
|
6
|
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
|
7
|
+
|
|
8
|
+
1. Definitions.
|
|
9
|
+
|
|
10
|
+
"License" shall mean the terms and conditions for use, reproduction,
|
|
11
|
+
and distribution as defined by Sections 1 through 9 of this document.
|
|
12
|
+
|
|
13
|
+
"Licensor" shall mean the copyright owner or entity authorized by
|
|
14
|
+
the copyright owner that is granting the License.
|
|
15
|
+
|
|
16
|
+
"Legal Entity" shall mean the union of the acting entity and all
|
|
17
|
+
other entities that control, are controlled by, or are under common
|
|
18
|
+
control with that entity. For the purposes of this definition,
|
|
19
|
+
"control" means (i) the power, direct or indirect, to cause the
|
|
20
|
+
direction or management of such entity, whether by contract or
|
|
21
|
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
|
22
|
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
|
23
|
+
|
|
24
|
+
"You" (or "Your") shall mean an individual or Legal Entity
|
|
25
|
+
exercising permissions granted by this License.
|
|
26
|
+
|
|
27
|
+
"Source" form shall mean the preferred form for making modifications,
|
|
28
|
+
including but not limited to software source code, documentation
|
|
29
|
+
source, and configuration files.
|
|
30
|
+
|
|
31
|
+
"Object" form shall mean any form resulting from mechanical
|
|
32
|
+
transformation or translation of a Source form, including but
|
|
33
|
+
not limited to compiled object code, generated documentation,
|
|
34
|
+
and conversions to other media types.
|
|
35
|
+
|
|
36
|
+
"Work" shall mean the work of authorship, whether in Source or
|
|
37
|
+
Object form, made available under the License, as indicated by a
|
|
38
|
+
copyright notice that is included in or attached to the work
|
|
39
|
+
(an example is provided in the Appendix below).
|
|
40
|
+
|
|
41
|
+
"Derivative Works" shall mean any work, whether in Source or Object
|
|
42
|
+
form, that is based on (or derived from) the Work and for which the
|
|
43
|
+
editorial revisions, annotations, elaborations, or other modifications
|
|
44
|
+
represent, as a whole, an original work of authorship. For the purposes
|
|
45
|
+
of this License, Derivative Works shall not include works that remain
|
|
46
|
+
separable from, or merely link (or bind by name) to the interfaces of,
|
|
47
|
+
the Work and Derivative Works thereof.
|
|
48
|
+
|
|
49
|
+
"Contribution" shall mean any work of authorship, including
|
|
50
|
+
the original version of the Work and any modifications or additions
|
|
51
|
+
to that Work or Derivative Works thereof, that is intentionally
|
|
52
|
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
|
53
|
+
or by an individual or Legal Entity authorized to submit on behalf of
|
|
54
|
+
the copyright owner. For the purposes of this definition, "submitted"
|
|
55
|
+
means any form of electronic, verbal, or written communication sent
|
|
56
|
+
to the Licensor or its representatives, including but not limited to
|
|
57
|
+
communication on electronic mailing lists, source code control systems,
|
|
58
|
+
and issue tracking systems that are managed by, or on behalf of, the
|
|
59
|
+
Licensor for the purpose of discussing and improving the Work, but
|
|
60
|
+
excluding communication that is conspicuously marked or otherwise
|
|
61
|
+
designated in writing by the copyright owner as "Not a Contribution."
|
|
62
|
+
|
|
63
|
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
|
64
|
+
on behalf of whom a Contribution has been received by Licensor and
|
|
65
|
+
subsequently incorporated within the Work.
|
|
66
|
+
|
|
67
|
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
|
68
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
69
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
70
|
+
copyright license to reproduce, prepare Derivative Works of,
|
|
71
|
+
publicly display, publicly perform, sublicense, and distribute the
|
|
72
|
+
Work and such Derivative Works in Source or Object form.
|
|
73
|
+
|
|
74
|
+
3. Grant of Patent License. Subject to the terms and conditions of
|
|
75
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
76
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
77
|
+
(except as stated in this section) patent license to make, have made,
|
|
78
|
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
|
79
|
+
where such license applies only to those patent claims licensable
|
|
80
|
+
by such Contributor that are necessarily infringed by their
|
|
81
|
+
Contribution(s) alone or by combination of their Contribution(s)
|
|
82
|
+
with the Work to which such Contribution(s) was submitted. If You
|
|
83
|
+
institute patent litigation against any entity (including a
|
|
84
|
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
|
85
|
+
or a Contribution incorporated within the Work constitutes direct
|
|
86
|
+
or contributory patent infringement, then any patent licenses
|
|
87
|
+
granted to You under this License for that Work shall terminate
|
|
88
|
+
as of the date such litigation is filed.
|
|
89
|
+
|
|
90
|
+
4. Redistribution. You may reproduce and distribute copies of the
|
|
91
|
+
Work or Derivative Works thereof in any medium, with or without
|
|
92
|
+
modifications, and in Source or Object form, provided that You
|
|
93
|
+
meet the following conditions:
|
|
94
|
+
|
|
95
|
+
(a) You must give any other recipients of the Work or
|
|
96
|
+
Derivative Works a copy of this License; and
|
|
97
|
+
|
|
98
|
+
(b) You must cause any modified files to carry prominent notices
|
|
99
|
+
stating that You changed the files; and
|
|
100
|
+
|
|
101
|
+
(c) You must retain, in the Source form of any Derivative Works
|
|
102
|
+
that You distribute, all copyright, patent, trademark, and
|
|
103
|
+
attribution notices from the Source form of the Work,
|
|
104
|
+
excluding those notices that do not pertain to any part of
|
|
105
|
+
the Derivative Works; and
|
|
106
|
+
|
|
107
|
+
(d) If the Work includes a "NOTICE" text file as part of its
|
|
108
|
+
distribution, then any Derivative Works that You distribute must
|
|
109
|
+
include a readable copy of the attribution notices contained
|
|
110
|
+
within such NOTICE file, excluding those notices that do not
|
|
111
|
+
pertain to any part of the Derivative Works, in at least one
|
|
112
|
+
of the following places: within a NOTICE text file distributed
|
|
113
|
+
as part of the Derivative Works; within the Source form or
|
|
114
|
+
documentation, if provided along with the Derivative Works; or,
|
|
115
|
+
within a display generated by the Derivative Works, if and
|
|
116
|
+
wherever such third-party notices normally appear. The contents
|
|
117
|
+
of the NOTICE file are for informational purposes only and
|
|
118
|
+
do not modify the License. You may add Your own attribution
|
|
119
|
+
notices within Derivative Works that You distribute, alongside
|
|
120
|
+
or as an addendum to the NOTICE text from the Work, provided
|
|
121
|
+
that such additional attribution notices cannot be construed
|
|
122
|
+
as modifying the License.
|
|
123
|
+
|
|
124
|
+
You may add Your own copyright statement to Your modifications and
|
|
125
|
+
may provide additional or different license terms and conditions
|
|
126
|
+
for use, reproduction, or distribution of Your modifications, or
|
|
127
|
+
for any such Derivative Works as a whole, provided Your use,
|
|
128
|
+
reproduction, and distribution of the Work otherwise complies with
|
|
129
|
+
the conditions stated in this License.
|
|
130
|
+
|
|
131
|
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
|
132
|
+
any Contribution intentionally submitted for inclusion in the Work
|
|
133
|
+
by You to the Licensor shall be under the terms and conditions of
|
|
134
|
+
this License, without any additional terms or conditions.
|
|
135
|
+
Notwithstanding the above, nothing herein shall supersede or modify
|
|
136
|
+
the terms of any separate license agreement you may have executed
|
|
137
|
+
with Licensor regarding such Contributions.
|
|
138
|
+
|
|
139
|
+
6. Trademarks. This License does not grant permission to use the trade
|
|
140
|
+
names, trademarks, service marks, or product names of the Licensor,
|
|
141
|
+
except as required for reasonable and customary use in describing the
|
|
142
|
+
origin of the Work and reproducing the content of the NOTICE file.
|
|
143
|
+
|
|
144
|
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
|
145
|
+
agreed to in writing, Licensor provides the Work (and each
|
|
146
|
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
|
147
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
148
|
+
implied, including, without limitation, any warranties or conditions
|
|
149
|
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
|
150
|
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
|
151
|
+
appropriateness of using or redistributing the Work and assume any
|
|
152
|
+
risks associated with Your exercise of permissions under this License.
|
|
153
|
+
|
|
154
|
+
8. Limitation of Liability. In no event and under no legal theory,
|
|
155
|
+
whether in tort (including negligence), contract, or otherwise,
|
|
156
|
+
unless required by applicable law (such as deliberate and grossly
|
|
157
|
+
negligent acts) or agreed to in writing, shall any Contributor be
|
|
158
|
+
liable to You for damages, including any direct, indirect, special,
|
|
159
|
+
incidental, or consequential damages of any character arising as a
|
|
160
|
+
result of this License or out of the use or inability to use the
|
|
161
|
+
Work (including but not limited to damages for loss of goodwill,
|
|
162
|
+
work stoppage, computer failure or malfunction, or any and all
|
|
163
|
+
other commercial damages or losses), even if such Contributor
|
|
164
|
+
has been advised of the possibility of such damages.
|
|
165
|
+
|
|
166
|
+
9. Accepting Warranty or Additional Liability. While redistributing
|
|
167
|
+
the Work or Derivative Works thereof, You may choose to offer,
|
|
168
|
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
|
169
|
+
or other liability obligations and/or rights consistent with this
|
|
170
|
+
License. However, in accepting such obligations, You may act only
|
|
171
|
+
on Your own behalf and on Your sole responsibility, not on behalf
|
|
172
|
+
of any other Contributor, and only if You agree to indemnify,
|
|
173
|
+
defend, and hold each Contributor harmless for any liability
|
|
174
|
+
incurred by, or claims asserted against, such Contributor by reason
|
|
175
|
+
of your accepting any such warranty or additional liability.
|
|
176
|
+
|
|
177
|
+
END OF TERMS AND CONDITIONS
|
|
178
|
+
|
|
179
|
+
APPENDIX: How to apply the Apache License to your work.
|
|
180
|
+
|
|
181
|
+
To apply the Apache License to your work, attach the following
|
|
182
|
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
|
183
|
+
replaced with your own identifying information. (Don't include
|
|
184
|
+
the brackets!) The text should be enclosed in the appropriate
|
|
185
|
+
comment syntax for the file format. We also recommend that a
|
|
186
|
+
file or class name and description of purpose be included on the
|
|
187
|
+
same "printed page" as the copyright notice for easier
|
|
188
|
+
identification within third-party archives.
|
|
189
|
+
|
|
190
|
+
Copyright [yyyy] [name of copyright owner]
|
|
191
|
+
|
|
192
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
193
|
+
you may not use this file except in compliance with the License.
|
|
194
|
+
You may obtain a copy of the License at
|
|
195
|
+
|
|
196
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
197
|
+
|
|
198
|
+
Unless required by applicable law or agreed to in writing, software
|
|
199
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
200
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
201
|
+
See the License for the specific language governing permissions and
|
|
202
|
+
limitations under the License.
|
pace_lab/__init__.py
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
"""pace_lab: client layer, data loaders and graders for the controlled experiments (Phase 4.1-4.4).
|
|
2
|
+
|
|
3
|
+
Submodules (import them explicitly; nothing here touches the network or reads a key at import time):
|
|
4
|
+
pace_lab.clients logged, cached, resumable LLM calls for Ollama, Groq and Gemini
|
|
5
|
+
pace_lab.concurrency_test OLLAMA_NUM_PARALLEL throughput/VRAM script (run by the author, needs the GPU)
|
|
6
|
+
pace_lab.data benchmark loaders with an id + SHA-256 manifest under data/pace_lab/
|
|
7
|
+
pace_lab.graders multiple-choice, math and hidden-profile answer extraction and grading
|
|
8
|
+
"""
|
|
9
|
+
__version__ = "0.1.0"
|