Benchtop 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- benchtop-0.1.0/LICENSE +21 -0
- benchtop-0.1.0/PKG-INFO +236 -0
- benchtop-0.1.0/README.md +217 -0
- benchtop-0.1.0/pyproject.toml +48 -0
- benchtop-0.1.0/setup.cfg +4 -0
- benchtop-0.1.0/src/Benchtop.egg-info/PKG-INFO +236 -0
- benchtop-0.1.0/src/Benchtop.egg-info/SOURCES.txt +37 -0
- benchtop-0.1.0/src/Benchtop.egg-info/dependency_links.txt +1 -0
- benchtop-0.1.0/src/Benchtop.egg-info/entry_points.txt +2 -0
- benchtop-0.1.0/src/Benchtop.egg-info/requires.txt +10 -0
- benchtop-0.1.0/src/Benchtop.egg-info/top_level.txt +3 -0
- benchtop-0.1.0/src/benchtop/__init__.py +6 -0
- benchtop-0.1.0/src/benchtop/_abstract_simulator.py +24 -0
- benchtop-0.1.0/src/benchtop/_observable_calculator.py +190 -0
- benchtop-0.1.0/src/benchtop/_organizer.py +147 -0
- benchtop-0.1.0/src/benchtop/_record.py +204 -0
- benchtop-0.1.0/src/benchtop/_results_cacher.py +137 -0
- benchtop-0.1.0/src/benchtop/_worker.py +186 -0
- benchtop-0.1.0/src/benchtop/arguments.py +99 -0
- benchtop-0.1.0/src/benchtop/experiment.py +394 -0
- benchtop-0.1.0/src/benchtop/file_loader.py +141 -0
- benchtop-0.1.0/src/benchtop/launcher.py +79 -0
- benchtop-0.1.0/src/benchtop/registry.py +28 -0
- benchtop-0.1.0/src/tests/conftest.py +40 -0
- benchtop-0.1.0/src/tests/data/plot-model.py +0 -0
- benchtop-0.1.0/src/tests/make_dummy.py +30 -0
- benchtop-0.1.0/src/tests/run_tests.py +11 -0
- benchtop-0.1.0/src/tests/test_benchtop.py +207 -0
- benchtop-0.1.0/src/tests/test_cache.py +166 -0
- benchtop-0.1.0/src/tests/test_obscalc.py +174 -0
- benchtop-0.1.0/src/tests/test_organizer.py +148 -0
- benchtop-0.1.0/src/tests/test_record.py +134 -0
- benchtop-0.1.0/src/tests/test_sparced_wrapper.py +18 -0
- benchtop-0.1.0/src/tests/test_worker.py +186 -0
- benchtop-0.1.0/src/wrappers/SingleCell.py +122 -0
- benchtop-0.1.0/src/wrappers/amici_wrapper.py +91 -0
- benchtop-0.1.0/src/wrappers/bngsim_wrapper.py +50 -0
- benchtop-0.1.0/src/wrappers/sparced_wrapper.py +167 -0
- benchtop-0.1.0/src/wrappers/tellurium_wrapper.py +52 -0
benchtop-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 Jonah Riley Huggins
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
benchtop-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,236 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: Benchtop
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: A tool for running simulation experiments formated as PEtab Problems.
|
|
5
|
+
Author-email: Jonah Huggins <JonahRileyHuggins@gmail.com>, Marc Birtwistle <marc.birtwistle@gmail.com>
|
|
6
|
+
License: GPL-2.0
|
|
7
|
+
Requires-Python: >=3.11
|
|
8
|
+
Description-Content-Type: text/markdown
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Requires-Dist: numpy==2.2.4
|
|
11
|
+
Requires-Dist: pandas==2.3.1
|
|
12
|
+
Requires-Dist: pyyaml==6.0.2
|
|
13
|
+
Requires-Dist: tellurium==2.2.11.2
|
|
14
|
+
Provides-Extra: amici
|
|
15
|
+
Requires-Dist: amici==0.32.0; extra == "amici"
|
|
16
|
+
Provides-Extra: dev
|
|
17
|
+
Requires-Dist: pytest>=8.0; extra == "dev"
|
|
18
|
+
Dynamic: license-file
|
|
19
|
+
|
|
20
|
+
```bash
|
|
21
|
+
. ° ....... °
|
|
22
|
+
| .° | :. .: ( .° )
|
|
23
|
+
| ° | .' '. |° |
|
|
24
|
+
|° . | | | /° ° \
|
|
25
|
+
(______) `_____' (_______)
|
|
26
|
+
▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌
|
|
27
|
+
▄▄▄▄▄▄▄▄▄ ▄▄▄▄▄▄▄▄▄▄ ▄▄ ▄ ▄▄▄▄▄▄▄▄▄▄ ▄ ▄ ▄▄▄▄▄▄▄▄▄▄ ▄▄▄▄▄▄▄▄▄▄ ▄▄▄▄▄▄▄▄▄▄
|
|
28
|
+
▐░░░░░░░░░▌ ▐░░░░░░░░░░▌▐░░▌ ▐░▌▐░░░░░░░░░░▌▐░▌ ▐░▌▐░░░░░░░░░░▌▐░░░░░░░░░░▌▐░░░░░░░░░░▌
|
|
29
|
+
▐░█▀▀▀▀▀▀█░▌▐░█▀▀▀▀▀▀▀▀ ▐░▌░▌ ▐░▌▐░█▀▀▀▀▀▀▀▀ ▐░▌ ▐░▌ ▀▀▀▀█░█▀▀▀ ▐░█▀▀▀▀▀▀█░▌▐░█▀▀▀▀▀▀█░▌
|
|
30
|
+
▐░█▄▄▄▄▄▄█░▌▐░█▄▄▄▄▄▄▄▄ ▐░▌ ▐░▌ ▐░▌▐░▌ ▐░█▄▄▄▄▄▄█░▌ ▐░▌ ▐░▌ ▐░▌▐░█▄▄▄▄▄▄█░▌
|
|
31
|
+
▐░░░░░░░░░▌ ▐░░░░░░░░░░▌▐░▌ ▐░▌ ▐░▌▐░▌ ▐░░░░░░░░░░▌ ▐░▌ ▐░▌ ▐░▌▐░░░░░░░░░░▌
|
|
32
|
+
▐░█▀▀▀▀▀▀█░▌▐░█▀▀▀▀▀▀▀▀ ▐░▌ ▐░▌ ▐░▌▐░▌ ▐░█▀▀▀▀▀▀█░▌ ▐░▌ ▐░▌ ▐░▌▐░█▀▀▀▀▀▀▀▀
|
|
33
|
+
▐░█▄▄▄▄▄▄█░▌▐░█▄▄▄▄▄▄▄▄ ▐░▌ ▐░▐░▌▐░█▄▄▄▄▄▄▄▄ ▐░▌ ▐░▌ ▐░▌ ▐░█▄▄▄▄▄▄█░▌▐░▌
|
|
34
|
+
▐░░░░░░░░░▌ ▐░░░░░░░░░░▌▐░▌ ▐░░▌▐░░░░░░░░░░▌▐░▌ ▐░▌ ▐░▌ ▐░░░░░░░░░░▌▐░▌
|
|
35
|
+
▀▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀▀▀▀▀ ▀ ▀▀ ▀▀▀▀▀▀▀▀▀▀ ▀ ▀ ▀ ▀▀▀▀▀▀▀▀▀▀ ▀
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
# Benchtop
|
|
39
|
+
|
|
40
|
+
**Parallel in-silico experiments for biological models**
|
|
41
|
+
|
|
42
|
+
Benchtop runs simulation benchmarks defined in a [PEtab](https://petab.readthedocs.io/)-inspired format. Given a YAML configuration and companion TSV/SBML files, it schedules conditions in dependency order, simulates across multiple CPU cores, caches intermediate results, and computes observables for model–data comparison.
|
|
43
|
+
|
|
44
|
+
## Features
|
|
45
|
+
|
|
46
|
+
- **PEtab-style benchmarks** — Define experiments with YAML plus standard TSV files for conditions, measurements, observables, and parameters.
|
|
47
|
+
- **Parallel execution** — Distributes simulation tasks across processes using a round-robin worker pool.
|
|
48
|
+
- **Preequilibration support** — Topologically sorts conditions so preequilibration runs before dependent simulations.
|
|
49
|
+
- **Single-cell replication** — Run multiple stochastic replicates per condition via `cell_count`.
|
|
50
|
+
- **Result caching** — Persists per-simulation pickle files and a cache index; supports resuming incomplete runs.
|
|
51
|
+
- **Observable calculation** — Evaluates observable formulas from simulation trajectories and aligns them with experimental measurements.
|
|
52
|
+
- **Pluggable simulators** — Built-in Tellurium wrapper; optional AMICI support via extras.
|
|
53
|
+
|
|
54
|
+
## How it works
|
|
55
|
+
|
|
56
|
+
```mermaid
|
|
57
|
+
flowchart LR
|
|
58
|
+
YAML[Benchmark YAML] --> Loader[FileLoader]
|
|
59
|
+
Loader --> Exp[Experiment]
|
|
60
|
+
Exp --> Org[Organizer]
|
|
61
|
+
Org -->|topological task order| Pool[Worker pool]
|
|
62
|
+
Pool --> Sim[Simulator wrapper]
|
|
63
|
+
Sim --> Cache[ResultCache]
|
|
64
|
+
Cache --> Obs[ObservableCalculator]
|
|
65
|
+
Obs --> Results[results/*.pkl]
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
1. **FileLoader** reads the benchmark YAML and loads PEtab TSV/SBML files into memory.
|
|
69
|
+
2. **Organizer** builds a dependency-aware task list (conditions × cell replicates) and assigns work to workers.
|
|
70
|
+
3. **Workers** load the SBML model, apply condition parameters, run preequilibration when needed, and simulate.
|
|
71
|
+
4. **ResultCache** stores raw trajectories as pickle files alongside a JSON index.
|
|
72
|
+
5. **ObservableCalculator** downsamples trajectories, evaluates observable formulas, and packages results next to experimental data.
|
|
73
|
+
|
|
74
|
+
## Requirements
|
|
75
|
+
|
|
76
|
+
- Python ≥ 3.12
|
|
77
|
+
- See [pyproject.toml](pyproject.toml) for pinned dependencies
|
|
78
|
+
|
|
79
|
+
## Installation
|
|
80
|
+
|
|
81
|
+
From the repository root:
|
|
82
|
+
|
|
83
|
+
```bash
|
|
84
|
+
pip install -e .
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
For AMICI support (optional):
|
|
88
|
+
|
|
89
|
+
```bash
|
|
90
|
+
pip install -e ".[amici]"
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
This installs the `benchtop` CLI entry point.
|
|
94
|
+
|
|
95
|
+
## Quick start
|
|
96
|
+
|
|
97
|
+
### Command line
|
|
98
|
+
|
|
99
|
+
Run a single benchmark:
|
|
100
|
+
|
|
101
|
+
```bash
|
|
102
|
+
benchtop experiment -p src/tests/data/LR-benchmark.yaml -s tellurium -c 4
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
Run all benchmarks in a directory:
|
|
106
|
+
|
|
107
|
+
```bash
|
|
108
|
+
benchtop experiment --run_all path/to/benchmarks/ -s tellurium
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
Useful flags:
|
|
112
|
+
|
|
113
|
+
| Flag | Description |
|
|
114
|
+
|------|-------------|
|
|
115
|
+
| `-p`, `--path` | Path to benchmark YAML |
|
|
116
|
+
| `-s`, `--simulator` | Simulator backend (`tellurium` or `amici`) |
|
|
117
|
+
| `-c`, `--cores` | Number of parallel processes (default: CPU count) |
|
|
118
|
+
| `--cache_dir` | Directory for simulation cache (default: `./.cache`) |
|
|
119
|
+
| `--load_index` | Resume from an existing cache index |
|
|
120
|
+
| `--No_Observables` | Skip observable calculation after simulation |
|
|
121
|
+
| `-v`, `--verbose` | Enable debug logging |
|
|
122
|
+
|
|
123
|
+
### Python API
|
|
124
|
+
|
|
125
|
+
```python
|
|
126
|
+
from benchtop.experiment import Experiment
|
|
127
|
+
|
|
128
|
+
experiment = Experiment(
|
|
129
|
+
"src/tests/data/LR-benchmark.yaml",
|
|
130
|
+
cores=4,
|
|
131
|
+
cache_dir="./.cache",
|
|
132
|
+
verbose=True,
|
|
133
|
+
)
|
|
134
|
+
|
|
135
|
+
experiment.run("tellurium")
|
|
136
|
+
experiment.observable_calculation()
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
You can also pass a custom simulator class that extends `AbstractSimulator` instead of a registry name.
|
|
140
|
+
|
|
141
|
+
A worked example is available in [demo/in-silico-experiment-demo.ipynb](demo/in-silico-experiment-demo.ipynb).
|
|
142
|
+
|
|
143
|
+
## Benchmark configuration
|
|
144
|
+
|
|
145
|
+
A benchmark is a directory containing a YAML file that references PEtab-style data files. Example ([src/tests/data/LR-benchmark.yaml](src/tests/data/LR-benchmark.yaml)):
|
|
146
|
+
|
|
147
|
+
```yaml
|
|
148
|
+
format_version: 1
|
|
149
|
+
parameter_file: LR Model - Parameters.tsv
|
|
150
|
+
problems:
|
|
151
|
+
- name: test-benchmark
|
|
152
|
+
condition_files:
|
|
153
|
+
- conditions.tsv
|
|
154
|
+
measurement_files:
|
|
155
|
+
- measurements.tsv
|
|
156
|
+
observable_files:
|
|
157
|
+
- observables.tsv
|
|
158
|
+
sbml_files:
|
|
159
|
+
- LR-model.xml
|
|
160
|
+
cell_count: 3
|
|
161
|
+
```
|
|
162
|
+
**Note**: The `cell_count` attribute is only applicable to `Benchtop` and specifies the number of replicates to simulate per-condition.
|
|
163
|
+
|
|
164
|
+
### Data files
|
|
165
|
+
|
|
166
|
+
| File | Purpose |
|
|
167
|
+
|------|---------|
|
|
168
|
+
| **Parameter TSV** | Model parameters (`parameterId`, `parameterScale`, `nominalValue`, …) |
|
|
169
|
+
| **conditions.tsv** | Simulation conditions — maps `conditionId` to parameter overrides |
|
|
170
|
+
| **measurements.tsv** | Links observables to conditions, optional preequilibration, and measurement timepoints |
|
|
171
|
+
| **observables.tsv** | Observable definitions (`observableId`, `observableFormula`, `noiseFormula`) |
|
|
172
|
+
| **SBML** | One or more model files (`.xml` or `.sbml`) |
|
|
173
|
+
|
|
174
|
+
When `preequilibrationConditionId` is set in `measurements.tsv`, Benchtop runs the preequilibration condition first and uses its final state as the initial condition for the simulation.
|
|
175
|
+
|
|
176
|
+
### Results
|
|
177
|
+
|
|
178
|
+
After observable calculation, results are written as a pickle file in a `results/` subdirectory next to the benchmark YAML (named after the problem or today's date). Each entry in the results dictionary contains:
|
|
179
|
+
|
|
180
|
+
- `conditionId` and `cell` replicate index
|
|
181
|
+
- Per-observable `experiment`, `simulation`, and `time` arrays
|
|
182
|
+
|
|
183
|
+
## Simulators
|
|
184
|
+
|
|
185
|
+
Benchtop uses a small registry of simulator wrappers in [src/wrappers/](src/wrappers/). Each wrapper implements `AbstractSimulator` with `load`, `modify`, and `simulate` methods.
|
|
186
|
+
|
|
187
|
+
| Name | Class | Install |
|
|
188
|
+
|------|-------|---------|
|
|
189
|
+
| `tellurium` | `TelluriumSimulator` | Included by default |
|
|
190
|
+
| `amici` | `AmiciSimulator` | `pip install -e ".[amici]"` |
|
|
191
|
+
|
|
192
|
+
To add a new backend, create a wrapper under `src/wrappers/` and register it in [src/benchtop/registry.py](src/benchtop/registry.py).
|
|
193
|
+
|
|
194
|
+
## Project structure
|
|
195
|
+
|
|
196
|
+
```
|
|
197
|
+
Benchtop/
|
|
198
|
+
├── demo/ # Jupyter notebook walkthrough
|
|
199
|
+
├── src/
|
|
200
|
+
│ ├── benchtop/ # Core library
|
|
201
|
+
│ │ ├── experiment.py # Main experiment orchestrator
|
|
202
|
+
│ │ ├── _organizer.py # Task scheduling and topological sort
|
|
203
|
+
│ │ ├── _worker.py # Per-simulation worker logic
|
|
204
|
+
│ │ ├── _record.py # Results dictionary management
|
|
205
|
+
│ │ ├── _results_cacher.py # Pickle cache and index
|
|
206
|
+
│ │ ├── _observable_calculator.py
|
|
207
|
+
│ │ ├── _abstract_simulator.py # Simulator interface
|
|
208
|
+
│ │ ├── file_loader.py # YAML and TSV loading
|
|
209
|
+
│ │ ├── registry.py # Simulator lookup
|
|
210
|
+
│ │ ├── launcher.py # CLI entry point
|
|
211
|
+
│ │ └── arguments.py # CLI argument parsing
|
|
212
|
+
│ ├── wrappers/ # Simulator backends
|
|
213
|
+
│ └── tests/ # Test suite and example data
|
|
214
|
+
├── pyproject.toml
|
|
215
|
+
└── README.md
|
|
216
|
+
```
|
|
217
|
+
|
|
218
|
+
## Running tests
|
|
219
|
+
|
|
220
|
+
From the repository root:
|
|
221
|
+
|
|
222
|
+
```bash
|
|
223
|
+
cd src/tests
|
|
224
|
+
python run_tests.py
|
|
225
|
+
```
|
|
226
|
+
|
|
227
|
+
The test suite uses the LR receptor benchmark in [src/tests/data/](src/tests/data/) and exercises experiment execution, caching, workers, organizers, records, and observable calculation.
|
|
228
|
+
|
|
229
|
+
## Authors
|
|
230
|
+
|
|
231
|
+
- Jonah Huggins — [JonahRileyHuggins@gmail.com](mailto:JonahRileyHuggins@gmail.com)
|
|
232
|
+
- Marc Birtwistle — [marc.birtwistle@gmail.com](mailto:marc.birtwistle@gmail.com)
|
|
233
|
+
|
|
234
|
+
## License
|
|
235
|
+
|
|
236
|
+
See [LICENSE](LICENSE).
|
benchtop-0.1.0/README.md
ADDED
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
```bash
|
|
2
|
+
. ° ....... °
|
|
3
|
+
| .° | :. .: ( .° )
|
|
4
|
+
| ° | .' '. |° |
|
|
5
|
+
|° . | | | /° ° \
|
|
6
|
+
(______) `_____' (_______)
|
|
7
|
+
▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌
|
|
8
|
+
▄▄▄▄▄▄▄▄▄ ▄▄▄▄▄▄▄▄▄▄ ▄▄ ▄ ▄▄▄▄▄▄▄▄▄▄ ▄ ▄ ▄▄▄▄▄▄▄▄▄▄ ▄▄▄▄▄▄▄▄▄▄ ▄▄▄▄▄▄▄▄▄▄
|
|
9
|
+
▐░░░░░░░░░▌ ▐░░░░░░░░░░▌▐░░▌ ▐░▌▐░░░░░░░░░░▌▐░▌ ▐░▌▐░░░░░░░░░░▌▐░░░░░░░░░░▌▐░░░░░░░░░░▌
|
|
10
|
+
▐░█▀▀▀▀▀▀█░▌▐░█▀▀▀▀▀▀▀▀ ▐░▌░▌ ▐░▌▐░█▀▀▀▀▀▀▀▀ ▐░▌ ▐░▌ ▀▀▀▀█░█▀▀▀ ▐░█▀▀▀▀▀▀█░▌▐░█▀▀▀▀▀▀█░▌
|
|
11
|
+
▐░█▄▄▄▄▄▄█░▌▐░█▄▄▄▄▄▄▄▄ ▐░▌ ▐░▌ ▐░▌▐░▌ ▐░█▄▄▄▄▄▄█░▌ ▐░▌ ▐░▌ ▐░▌▐░█▄▄▄▄▄▄█░▌
|
|
12
|
+
▐░░░░░░░░░▌ ▐░░░░░░░░░░▌▐░▌ ▐░▌ ▐░▌▐░▌ ▐░░░░░░░░░░▌ ▐░▌ ▐░▌ ▐░▌▐░░░░░░░░░░▌
|
|
13
|
+
▐░█▀▀▀▀▀▀█░▌▐░█▀▀▀▀▀▀▀▀ ▐░▌ ▐░▌ ▐░▌▐░▌ ▐░█▀▀▀▀▀▀█░▌ ▐░▌ ▐░▌ ▐░▌▐░█▀▀▀▀▀▀▀▀
|
|
14
|
+
▐░█▄▄▄▄▄▄█░▌▐░█▄▄▄▄▄▄▄▄ ▐░▌ ▐░▐░▌▐░█▄▄▄▄▄▄▄▄ ▐░▌ ▐░▌ ▐░▌ ▐░█▄▄▄▄▄▄█░▌▐░▌
|
|
15
|
+
▐░░░░░░░░░▌ ▐░░░░░░░░░░▌▐░▌ ▐░░▌▐░░░░░░░░░░▌▐░▌ ▐░▌ ▐░▌ ▐░░░░░░░░░░▌▐░▌
|
|
16
|
+
▀▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀▀▀▀▀ ▀ ▀▀ ▀▀▀▀▀▀▀▀▀▀ ▀ ▀ ▀ ▀▀▀▀▀▀▀▀▀▀ ▀
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
# Benchtop
|
|
20
|
+
|
|
21
|
+
**Parallel in-silico experiments for biological models**
|
|
22
|
+
|
|
23
|
+
Benchtop runs simulation benchmarks defined in a [PEtab](https://petab.readthedocs.io/)-inspired format. Given a YAML configuration and companion TSV/SBML files, it schedules conditions in dependency order, simulates across multiple CPU cores, caches intermediate results, and computes observables for model–data comparison.
|
|
24
|
+
|
|
25
|
+
## Features
|
|
26
|
+
|
|
27
|
+
- **PEtab-style benchmarks** — Define experiments with YAML plus standard TSV files for conditions, measurements, observables, and parameters.
|
|
28
|
+
- **Parallel execution** — Distributes simulation tasks across processes using a round-robin worker pool.
|
|
29
|
+
- **Preequilibration support** — Topologically sorts conditions so preequilibration runs before dependent simulations.
|
|
30
|
+
- **Single-cell replication** — Run multiple stochastic replicates per condition via `cell_count`.
|
|
31
|
+
- **Result caching** — Persists per-simulation pickle files and a cache index; supports resuming incomplete runs.
|
|
32
|
+
- **Observable calculation** — Evaluates observable formulas from simulation trajectories and aligns them with experimental measurements.
|
|
33
|
+
- **Pluggable simulators** — Built-in Tellurium wrapper; optional AMICI support via extras.
|
|
34
|
+
|
|
35
|
+
## How it works
|
|
36
|
+
|
|
37
|
+
```mermaid
|
|
38
|
+
flowchart LR
|
|
39
|
+
YAML[Benchmark YAML] --> Loader[FileLoader]
|
|
40
|
+
Loader --> Exp[Experiment]
|
|
41
|
+
Exp --> Org[Organizer]
|
|
42
|
+
Org -->|topological task order| Pool[Worker pool]
|
|
43
|
+
Pool --> Sim[Simulator wrapper]
|
|
44
|
+
Sim --> Cache[ResultCache]
|
|
45
|
+
Cache --> Obs[ObservableCalculator]
|
|
46
|
+
Obs --> Results[results/*.pkl]
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
1. **FileLoader** reads the benchmark YAML and loads PEtab TSV/SBML files into memory.
|
|
50
|
+
2. **Organizer** builds a dependency-aware task list (conditions × cell replicates) and assigns work to workers.
|
|
51
|
+
3. **Workers** load the SBML model, apply condition parameters, run preequilibration when needed, and simulate.
|
|
52
|
+
4. **ResultCache** stores raw trajectories as pickle files alongside a JSON index.
|
|
53
|
+
5. **ObservableCalculator** downsamples trajectories, evaluates observable formulas, and packages results next to experimental data.
|
|
54
|
+
|
|
55
|
+
## Requirements
|
|
56
|
+
|
|
57
|
+
- Python ≥ 3.12
|
|
58
|
+
- See [pyproject.toml](pyproject.toml) for pinned dependencies
|
|
59
|
+
|
|
60
|
+
## Installation
|
|
61
|
+
|
|
62
|
+
From the repository root:
|
|
63
|
+
|
|
64
|
+
```bash
|
|
65
|
+
pip install -e .
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
For AMICI support (optional):
|
|
69
|
+
|
|
70
|
+
```bash
|
|
71
|
+
pip install -e ".[amici]"
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
This installs the `benchtop` CLI entry point.
|
|
75
|
+
|
|
76
|
+
## Quick start
|
|
77
|
+
|
|
78
|
+
### Command line
|
|
79
|
+
|
|
80
|
+
Run a single benchmark:
|
|
81
|
+
|
|
82
|
+
```bash
|
|
83
|
+
benchtop experiment -p src/tests/data/LR-benchmark.yaml -s tellurium -c 4
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
Run all benchmarks in a directory:
|
|
87
|
+
|
|
88
|
+
```bash
|
|
89
|
+
benchtop experiment --run_all path/to/benchmarks/ -s tellurium
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
Useful flags:
|
|
93
|
+
|
|
94
|
+
| Flag | Description |
|
|
95
|
+
|------|-------------|
|
|
96
|
+
| `-p`, `--path` | Path to benchmark YAML |
|
|
97
|
+
| `-s`, `--simulator` | Simulator backend (`tellurium` or `amici`) |
|
|
98
|
+
| `-c`, `--cores` | Number of parallel processes (default: CPU count) |
|
|
99
|
+
| `--cache_dir` | Directory for simulation cache (default: `./.cache`) |
|
|
100
|
+
| `--load_index` | Resume from an existing cache index |
|
|
101
|
+
| `--No_Observables` | Skip observable calculation after simulation |
|
|
102
|
+
| `-v`, `--verbose` | Enable debug logging |
|
|
103
|
+
|
|
104
|
+
### Python API
|
|
105
|
+
|
|
106
|
+
```python
|
|
107
|
+
from benchtop.experiment import Experiment
|
|
108
|
+
|
|
109
|
+
experiment = Experiment(
|
|
110
|
+
"src/tests/data/LR-benchmark.yaml",
|
|
111
|
+
cores=4,
|
|
112
|
+
cache_dir="./.cache",
|
|
113
|
+
verbose=True,
|
|
114
|
+
)
|
|
115
|
+
|
|
116
|
+
experiment.run("tellurium")
|
|
117
|
+
experiment.observable_calculation()
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
You can also pass a custom simulator class that extends `AbstractSimulator` instead of a registry name.
|
|
121
|
+
|
|
122
|
+
A worked example is available in [demo/in-silico-experiment-demo.ipynb](demo/in-silico-experiment-demo.ipynb).
|
|
123
|
+
|
|
124
|
+
## Benchmark configuration
|
|
125
|
+
|
|
126
|
+
A benchmark is a directory containing a YAML file that references PEtab-style data files. Example ([src/tests/data/LR-benchmark.yaml](src/tests/data/LR-benchmark.yaml)):
|
|
127
|
+
|
|
128
|
+
```yaml
|
|
129
|
+
format_version: 1
|
|
130
|
+
parameter_file: LR Model - Parameters.tsv
|
|
131
|
+
problems:
|
|
132
|
+
- name: test-benchmark
|
|
133
|
+
condition_files:
|
|
134
|
+
- conditions.tsv
|
|
135
|
+
measurement_files:
|
|
136
|
+
- measurements.tsv
|
|
137
|
+
observable_files:
|
|
138
|
+
- observables.tsv
|
|
139
|
+
sbml_files:
|
|
140
|
+
- LR-model.xml
|
|
141
|
+
cell_count: 3
|
|
142
|
+
```
|
|
143
|
+
**Note**: The `cell_count` attribute is only applicable to `Benchtop` and specifies the number of replicates to simulate per-condition.
|
|
144
|
+
|
|
145
|
+
### Data files
|
|
146
|
+
|
|
147
|
+
| File | Purpose |
|
|
148
|
+
|------|---------|
|
|
149
|
+
| **Parameter TSV** | Model parameters (`parameterId`, `parameterScale`, `nominalValue`, …) |
|
|
150
|
+
| **conditions.tsv** | Simulation conditions — maps `conditionId` to parameter overrides |
|
|
151
|
+
| **measurements.tsv** | Links observables to conditions, optional preequilibration, and measurement timepoints |
|
|
152
|
+
| **observables.tsv** | Observable definitions (`observableId`, `observableFormula`, `noiseFormula`) |
|
|
153
|
+
| **SBML** | One or more model files (`.xml` or `.sbml`) |
|
|
154
|
+
|
|
155
|
+
When `preequilibrationConditionId` is set in `measurements.tsv`, Benchtop runs the preequilibration condition first and uses its final state as the initial condition for the simulation.
|
|
156
|
+
|
|
157
|
+
### Results
|
|
158
|
+
|
|
159
|
+
After observable calculation, results are written as a pickle file in a `results/` subdirectory next to the benchmark YAML (named after the problem or today's date). Each entry in the results dictionary contains:
|
|
160
|
+
|
|
161
|
+
- `conditionId` and `cell` replicate index
|
|
162
|
+
- Per-observable `experiment`, `simulation`, and `time` arrays
|
|
163
|
+
|
|
164
|
+
## Simulators
|
|
165
|
+
|
|
166
|
+
Benchtop uses a small registry of simulator wrappers in [src/wrappers/](src/wrappers/). Each wrapper implements `AbstractSimulator` with `load`, `modify`, and `simulate` methods.
|
|
167
|
+
|
|
168
|
+
| Name | Class | Install |
|
|
169
|
+
|------|-------|---------|
|
|
170
|
+
| `tellurium` | `TelluriumSimulator` | Included by default |
|
|
171
|
+
| `amici` | `AmiciSimulator` | `pip install -e ".[amici]"` |
|
|
172
|
+
|
|
173
|
+
To add a new backend, create a wrapper under `src/wrappers/` and register it in [src/benchtop/registry.py](src/benchtop/registry.py).
|
|
174
|
+
|
|
175
|
+
## Project structure
|
|
176
|
+
|
|
177
|
+
```
|
|
178
|
+
Benchtop/
|
|
179
|
+
├── demo/ # Jupyter notebook walkthrough
|
|
180
|
+
├── src/
|
|
181
|
+
│ ├── benchtop/ # Core library
|
|
182
|
+
│ │ ├── experiment.py # Main experiment orchestrator
|
|
183
|
+
│ │ ├── _organizer.py # Task scheduling and topological sort
|
|
184
|
+
│ │ ├── _worker.py # Per-simulation worker logic
|
|
185
|
+
│ │ ├── _record.py # Results dictionary management
|
|
186
|
+
│ │ ├── _results_cacher.py # Pickle cache and index
|
|
187
|
+
│ │ ├── _observable_calculator.py
|
|
188
|
+
│ │ ├── _abstract_simulator.py # Simulator interface
|
|
189
|
+
│ │ ├── file_loader.py # YAML and TSV loading
|
|
190
|
+
│ │ ├── registry.py # Simulator lookup
|
|
191
|
+
│ │ ├── launcher.py # CLI entry point
|
|
192
|
+
│ │ └── arguments.py # CLI argument parsing
|
|
193
|
+
│ ├── wrappers/ # Simulator backends
|
|
194
|
+
│ └── tests/ # Test suite and example data
|
|
195
|
+
├── pyproject.toml
|
|
196
|
+
└── README.md
|
|
197
|
+
```
|
|
198
|
+
|
|
199
|
+
## Running tests
|
|
200
|
+
|
|
201
|
+
From the repository root:
|
|
202
|
+
|
|
203
|
+
```bash
|
|
204
|
+
cd src/tests
|
|
205
|
+
python run_tests.py
|
|
206
|
+
```
|
|
207
|
+
|
|
208
|
+
The test suite uses the LR receptor benchmark in [src/tests/data/](src/tests/data/) and exercises experiment execution, caching, workers, organizers, records, and observable calculation.
|
|
209
|
+
|
|
210
|
+
## Authors
|
|
211
|
+
|
|
212
|
+
- Jonah Huggins — [JonahRileyHuggins@gmail.com](mailto:JonahRileyHuggins@gmail.com)
|
|
213
|
+
- Marc Birtwistle — [marc.birtwistle@gmail.com](mailto:marc.birtwistle@gmail.com)
|
|
214
|
+
|
|
215
|
+
## License
|
|
216
|
+
|
|
217
|
+
See [LICENSE](LICENSE).
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=61"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "Benchtop"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "A tool for running simulation experiments formated as PEtab Problems."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.11"
|
|
11
|
+
|
|
12
|
+
authors = [
|
|
13
|
+
{ name = "Jonah Huggins", email = "JonahRileyHuggins@gmail.com" },
|
|
14
|
+
{ name = "Marc Birtwistle", email = "marc.birtwistle@gmail.com" },
|
|
15
|
+
]
|
|
16
|
+
|
|
17
|
+
license = {text = "GPL-2.0"}
|
|
18
|
+
|
|
19
|
+
dependencies = [
|
|
20
|
+
"numpy==2.2.4",
|
|
21
|
+
"pandas==2.3.1",
|
|
22
|
+
"pyyaml==6.0.2",
|
|
23
|
+
"tellurium==2.2.11.2"
|
|
24
|
+
]
|
|
25
|
+
|
|
26
|
+
[project.optional-dependencies]
|
|
27
|
+
amici = [
|
|
28
|
+
"amici==0.32.0"
|
|
29
|
+
]
|
|
30
|
+
dev = [
|
|
31
|
+
"pytest>=8.0",
|
|
32
|
+
]
|
|
33
|
+
|
|
34
|
+
[project.scripts]
|
|
35
|
+
benchtop = "benchtop.launcher:main"
|
|
36
|
+
|
|
37
|
+
[tool.setuptools]
|
|
38
|
+
package-dir = {"" = "src"}
|
|
39
|
+
|
|
40
|
+
[tool.setuptools.packages.find]
|
|
41
|
+
where = ["src"]
|
|
42
|
+
|
|
43
|
+
[tool.pytest.ini_options]
|
|
44
|
+
testpaths = ["src/tests"]
|
|
45
|
+
pythonpath = ["src"]
|
|
46
|
+
filterwarnings = [
|
|
47
|
+
"ignore::DeprecationWarning",
|
|
48
|
+
]
|
benchtop-0.1.0/setup.cfg
ADDED