Benchtop 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. benchtop-0.1.0/LICENSE +21 -0
  2. benchtop-0.1.0/PKG-INFO +236 -0
  3. benchtop-0.1.0/README.md +217 -0
  4. benchtop-0.1.0/pyproject.toml +48 -0
  5. benchtop-0.1.0/setup.cfg +4 -0
  6. benchtop-0.1.0/src/Benchtop.egg-info/PKG-INFO +236 -0
  7. benchtop-0.1.0/src/Benchtop.egg-info/SOURCES.txt +37 -0
  8. benchtop-0.1.0/src/Benchtop.egg-info/dependency_links.txt +1 -0
  9. benchtop-0.1.0/src/Benchtop.egg-info/entry_points.txt +2 -0
  10. benchtop-0.1.0/src/Benchtop.egg-info/requires.txt +10 -0
  11. benchtop-0.1.0/src/Benchtop.egg-info/top_level.txt +3 -0
  12. benchtop-0.1.0/src/benchtop/__init__.py +6 -0
  13. benchtop-0.1.0/src/benchtop/_abstract_simulator.py +24 -0
  14. benchtop-0.1.0/src/benchtop/_observable_calculator.py +190 -0
  15. benchtop-0.1.0/src/benchtop/_organizer.py +147 -0
  16. benchtop-0.1.0/src/benchtop/_record.py +204 -0
  17. benchtop-0.1.0/src/benchtop/_results_cacher.py +137 -0
  18. benchtop-0.1.0/src/benchtop/_worker.py +186 -0
  19. benchtop-0.1.0/src/benchtop/arguments.py +99 -0
  20. benchtop-0.1.0/src/benchtop/experiment.py +394 -0
  21. benchtop-0.1.0/src/benchtop/file_loader.py +141 -0
  22. benchtop-0.1.0/src/benchtop/launcher.py +79 -0
  23. benchtop-0.1.0/src/benchtop/registry.py +28 -0
  24. benchtop-0.1.0/src/tests/conftest.py +40 -0
  25. benchtop-0.1.0/src/tests/data/plot-model.py +0 -0
  26. benchtop-0.1.0/src/tests/make_dummy.py +30 -0
  27. benchtop-0.1.0/src/tests/run_tests.py +11 -0
  28. benchtop-0.1.0/src/tests/test_benchtop.py +207 -0
  29. benchtop-0.1.0/src/tests/test_cache.py +166 -0
  30. benchtop-0.1.0/src/tests/test_obscalc.py +174 -0
  31. benchtop-0.1.0/src/tests/test_organizer.py +148 -0
  32. benchtop-0.1.0/src/tests/test_record.py +134 -0
  33. benchtop-0.1.0/src/tests/test_sparced_wrapper.py +18 -0
  34. benchtop-0.1.0/src/tests/test_worker.py +186 -0
  35. benchtop-0.1.0/src/wrappers/SingleCell.py +122 -0
  36. benchtop-0.1.0/src/wrappers/amici_wrapper.py +91 -0
  37. benchtop-0.1.0/src/wrappers/bngsim_wrapper.py +50 -0
  38. benchtop-0.1.0/src/wrappers/sparced_wrapper.py +167 -0
  39. benchtop-0.1.0/src/wrappers/tellurium_wrapper.py +52 -0
benchtop-0.1.0/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2025 Jonah Riley Huggins
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,236 @@
1
+ Metadata-Version: 2.4
2
+ Name: Benchtop
3
+ Version: 0.1.0
4
+ Summary: A tool for running simulation experiments formated as PEtab Problems.
5
+ Author-email: Jonah Huggins <JonahRileyHuggins@gmail.com>, Marc Birtwistle <marc.birtwistle@gmail.com>
6
+ License: GPL-2.0
7
+ Requires-Python: >=3.11
8
+ Description-Content-Type: text/markdown
9
+ License-File: LICENSE
10
+ Requires-Dist: numpy==2.2.4
11
+ Requires-Dist: pandas==2.3.1
12
+ Requires-Dist: pyyaml==6.0.2
13
+ Requires-Dist: tellurium==2.2.11.2
14
+ Provides-Extra: amici
15
+ Requires-Dist: amici==0.32.0; extra == "amici"
16
+ Provides-Extra: dev
17
+ Requires-Dist: pytest>=8.0; extra == "dev"
18
+ Dynamic: license-file
19
+
20
+ ```bash
21
+ . ° ....... °
22
+ | .° | :. .: ( .° )
23
+ | ° | .' '. |° |
24
+ |° . | | | /° ° \
25
+ (______) `_____' (_______)
26
+ ▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌
27
+ ▄▄▄▄▄▄▄▄▄ ▄▄▄▄▄▄▄▄▄▄ ▄▄ ▄ ▄▄▄▄▄▄▄▄▄▄ ▄ ▄ ▄▄▄▄▄▄▄▄▄▄ ▄▄▄▄▄▄▄▄▄▄ ▄▄▄▄▄▄▄▄▄▄
28
+ ▐░░░░░░░░░▌ ▐░░░░░░░░░░▌▐░░▌ ▐░▌▐░░░░░░░░░░▌▐░▌ ▐░▌▐░░░░░░░░░░▌▐░░░░░░░░░░▌▐░░░░░░░░░░▌
29
+ ▐░█▀▀▀▀▀▀█░▌▐░█▀▀▀▀▀▀▀▀ ▐░▌░▌ ▐░▌▐░█▀▀▀▀▀▀▀▀ ▐░▌ ▐░▌ ▀▀▀▀█░█▀▀▀ ▐░█▀▀▀▀▀▀█░▌▐░█▀▀▀▀▀▀█░▌
30
+ ▐░█▄▄▄▄▄▄█░▌▐░█▄▄▄▄▄▄▄▄ ▐░▌ ▐░▌ ▐░▌▐░▌ ▐░█▄▄▄▄▄▄█░▌ ▐░▌ ▐░▌ ▐░▌▐░█▄▄▄▄▄▄█░▌
31
+ ▐░░░░░░░░░▌ ▐░░░░░░░░░░▌▐░▌ ▐░▌ ▐░▌▐░▌ ▐░░░░░░░░░░▌ ▐░▌ ▐░▌ ▐░▌▐░░░░░░░░░░▌
32
+ ▐░█▀▀▀▀▀▀█░▌▐░█▀▀▀▀▀▀▀▀ ▐░▌ ▐░▌ ▐░▌▐░▌ ▐░█▀▀▀▀▀▀█░▌ ▐░▌ ▐░▌ ▐░▌▐░█▀▀▀▀▀▀▀▀
33
+ ▐░█▄▄▄▄▄▄█░▌▐░█▄▄▄▄▄▄▄▄ ▐░▌ ▐░▐░▌▐░█▄▄▄▄▄▄▄▄ ▐░▌ ▐░▌ ▐░▌ ▐░█▄▄▄▄▄▄█░▌▐░▌
34
+ ▐░░░░░░░░░▌ ▐░░░░░░░░░░▌▐░▌ ▐░░▌▐░░░░░░░░░░▌▐░▌ ▐░▌ ▐░▌ ▐░░░░░░░░░░▌▐░▌
35
+ ▀▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀▀▀▀▀ ▀ ▀▀ ▀▀▀▀▀▀▀▀▀▀ ▀ ▀ ▀ ▀▀▀▀▀▀▀▀▀▀ ▀
36
+ ```
37
+
38
+ # Benchtop
39
+
40
+ **Parallel in-silico experiments for biological models**
41
+
42
+ Benchtop runs simulation benchmarks defined in a [PEtab](https://petab.readthedocs.io/)-inspired format. Given a YAML configuration and companion TSV/SBML files, it schedules conditions in dependency order, simulates across multiple CPU cores, caches intermediate results, and computes observables for model–data comparison.
43
+
44
+ ## Features
45
+
46
+ - **PEtab-style benchmarks** — Define experiments with YAML plus standard TSV files for conditions, measurements, observables, and parameters.
47
+ - **Parallel execution** — Distributes simulation tasks across processes using a round-robin worker pool.
48
+ - **Preequilibration support** — Topologically sorts conditions so preequilibration runs before dependent simulations.
49
+ - **Single-cell replication** — Run multiple stochastic replicates per condition via `cell_count`.
50
+ - **Result caching** — Persists per-simulation pickle files and a cache index; supports resuming incomplete runs.
51
+ - **Observable calculation** — Evaluates observable formulas from simulation trajectories and aligns them with experimental measurements.
52
+ - **Pluggable simulators** — Built-in Tellurium wrapper; optional AMICI support via extras.
53
+
54
+ ## How it works
55
+
56
+ ```mermaid
57
+ flowchart LR
58
+ YAML[Benchmark YAML] --> Loader[FileLoader]
59
+ Loader --> Exp[Experiment]
60
+ Exp --> Org[Organizer]
61
+ Org -->|topological task order| Pool[Worker pool]
62
+ Pool --> Sim[Simulator wrapper]
63
+ Sim --> Cache[ResultCache]
64
+ Cache --> Obs[ObservableCalculator]
65
+ Obs --> Results[results/*.pkl]
66
+ ```
67
+
68
+ 1. **FileLoader** reads the benchmark YAML and loads PEtab TSV/SBML files into memory.
69
+ 2. **Organizer** builds a dependency-aware task list (conditions × cell replicates) and assigns work to workers.
70
+ 3. **Workers** load the SBML model, apply condition parameters, run preequilibration when needed, and simulate.
71
+ 4. **ResultCache** stores raw trajectories as pickle files alongside a JSON index.
72
+ 5. **ObservableCalculator** downsamples trajectories, evaluates observable formulas, and packages results next to experimental data.
73
+
74
+ ## Requirements
75
+
76
+ - Python ≥ 3.12
77
+ - See [pyproject.toml](pyproject.toml) for pinned dependencies
78
+
79
+ ## Installation
80
+
81
+ From the repository root:
82
+
83
+ ```bash
84
+ pip install -e .
85
+ ```
86
+
87
+ For AMICI support (optional):
88
+
89
+ ```bash
90
+ pip install -e ".[amici]"
91
+ ```
92
+
93
+ This installs the `benchtop` CLI entry point.
94
+
95
+ ## Quick start
96
+
97
+ ### Command line
98
+
99
+ Run a single benchmark:
100
+
101
+ ```bash
102
+ benchtop experiment -p src/tests/data/LR-benchmark.yaml -s tellurium -c 4
103
+ ```
104
+
105
+ Run all benchmarks in a directory:
106
+
107
+ ```bash
108
+ benchtop experiment --run_all path/to/benchmarks/ -s tellurium
109
+ ```
110
+
111
+ Useful flags:
112
+
113
+ | Flag | Description |
114
+ |------|-------------|
115
+ | `-p`, `--path` | Path to benchmark YAML |
116
+ | `-s`, `--simulator` | Simulator backend (`tellurium` or `amici`) |
117
+ | `-c`, `--cores` | Number of parallel processes (default: CPU count) |
118
+ | `--cache_dir` | Directory for simulation cache (default: `./.cache`) |
119
+ | `--load_index` | Resume from an existing cache index |
120
+ | `--No_Observables` | Skip observable calculation after simulation |
121
+ | `-v`, `--verbose` | Enable debug logging |
122
+
123
+ ### Python API
124
+
125
+ ```python
126
+ from benchtop.experiment import Experiment
127
+
128
+ experiment = Experiment(
129
+ "src/tests/data/LR-benchmark.yaml",
130
+ cores=4,
131
+ cache_dir="./.cache",
132
+ verbose=True,
133
+ )
134
+
135
+ experiment.run("tellurium")
136
+ experiment.observable_calculation()
137
+ ```
138
+
139
+ You can also pass a custom simulator class that extends `AbstractSimulator` instead of a registry name.
140
+
141
+ A worked example is available in [demo/in-silico-experiment-demo.ipynb](demo/in-silico-experiment-demo.ipynb).
142
+
143
+ ## Benchmark configuration
144
+
145
+ A benchmark is a directory containing a YAML file that references PEtab-style data files. Example ([src/tests/data/LR-benchmark.yaml](src/tests/data/LR-benchmark.yaml)):
146
+
147
+ ```yaml
148
+ format_version: 1
149
+ parameter_file: LR Model - Parameters.tsv
150
+ problems:
151
+ - name: test-benchmark
152
+ condition_files:
153
+ - conditions.tsv
154
+ measurement_files:
155
+ - measurements.tsv
156
+ observable_files:
157
+ - observables.tsv
158
+ sbml_files:
159
+ - LR-model.xml
160
+ cell_count: 3
161
+ ```
162
+ **Note**: The `cell_count` attribute is only applicable to `Benchtop` and specifies the number of replicates to simulate per-condition.
163
+
164
+ ### Data files
165
+
166
+ | File | Purpose |
167
+ |------|---------|
168
+ | **Parameter TSV** | Model parameters (`parameterId`, `parameterScale`, `nominalValue`, …) |
169
+ | **conditions.tsv** | Simulation conditions — maps `conditionId` to parameter overrides |
170
+ | **measurements.tsv** | Links observables to conditions, optional preequilibration, and measurement timepoints |
171
+ | **observables.tsv** | Observable definitions (`observableId`, `observableFormula`, `noiseFormula`) |
172
+ | **SBML** | One or more model files (`.xml` or `.sbml`) |
173
+
174
+ When `preequilibrationConditionId` is set in `measurements.tsv`, Benchtop runs the preequilibration condition first and uses its final state as the initial condition for the simulation.
175
+
176
+ ### Results
177
+
178
+ After observable calculation, results are written as a pickle file in a `results/` subdirectory next to the benchmark YAML (named after the problem or today's date). Each entry in the results dictionary contains:
179
+
180
+ - `conditionId` and `cell` replicate index
181
+ - Per-observable `experiment`, `simulation`, and `time` arrays
182
+
183
+ ## Simulators
184
+
185
+ Benchtop uses a small registry of simulator wrappers in [src/wrappers/](src/wrappers/). Each wrapper implements `AbstractSimulator` with `load`, `modify`, and `simulate` methods.
186
+
187
+ | Name | Class | Install |
188
+ |------|-------|---------|
189
+ | `tellurium` | `TelluriumSimulator` | Included by default |
190
+ | `amici` | `AmiciSimulator` | `pip install -e ".[amici]"` |
191
+
192
+ To add a new backend, create a wrapper under `src/wrappers/` and register it in [src/benchtop/registry.py](src/benchtop/registry.py).
193
+
194
+ ## Project structure
195
+
196
+ ```
197
+ Benchtop/
198
+ ├── demo/ # Jupyter notebook walkthrough
199
+ ├── src/
200
+ │ ├── benchtop/ # Core library
201
+ │ │ ├── experiment.py # Main experiment orchestrator
202
+ │ │ ├── _organizer.py # Task scheduling and topological sort
203
+ │ │ ├── _worker.py # Per-simulation worker logic
204
+ │ │ ├── _record.py # Results dictionary management
205
+ │ │ ├── _results_cacher.py # Pickle cache and index
206
+ │ │ ├── _observable_calculator.py
207
+ │ │ ├── _abstract_simulator.py # Simulator interface
208
+ │ │ ├── file_loader.py # YAML and TSV loading
209
+ │ │ ├── registry.py # Simulator lookup
210
+ │ │ ├── launcher.py # CLI entry point
211
+ │ │ └── arguments.py # CLI argument parsing
212
+ │ ├── wrappers/ # Simulator backends
213
+ │ └── tests/ # Test suite and example data
214
+ ├── pyproject.toml
215
+ └── README.md
216
+ ```
217
+
218
+ ## Running tests
219
+
220
+ From the repository root:
221
+
222
+ ```bash
223
+ cd src/tests
224
+ python run_tests.py
225
+ ```
226
+
227
+ The test suite uses the LR receptor benchmark in [src/tests/data/](src/tests/data/) and exercises experiment execution, caching, workers, organizers, records, and observable calculation.
228
+
229
+ ## Authors
230
+
231
+ - Jonah Huggins — [JonahRileyHuggins@gmail.com](mailto:JonahRileyHuggins@gmail.com)
232
+ - Marc Birtwistle — [marc.birtwistle@gmail.com](mailto:marc.birtwistle@gmail.com)
233
+
234
+ ## License
235
+
236
+ See [LICENSE](LICENSE).
@@ -0,0 +1,217 @@
1
+ ```bash
2
+ . ° ....... °
3
+ | .° | :. .: ( .° )
4
+ | ° | .' '. |° |
5
+ |° . | | | /° ° \
6
+ (______) `_____' (_______)
7
+ ▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌▌
8
+ ▄▄▄▄▄▄▄▄▄ ▄▄▄▄▄▄▄▄▄▄ ▄▄ ▄ ▄▄▄▄▄▄▄▄▄▄ ▄ ▄ ▄▄▄▄▄▄▄▄▄▄ ▄▄▄▄▄▄▄▄▄▄ ▄▄▄▄▄▄▄▄▄▄
9
+ ▐░░░░░░░░░▌ ▐░░░░░░░░░░▌▐░░▌ ▐░▌▐░░░░░░░░░░▌▐░▌ ▐░▌▐░░░░░░░░░░▌▐░░░░░░░░░░▌▐░░░░░░░░░░▌
10
+ ▐░█▀▀▀▀▀▀█░▌▐░█▀▀▀▀▀▀▀▀ ▐░▌░▌ ▐░▌▐░█▀▀▀▀▀▀▀▀ ▐░▌ ▐░▌ ▀▀▀▀█░█▀▀▀ ▐░█▀▀▀▀▀▀█░▌▐░█▀▀▀▀▀▀█░▌
11
+ ▐░█▄▄▄▄▄▄█░▌▐░█▄▄▄▄▄▄▄▄ ▐░▌ ▐░▌ ▐░▌▐░▌ ▐░█▄▄▄▄▄▄█░▌ ▐░▌ ▐░▌ ▐░▌▐░█▄▄▄▄▄▄█░▌
12
+ ▐░░░░░░░░░▌ ▐░░░░░░░░░░▌▐░▌ ▐░▌ ▐░▌▐░▌ ▐░░░░░░░░░░▌ ▐░▌ ▐░▌ ▐░▌▐░░░░░░░░░░▌
13
+ ▐░█▀▀▀▀▀▀█░▌▐░█▀▀▀▀▀▀▀▀ ▐░▌ ▐░▌ ▐░▌▐░▌ ▐░█▀▀▀▀▀▀█░▌ ▐░▌ ▐░▌ ▐░▌▐░█▀▀▀▀▀▀▀▀
14
+ ▐░█▄▄▄▄▄▄█░▌▐░█▄▄▄▄▄▄▄▄ ▐░▌ ▐░▐░▌▐░█▄▄▄▄▄▄▄▄ ▐░▌ ▐░▌ ▐░▌ ▐░█▄▄▄▄▄▄█░▌▐░▌
15
+ ▐░░░░░░░░░▌ ▐░░░░░░░░░░▌▐░▌ ▐░░▌▐░░░░░░░░░░▌▐░▌ ▐░▌ ▐░▌ ▐░░░░░░░░░░▌▐░▌
16
+ ▀▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀▀▀▀▀ ▀ ▀▀ ▀▀▀▀▀▀▀▀▀▀ ▀ ▀ ▀ ▀▀▀▀▀▀▀▀▀▀ ▀
17
+ ```
18
+
19
+ # Benchtop
20
+
21
+ **Parallel in-silico experiments for biological models**
22
+
23
+ Benchtop runs simulation benchmarks defined in a [PEtab](https://petab.readthedocs.io/)-inspired format. Given a YAML configuration and companion TSV/SBML files, it schedules conditions in dependency order, simulates across multiple CPU cores, caches intermediate results, and computes observables for model–data comparison.
24
+
25
+ ## Features
26
+
27
+ - **PEtab-style benchmarks** — Define experiments with YAML plus standard TSV files for conditions, measurements, observables, and parameters.
28
+ - **Parallel execution** — Distributes simulation tasks across processes using a round-robin worker pool.
29
+ - **Preequilibration support** — Topologically sorts conditions so preequilibration runs before dependent simulations.
30
+ - **Single-cell replication** — Run multiple stochastic replicates per condition via `cell_count`.
31
+ - **Result caching** — Persists per-simulation pickle files and a cache index; supports resuming incomplete runs.
32
+ - **Observable calculation** — Evaluates observable formulas from simulation trajectories and aligns them with experimental measurements.
33
+ - **Pluggable simulators** — Built-in Tellurium wrapper; optional AMICI support via extras.
34
+
35
+ ## How it works
36
+
37
+ ```mermaid
38
+ flowchart LR
39
+ YAML[Benchmark YAML] --> Loader[FileLoader]
40
+ Loader --> Exp[Experiment]
41
+ Exp --> Org[Organizer]
42
+ Org -->|topological task order| Pool[Worker pool]
43
+ Pool --> Sim[Simulator wrapper]
44
+ Sim --> Cache[ResultCache]
45
+ Cache --> Obs[ObservableCalculator]
46
+ Obs --> Results[results/*.pkl]
47
+ ```
48
+
49
+ 1. **FileLoader** reads the benchmark YAML and loads PEtab TSV/SBML files into memory.
50
+ 2. **Organizer** builds a dependency-aware task list (conditions × cell replicates) and assigns work to workers.
51
+ 3. **Workers** load the SBML model, apply condition parameters, run preequilibration when needed, and simulate.
52
+ 4. **ResultCache** stores raw trajectories as pickle files alongside a JSON index.
53
+ 5. **ObservableCalculator** downsamples trajectories, evaluates observable formulas, and packages results next to experimental data.
54
+
55
+ ## Requirements
56
+
57
+ - Python ≥ 3.12
58
+ - See [pyproject.toml](pyproject.toml) for pinned dependencies
59
+
60
+ ## Installation
61
+
62
+ From the repository root:
63
+
64
+ ```bash
65
+ pip install -e .
66
+ ```
67
+
68
+ For AMICI support (optional):
69
+
70
+ ```bash
71
+ pip install -e ".[amici]"
72
+ ```
73
+
74
+ This installs the `benchtop` CLI entry point.
75
+
76
+ ## Quick start
77
+
78
+ ### Command line
79
+
80
+ Run a single benchmark:
81
+
82
+ ```bash
83
+ benchtop experiment -p src/tests/data/LR-benchmark.yaml -s tellurium -c 4
84
+ ```
85
+
86
+ Run all benchmarks in a directory:
87
+
88
+ ```bash
89
+ benchtop experiment --run_all path/to/benchmarks/ -s tellurium
90
+ ```
91
+
92
+ Useful flags:
93
+
94
+ | Flag | Description |
95
+ |------|-------------|
96
+ | `-p`, `--path` | Path to benchmark YAML |
97
+ | `-s`, `--simulator` | Simulator backend (`tellurium` or `amici`) |
98
+ | `-c`, `--cores` | Number of parallel processes (default: CPU count) |
99
+ | `--cache_dir` | Directory for simulation cache (default: `./.cache`) |
100
+ | `--load_index` | Resume from an existing cache index |
101
+ | `--No_Observables` | Skip observable calculation after simulation |
102
+ | `-v`, `--verbose` | Enable debug logging |
103
+
104
+ ### Python API
105
+
106
+ ```python
107
+ from benchtop.experiment import Experiment
108
+
109
+ experiment = Experiment(
110
+ "src/tests/data/LR-benchmark.yaml",
111
+ cores=4,
112
+ cache_dir="./.cache",
113
+ verbose=True,
114
+ )
115
+
116
+ experiment.run("tellurium")
117
+ experiment.observable_calculation()
118
+ ```
119
+
120
+ You can also pass a custom simulator class that extends `AbstractSimulator` instead of a registry name.
121
+
122
+ A worked example is available in [demo/in-silico-experiment-demo.ipynb](demo/in-silico-experiment-demo.ipynb).
123
+
124
+ ## Benchmark configuration
125
+
126
+ A benchmark is a directory containing a YAML file that references PEtab-style data files. Example ([src/tests/data/LR-benchmark.yaml](src/tests/data/LR-benchmark.yaml)):
127
+
128
+ ```yaml
129
+ format_version: 1
130
+ parameter_file: LR Model - Parameters.tsv
131
+ problems:
132
+ - name: test-benchmark
133
+ condition_files:
134
+ - conditions.tsv
135
+ measurement_files:
136
+ - measurements.tsv
137
+ observable_files:
138
+ - observables.tsv
139
+ sbml_files:
140
+ - LR-model.xml
141
+ cell_count: 3
142
+ ```
143
+ **Note**: The `cell_count` attribute is only applicable to `Benchtop` and specifies the number of replicates to simulate per-condition.
144
+
145
+ ### Data files
146
+
147
+ | File | Purpose |
148
+ |------|---------|
149
+ | **Parameter TSV** | Model parameters (`parameterId`, `parameterScale`, `nominalValue`, …) |
150
+ | **conditions.tsv** | Simulation conditions — maps `conditionId` to parameter overrides |
151
+ | **measurements.tsv** | Links observables to conditions, optional preequilibration, and measurement timepoints |
152
+ | **observables.tsv** | Observable definitions (`observableId`, `observableFormula`, `noiseFormula`) |
153
+ | **SBML** | One or more model files (`.xml` or `.sbml`) |
154
+
155
+ When `preequilibrationConditionId` is set in `measurements.tsv`, Benchtop runs the preequilibration condition first and uses its final state as the initial condition for the simulation.
156
+
157
+ ### Results
158
+
159
+ After observable calculation, results are written as a pickle file in a `results/` subdirectory next to the benchmark YAML (named after the problem or today's date). Each entry in the results dictionary contains:
160
+
161
+ - `conditionId` and `cell` replicate index
162
+ - Per-observable `experiment`, `simulation`, and `time` arrays
163
+
164
+ ## Simulators
165
+
166
+ Benchtop uses a small registry of simulator wrappers in [src/wrappers/](src/wrappers/). Each wrapper implements `AbstractSimulator` with `load`, `modify`, and `simulate` methods.
167
+
168
+ | Name | Class | Install |
169
+ |------|-------|---------|
170
+ | `tellurium` | `TelluriumSimulator` | Included by default |
171
+ | `amici` | `AmiciSimulator` | `pip install -e ".[amici]"` |
172
+
173
+ To add a new backend, create a wrapper under `src/wrappers/` and register it in [src/benchtop/registry.py](src/benchtop/registry.py).
174
+
175
+ ## Project structure
176
+
177
+ ```
178
+ Benchtop/
179
+ ├── demo/ # Jupyter notebook walkthrough
180
+ ├── src/
181
+ │ ├── benchtop/ # Core library
182
+ │ │ ├── experiment.py # Main experiment orchestrator
183
+ │ │ ├── _organizer.py # Task scheduling and topological sort
184
+ │ │ ├── _worker.py # Per-simulation worker logic
185
+ │ │ ├── _record.py # Results dictionary management
186
+ │ │ ├── _results_cacher.py # Pickle cache and index
187
+ │ │ ├── _observable_calculator.py
188
+ │ │ ├── _abstract_simulator.py # Simulator interface
189
+ │ │ ├── file_loader.py # YAML and TSV loading
190
+ │ │ ├── registry.py # Simulator lookup
191
+ │ │ ├── launcher.py # CLI entry point
192
+ │ │ └── arguments.py # CLI argument parsing
193
+ │ ├── wrappers/ # Simulator backends
194
+ │ └── tests/ # Test suite and example data
195
+ ├── pyproject.toml
196
+ └── README.md
197
+ ```
198
+
199
+ ## Running tests
200
+
201
+ From the repository root:
202
+
203
+ ```bash
204
+ cd src/tests
205
+ python run_tests.py
206
+ ```
207
+
208
+ The test suite uses the LR receptor benchmark in [src/tests/data/](src/tests/data/) and exercises experiment execution, caching, workers, organizers, records, and observable calculation.
209
+
210
+ ## Authors
211
+
212
+ - Jonah Huggins — [JonahRileyHuggins@gmail.com](mailto:JonahRileyHuggins@gmail.com)
213
+ - Marc Birtwistle — [marc.birtwistle@gmail.com](mailto:marc.birtwistle@gmail.com)
214
+
215
+ ## License
216
+
217
+ See [LICENSE](LICENSE).
@@ -0,0 +1,48 @@
1
+ [build-system]
2
+ requires = ["setuptools>=61"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "Benchtop"
7
+ version = "0.1.0"
8
+ description = "A tool for running simulation experiments formated as PEtab Problems."
9
+ readme = "README.md"
10
+ requires-python = ">=3.11"
11
+
12
+ authors = [
13
+ { name = "Jonah Huggins", email = "JonahRileyHuggins@gmail.com" },
14
+ { name = "Marc Birtwistle", email = "marc.birtwistle@gmail.com" },
15
+ ]
16
+
17
+ license = {text = "GPL-2.0"}
18
+
19
+ dependencies = [
20
+ "numpy==2.2.4",
21
+ "pandas==2.3.1",
22
+ "pyyaml==6.0.2",
23
+ "tellurium==2.2.11.2"
24
+ ]
25
+
26
+ [project.optional-dependencies]
27
+ amici = [
28
+ "amici==0.32.0"
29
+ ]
30
+ dev = [
31
+ "pytest>=8.0",
32
+ ]
33
+
34
+ [project.scripts]
35
+ benchtop = "benchtop.launcher:main"
36
+
37
+ [tool.setuptools]
38
+ package-dir = {"" = "src"}
39
+
40
+ [tool.setuptools.packages.find]
41
+ where = ["src"]
42
+
43
+ [tool.pytest.ini_options]
44
+ testpaths = ["src/tests"]
45
+ pythonpath = ["src"]
46
+ filterwarnings = [
47
+ "ignore::DeprecationWarning",
48
+ ]
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+