bend2 1.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- bend2-1.0.0/CHANGELOG.md +13 -0
- bend2-1.0.0/CITATION.cff +42 -0
- bend2-1.0.0/CONTRIBUTING.md +49 -0
- bend2-1.0.0/LICENSE +21 -0
- bend2-1.0.0/MANIFEST.in +5 -0
- bend2-1.0.0/PKG-INFO +145 -0
- bend2-1.0.0/README-pypi.md +100 -0
- bend2-1.0.0/README.md +181 -0
- bend2-1.0.0/checks/test_workflow.py +137 -0
- bend2-1.0.0/docs/assets/capabilities-narrow-dark.svg +38 -0
- bend2-1.0.0/docs/assets/capabilities-narrow-light.svg +38 -0
- bend2-1.0.0/docs/assets/capabilities-wide-dark.svg +38 -0
- bend2-1.0.0/docs/assets/capabilities-wide-light.svg +38 -0
- bend2-1.0.0/docs/assets/readme-banner.png +0 -0
- bend2-1.0.0/docs/assets/results-dark.svg +346 -0
- bend2-1.0.0/docs/assets/results-light.svg +346 -0
- bend2-1.0.0/docs/data.md +107 -0
- bend2-1.0.0/docs/models.md +154 -0
- bend2-1.0.0/docs/releasing.md +64 -0
- bend2-1.0.0/docs/reviewing.md +95 -0
- bend2-1.0.0/docs/submitting.md +102 -0
- bend2-1.0.0/docs/tasks.md +112 -0
- bend2-1.0.0/pyproject.toml +57 -0
- bend2-1.0.0/setup.cfg +4 -0
- bend2-1.0.0/src/bend2/__init__.py +12 -0
- bend2-1.0.0/src/bend2/__main__.py +3 -0
- bend2-1.0.0/src/bend2/_combine.py +191 -0
- bend2-1.0.0/src/bend2/_data/atlas-candidate-activities.parquet +0 -0
- bend2-1.0.0/src/bend2/_data/atlas-population.parquet +0 -0
- bend2-1.0.0/src/bend2/_data/atlas-programs.csv +17 -0
- bend2-1.0.0/src/bend2/_data/atlas-source-activities.parquet +0 -0
- bend2-1.0.0/src/bend2/_data/gtex-genes.tsv +19283 -0
- bend2-1.0.0/src/bend2/_data/gtex-targets.npz +0 -0
- bend2-1.0.0/src/bend2/_data/prior-panel.parquet +0 -0
- bend2-1.0.0/src/bend2/_data/prior-query-seeds.json +2063 -0
- bend2-1.0.0/src/bend2/_data/siraj-numeric-reference.npz +0 -0
- bend2-1.0.0/src/bend2/_data/siraj_selective.parquet +0 -0
- bend2-1.0.0/src/bend2/_examples/custom.py +26 -0
- bend2-1.0.0/src/bend2/_examples/dnabert2.py +10 -0
- bend2-1.0.0/src/bend2/_examples/huggingface.py +17 -0
- bend2-1.0.0/src/bend2/_examples/ntv3_prior.py +12 -0
- bend2-1.0.0/src/bend2/_linear.py +141 -0
- bend2-1.0.0/src/bend2/_rescoring/__init__.py +1 -0
- bend2-1.0.0/src/bend2/_rescoring/interventions.py +257 -0
- bend2-1.0.0/src/bend2/_rescoring/panels.py +272 -0
- bend2-1.0.0/src/bend2/_tasks/__init__.py +14 -0
- bend2-1.0.0/src/bend2/_tasks/_siraj.py +33 -0
- bend2-1.0.0/src/bend2/_tasks/atlas_profile.py +189 -0
- bend2-1.0.0/src/bend2/_tasks/background.py +56 -0
- bend2-1.0.0/src/bend2/_tasks/cell_selective.py +113 -0
- bend2-1.0.0/src/bend2/_tasks/context.py +28 -0
- bend2-1.0.0/src/bend2/_tasks/opensplice.py +33 -0
- bend2-1.0.0/src/bend2/_tasks/prior.py +90 -0
- bend2-1.0.0/src/bend2/_tasks/repair_percentile.py +72 -0
- bend2-1.0.0/src/bend2/_tasks/tissue_identification.py +182 -0
- bend2-1.0.0/src/bend2/assets.json +147 -0
- bend2-1.0.0/src/bend2/cache.py +212 -0
- bend2-1.0.0/src/bend2/catalog.py +52 -0
- bend2-1.0.0/src/bend2/cli.py +444 -0
- bend2-1.0.0/src/bend2/datasets.py +565 -0
- bend2-1.0.0/src/bend2/github.py +384 -0
- bend2-1.0.0/src/bend2/huggingface.py +183 -0
- bend2-1.0.0/src/bend2/model.py +215 -0
- bend2-1.0.0/src/bend2/planning.py +251 -0
- bend2-1.0.0/src/bend2/readouts.py +180 -0
- bend2-1.0.0/src/bend2/reference_models.py +433 -0
- bend2-1.0.0/src/bend2/results.py +186 -0
- bend2-1.0.0/src/bend2/reviews.py +365 -0
- bend2-1.0.0/src/bend2/runner.py +362 -0
- bend2-1.0.0/src/bend2/submissions.py +632 -0
- bend2-1.0.0/src/bend2.egg-info/PKG-INFO +145 -0
- bend2-1.0.0/src/bend2.egg-info/SOURCES.txt +74 -0
- bend2-1.0.0/src/bend2.egg-info/dependency_links.txt +1 -0
- bend2-1.0.0/src/bend2.egg-info/entry_points.txt +2 -0
- bend2-1.0.0/src/bend2.egg-info/requires.txt +24 -0
- bend2-1.0.0/src/bend2.egg-info/top_level.txt +1 -0
bend2-1.0.0/CHANGELOG.md
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## 1.0.0 — 2026-11-02
|
|
4
|
+
|
|
5
|
+
Initial standalone release of BEND 2.0.
|
|
6
|
+
|
|
7
|
+
- Eight evaluation tasks spanning sequence priors, functional prediction,
|
|
8
|
+
biological context, genetic interactions and genomic design.
|
|
9
|
+
- A Python model adapter, resumable inference and task-specific fitted readouts.
|
|
10
|
+
- Versioned datasets with size and SHA-256 verification, plus a bundled Atlas
|
|
11
|
+
example that runs without downloading data or model weights.
|
|
12
|
+
- Results and prediction exports, contribution validation and maintainer rescoring.
|
|
13
|
+
- An interactive leaderboard with reference results and task explanations.
|
bend2-1.0.0/CITATION.cff
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
cff-version: 1.2.0
|
|
2
|
+
message: "If you use BEND 2.0, cite the benchmark and acknowledge the datasets used by your selected tasks."
|
|
3
|
+
type: software
|
|
4
|
+
title: "BEND 2.0"
|
|
5
|
+
version: "1.0.0"
|
|
6
|
+
date-released: 2026-11-02
|
|
7
|
+
abstract: "A benchmark of genomic models across sequence priors, functional prediction, biological context, genetic interactions, and genomic design."
|
|
8
|
+
url: "https://machinelearninglifescience.github.io/BEND2/"
|
|
9
|
+
authors:
|
|
10
|
+
- name: "BEND 2.0 contributors"
|
|
11
|
+
preferred-citation:
|
|
12
|
+
type: generic
|
|
13
|
+
title: "BEND 2.0: Benchmarking the Capabilities Needed to Design DNA"
|
|
14
|
+
authors:
|
|
15
|
+
- family-names: "Bjerregaard"
|
|
16
|
+
given-names: "Andreas"
|
|
17
|
+
- family-names: "Brilli"
|
|
18
|
+
given-names: "Dionysia Danai"
|
|
19
|
+
- family-names: "DeVries"
|
|
20
|
+
given-names: "Rachael"
|
|
21
|
+
- family-names: "Antoniadis"
|
|
22
|
+
given-names: "Panagiotis"
|
|
23
|
+
- family-names: "Li"
|
|
24
|
+
given-names: "Yan"
|
|
25
|
+
- family-names: "Krogh"
|
|
26
|
+
given-names: "Anders"
|
|
27
|
+
- family-names: "Boomsma"
|
|
28
|
+
given-names: "Wouter"
|
|
29
|
+
- family-names: "Winther"
|
|
30
|
+
given-names: "Ole"
|
|
31
|
+
- family-names: "Marin"
|
|
32
|
+
given-names: "Frederikke Isa"
|
|
33
|
+
year: 2026
|
|
34
|
+
repository-code: "https://github.com/MachineLearningLifeScience/BEND2"
|
|
35
|
+
license: MIT
|
|
36
|
+
keywords:
|
|
37
|
+
- genomics
|
|
38
|
+
- DNA
|
|
39
|
+
- machine learning
|
|
40
|
+
- benchmarking
|
|
41
|
+
- foundation models
|
|
42
|
+
- genomic design
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
# Contributing to BEND 2.0
|
|
2
|
+
|
|
3
|
+
For leaderboard submissions, see the [submission guide](docs/submitting.md).
|
|
4
|
+
For code changes, describe the problem and how to reproduce it in your issue
|
|
5
|
+
or pull request.
|
|
6
|
+
|
|
7
|
+
## Development
|
|
8
|
+
|
|
9
|
+
```bash
|
|
10
|
+
git clone --single-branch --branch main https://github.com/MachineLearningLifeScience/BEND2.git
|
|
11
|
+
cd BEND2
|
|
12
|
+
python -m venv .venv
|
|
13
|
+
source .venv/bin/activate
|
|
14
|
+
python -m pip install -e .
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
Keep model frameworks and weights in optional adapters. The core package
|
|
18
|
+
should run on a CPU without cluster tools or private data.
|
|
19
|
+
Examples live in `src/bend2/_examples/`. `bend2 init` copies them from the
|
|
20
|
+
installed package. Use portable paths and load weights on first inference.
|
|
21
|
+
|
|
22
|
+
Keep automated checks few and focused on evaluation and evidence integrity.
|
|
23
|
+
Exercise behavior changes with a small example. Build and install the wheel
|
|
24
|
+
when changing package contents, and check documented commands against the code.
|
|
25
|
+
|
|
26
|
+
Use short, factual prose. Comments should explain constraints or non-obvious
|
|
27
|
+
decisions; omit comments that restate the code.
|
|
28
|
+
|
|
29
|
+
## Preserve the benchmark definition
|
|
30
|
+
|
|
31
|
+
Define each task's scoring in its task module, shared by evaluation and evidence
|
|
32
|
+
review. Review code independently validates the cohort, labels and submitted
|
|
33
|
+
evidence before accepting the shared scorer's result.
|
|
34
|
+
|
|
35
|
+
Increment the affected entries in `TASK_VERSIONS` in `src/bend2/catalog.py` when
|
|
36
|
+
changing a task's inputs, splits, readout, selection rules or scores. A shared
|
|
37
|
+
readout change needs an increment for every affected task. Formatting, comments
|
|
38
|
+
and refactors that preserve results keep the same version. Explain whether
|
|
39
|
+
results remain comparable in the pull request.
|
|
40
|
+
|
|
41
|
+
Keep experimental labels out of model adapters. Use predictions to select candidates
|
|
42
|
+
before reading their measured outcomes. Record checkpoints, training data and
|
|
43
|
+
source attribution.
|
|
44
|
+
|
|
45
|
+
Develop on branches based on `main`. The historical paper-review snapshot
|
|
46
|
+
remains in the original research repository.
|
|
47
|
+
|
|
48
|
+
Keep generated runs, checkpoints, job scripts, manuscripts and experiment
|
|
49
|
+
archives outside the repository. Link to them when needed for review.
|
bend2-1.0.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 The Authors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
bend2-1.0.0/MANIFEST.in
ADDED
bend2-1.0.0/PKG-INFO
ADDED
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: bend2
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: BEND 2.0 — benchmark the capabilities needed to design DNA
|
|
5
|
+
Author: BEND 2.0 contributors
|
|
6
|
+
License-Expression: MIT AND CC-BY-4.0 AND CC-BY-NC-4.0
|
|
7
|
+
Project-URL: Homepage, https://machinelearninglifescience.github.io/BEND2/
|
|
8
|
+
Project-URL: Repository, https://github.com/MachineLearningLifeScience/BEND2
|
|
9
|
+
Project-URL: Documentation, https://github.com/MachineLearningLifeScience/BEND2/blob/main/README.md
|
|
10
|
+
Project-URL: Issues, https://github.com/MachineLearningLifeScience/BEND2/issues
|
|
11
|
+
Project-URL: Data, https://huggingface.co/datasets/yhsure/BEND2
|
|
12
|
+
Project-URL: Changelog, https://github.com/MachineLearningLifeScience/BEND2/releases
|
|
13
|
+
Keywords: genomics,DNA,benchmark,foundation models,sequence design,gene regulation
|
|
14
|
+
Classifier: Environment :: Console
|
|
15
|
+
Classifier: Intended Audience :: Science/Research
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
18
|
+
Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
|
|
19
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
20
|
+
Requires-Python: >=3.11
|
|
21
|
+
Description-Content-Type: text/markdown
|
|
22
|
+
License-File: LICENSE
|
|
23
|
+
License-File: docs/data.md
|
|
24
|
+
Requires-Dist: numpy>=2.0
|
|
25
|
+
Requires-Dist: pandas>=2.2
|
|
26
|
+
Requires-Dist: pyarrow>=17
|
|
27
|
+
Requires-Dist: scipy>=1.13
|
|
28
|
+
Requires-Dist: scikit-learn>=1.5
|
|
29
|
+
Requires-Dist: filelock>=3.16
|
|
30
|
+
Provides-Extra: genome
|
|
31
|
+
Requires-Dist: py2bit>=1.0.1; extra == "genome"
|
|
32
|
+
Provides-Extra: huggingface
|
|
33
|
+
Requires-Dist: torch>=2.7; extra == "huggingface"
|
|
34
|
+
Requires-Dist: transformers<5,>=4.57; extra == "huggingface"
|
|
35
|
+
Provides-Extra: reference
|
|
36
|
+
Requires-Dist: torch>=2.7; extra == "reference"
|
|
37
|
+
Requires-Dist: transformers<5,>=4.57; extra == "reference"
|
|
38
|
+
Requires-Dist: einops>=0.8; extra == "reference"
|
|
39
|
+
Provides-Extra: dev
|
|
40
|
+
Requires-Dist: build>=1.2; extra == "dev"
|
|
41
|
+
Requires-Dist: twine>=6; extra == "dev"
|
|
42
|
+
Requires-Dist: pytest>=8; extra == "dev"
|
|
43
|
+
Requires-Dist: ruff>=0.9; extra == "dev"
|
|
44
|
+
Dynamic: license-file
|
|
45
|
+
|
|
46
|
+

|
|
47
|
+
|
|
48
|
+
We built BEND 2.0 to test whether genomic models can guide the design of DNA.
|
|
49
|
+
Across eight tasks, we examine how they predict sequence function, account for
|
|
50
|
+
cell type and genetic background, and choose sequences that produce a desired
|
|
51
|
+
response.
|
|
52
|
+
|
|
53
|
+

|
|
54
|
+
|
|
55
|
+
BEND prepares the data, fits linear readouts to your model's frozen embeddings,
|
|
56
|
+
and reports task scores and predictions. Python 3.11 or later is required.
|
|
57
|
+
Your model's framework and weights are installed separately.
|
|
58
|
+
|
|
59
|
+
```bash
|
|
60
|
+
python -m pip install bend2
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
[Benchmark data](https://huggingface.co/datasets/yhsure/BEND2)
|
|
64
|
+
|
|
65
|
+
The package is available now; the GitHub repository and online leaderboard are
|
|
66
|
+
awaiting their public launch. Full documentation is included in the source
|
|
67
|
+
distribution under **Download files** on this page.
|
|
68
|
+
|
|
69
|
+
## Evaluate your model
|
|
70
|
+
|
|
71
|
+
With BEND installed, save an adapter as `my_model.py`:
|
|
72
|
+
|
|
73
|
+
```python
|
|
74
|
+
import numpy as np
|
|
75
|
+
from bend2 import Model
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def embed(sequences: list[str]) -> np.ndarray:
|
|
79
|
+
return your_encoder(sequences) # [batch, features]
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
model = Model("My model", embed=embed, revision="checkpoint-v1")
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
Replace `your_encoder` with your model's inference. Return one finite,
|
|
86
|
+
fixed-width CPU NumPy vector per complete sequence, in input order, and record
|
|
87
|
+
the actual model revision.
|
|
88
|
+
|
|
89
|
+
Start with Atlas: 7,657 short sequences, sixteen multi-cell design patterns,
|
|
90
|
+
and data included with the package.
|
|
91
|
+
|
|
92
|
+
```bash
|
|
93
|
+
bend2 run my_model.py --tasks atlas_profile --output results/my-model
|
|
94
|
+
bend2 show results/my-model
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
To try the included base-composition baseline without model weights:
|
|
98
|
+
|
|
99
|
+
```bash
|
|
100
|
+
bend2 run composition --tasks atlas_profile --output results/composition
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
## Tasks
|
|
104
|
+
|
|
105
|
+
| Task | What it tests |
|
|
106
|
+
| --- | --- |
|
|
107
|
+
| `prior` | Recovering a hidden DNA base |
|
|
108
|
+
| `opensplice` | Predicting mutation effects on splicing |
|
|
109
|
+
| `context` | Predicting effect differences between cells |
|
|
110
|
+
| `tissue_identification` | Identifying tissues from expression predictions |
|
|
111
|
+
| `background` | Predicting interactions between mutations |
|
|
112
|
+
| `repair_percentile` | Choosing an edit that matches a reference response |
|
|
113
|
+
| `cell_selective` | Increasing expression in one cell while limiting changes in another |
|
|
114
|
+
| `atlas_profile` | Choosing enhancers for a desired pattern of activity across cells |
|
|
115
|
+
|
|
116
|
+
Seven tasks use embeddings; `prior` needs nucleotide probabilities. The `siraj`
|
|
117
|
+
task group runs `context`, `background`, `repair_percentile` and `cell_selective`
|
|
118
|
+
together, sharing inference work. Inspect requirements before a larger run:
|
|
119
|
+
|
|
120
|
+
```bash
|
|
121
|
+
bend2 plan my_model.py --tasks siraj
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
## Data and results
|
|
125
|
+
|
|
126
|
+
Small datasets ship with the package. Larger files download from the public
|
|
127
|
+
[BEND 2.0 data release](https://huggingface.co/datasets/yhsure/BEND2), pinned to a
|
|
128
|
+
fixed revision and checked against the original file sizes and checksums.
|
|
129
|
+
Source-specific terms apply to the data; see the
|
|
130
|
+
[data guide](https://huggingface.co/datasets/yhsure/BEND2/blob/main/FILES.md) and
|
|
131
|
+
[source terms](https://huggingface.co/datasets/yhsure/BEND2/blob/main/LICENSE.md).
|
|
132
|
+
|
|
133
|
+
Saved results can be opened without loading the model:
|
|
134
|
+
|
|
135
|
+
```python
|
|
136
|
+
from bend2 import Run
|
|
137
|
+
|
|
138
|
+
run = Run("results/my-model")
|
|
139
|
+
scores = run.to_dataframe()
|
|
140
|
+
predictions = run.table("atlas_profile")
|
|
141
|
+
```
|
|
142
|
+
|
|
143
|
+
Runs stay local until you choose to share them. Use `bend2 --help` for commands
|
|
144
|
+
and `bend2 tasks` for task requirements. The source distribution includes
|
|
145
|
+
`docs/submitting.md` for preparing a future leaderboard contribution.
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+

|
|
2
|
+
|
|
3
|
+
We built BEND 2.0 to test whether genomic models can guide the design of DNA.
|
|
4
|
+
Across eight tasks, we examine how they predict sequence function, account for
|
|
5
|
+
cell type and genetic background, and choose sequences that produce a desired
|
|
6
|
+
response.
|
|
7
|
+
|
|
8
|
+

|
|
9
|
+
|
|
10
|
+
BEND prepares the data, fits linear readouts to your model's frozen embeddings,
|
|
11
|
+
and reports task scores and predictions. Python 3.11 or later is required.
|
|
12
|
+
Your model's framework and weights are installed separately.
|
|
13
|
+
|
|
14
|
+
```bash
|
|
15
|
+
python -m pip install bend2
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
[Benchmark data](https://huggingface.co/datasets/yhsure/BEND2)
|
|
19
|
+
|
|
20
|
+
The package is available now; the GitHub repository and online leaderboard are
|
|
21
|
+
awaiting their public launch. Full documentation is included in the source
|
|
22
|
+
distribution under **Download files** on this page.
|
|
23
|
+
|
|
24
|
+
## Evaluate your model
|
|
25
|
+
|
|
26
|
+
With BEND installed, save an adapter as `my_model.py`:
|
|
27
|
+
|
|
28
|
+
```python
|
|
29
|
+
import numpy as np
|
|
30
|
+
from bend2 import Model
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def embed(sequences: list[str]) -> np.ndarray:
|
|
34
|
+
return your_encoder(sequences) # [batch, features]
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
model = Model("My model", embed=embed, revision="checkpoint-v1")
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
Replace `your_encoder` with your model's inference. Return one finite,
|
|
41
|
+
fixed-width CPU NumPy vector per complete sequence, in input order, and record
|
|
42
|
+
the actual model revision.
|
|
43
|
+
|
|
44
|
+
Start with Atlas: 7,657 short sequences, sixteen multi-cell design patterns,
|
|
45
|
+
and data included with the package.
|
|
46
|
+
|
|
47
|
+
```bash
|
|
48
|
+
bend2 run my_model.py --tasks atlas_profile --output results/my-model
|
|
49
|
+
bend2 show results/my-model
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
To try the included base-composition baseline without model weights:
|
|
53
|
+
|
|
54
|
+
```bash
|
|
55
|
+
bend2 run composition --tasks atlas_profile --output results/composition
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
## Tasks
|
|
59
|
+
|
|
60
|
+
| Task | What it tests |
|
|
61
|
+
| --- | --- |
|
|
62
|
+
| `prior` | Recovering a hidden DNA base |
|
|
63
|
+
| `opensplice` | Predicting mutation effects on splicing |
|
|
64
|
+
| `context` | Predicting effect differences between cells |
|
|
65
|
+
| `tissue_identification` | Identifying tissues from expression predictions |
|
|
66
|
+
| `background` | Predicting interactions between mutations |
|
|
67
|
+
| `repair_percentile` | Choosing an edit that matches a reference response |
|
|
68
|
+
| `cell_selective` | Increasing expression in one cell while limiting changes in another |
|
|
69
|
+
| `atlas_profile` | Choosing enhancers for a desired pattern of activity across cells |
|
|
70
|
+
|
|
71
|
+
Seven tasks use embeddings; `prior` needs nucleotide probabilities. The `siraj`
|
|
72
|
+
task group runs `context`, `background`, `repair_percentile` and `cell_selective`
|
|
73
|
+
together, sharing inference work. Inspect requirements before a larger run:
|
|
74
|
+
|
|
75
|
+
```bash
|
|
76
|
+
bend2 plan my_model.py --tasks siraj
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
## Data and results
|
|
80
|
+
|
|
81
|
+
Small datasets ship with the package. Larger files download from the public
|
|
82
|
+
[BEND 2.0 data release](https://huggingface.co/datasets/yhsure/BEND2), pinned to a
|
|
83
|
+
fixed revision and checked against the original file sizes and checksums.
|
|
84
|
+
Source-specific terms apply to the data; see the
|
|
85
|
+
[data guide](https://huggingface.co/datasets/yhsure/BEND2/blob/main/FILES.md) and
|
|
86
|
+
[source terms](https://huggingface.co/datasets/yhsure/BEND2/blob/main/LICENSE.md).
|
|
87
|
+
|
|
88
|
+
Saved results can be opened without loading the model:
|
|
89
|
+
|
|
90
|
+
```python
|
|
91
|
+
from bend2 import Run
|
|
92
|
+
|
|
93
|
+
run = Run("results/my-model")
|
|
94
|
+
scores = run.to_dataframe()
|
|
95
|
+
predictions = run.table("atlas_profile")
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
Runs stay local until you choose to share them. Use `bend2 --help` for commands
|
|
99
|
+
and `bend2 tasks` for task requirements. The source distribution includes
|
|
100
|
+
`docs/submitting.md` for preparing a future leaderboard contribution.
|
bend2-1.0.0/README.md
ADDED
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+

|
|
2
|
+
|
|
3
|
+
We built BEND 2.0 to test whether genomic models can guide the design of DNA. Across eight tasks, we examine how they predict sequence function, account for cell type and genetic background, and choose sequences that produce a desired response.
|
|
4
|
+
|
|
5
|
+
<picture>
|
|
6
|
+
<source media="(max-width: 600px) and (prefers-color-scheme: dark)" srcset="docs/assets/capabilities-narrow-dark.svg">
|
|
7
|
+
<source media="(max-width: 600px)" srcset="docs/assets/capabilities-narrow-light.svg">
|
|
8
|
+
<source media="(prefers-color-scheme: dark)" srcset="docs/assets/capabilities-wide-dark.svg">
|
|
9
|
+
<img src="docs/assets/capabilities-wide-light.svg" alt="Five capabilities: recover a hidden base; predict an edit’s effect; account for cells and tissues; account for DNA background; select DNA for a target response.">
|
|
10
|
+
</picture>
|
|
11
|
+
|
|
12
|
+
<!-- BEGIN GENERATED RESULTS: node website/scripts/readme-results.mjs -->
|
|
13
|
+
## Benchmark results
|
|
14
|
+
|
|
15
|
+
<a href="docs/assets/results-light.svg">
|
|
16
|
+
<picture>
|
|
17
|
+
<source media="(prefers-color-scheme: dark)" srcset="docs/assets/results-dark.svg">
|
|
18
|
+
<img src="docs/assets/results-light.svg" alt="BEND 2.0 results for 8 tasks and 13 model/readout configurations, ordered by combined score. Full values are available in the text table below.">
|
|
19
|
+
</picture>
|
|
20
|
+
</a>
|
|
21
|
+
|
|
22
|
+
<details>
|
|
23
|
+
<summary>Results as a text table</summary>
|
|
24
|
+
|
|
25
|
+
| Model / readout | Combined<br>score ↑ | Hidden-base<br>loss<br>bits ↓ | Splicing<br>effects<br>Spearman ↑ | Cell<br>contrast<br>Spearman ↑ | Tissue<br>identification<br>top-1 % ↑ | Background<br>interaction<br>Spearman ↑ | Single-edit<br>rank<br>percentile ↓ | Cell<br>selectivity<br>log₂ utility ↑ | Multi-cell<br>gain<br>log₂ gain ↑ |
|
|
26
|
+
| :--- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
|
|
27
|
+
| AlphaGenome<br><sub>Fitted readouts</sub> | 136.0 | — | 0.650 | 0.402 | 64.7 | 0.269 | 25.7 | 0.515 | 0.833 |
|
|
28
|
+
| AlphaGenome<br><sub>Native outputs</sub> | 129.1 | — | 0.621 | 0.398 | 68.6 | 0.260 | 24.3 | 0.384 | 0.707 |
|
|
29
|
+
| NTv3 650M post<br><sub>Fitted readouts</sub> | 100.0 | 1.375 | 0.564 | 0.256 | 56.9 | 0.187 | 30.0 | 0.287 | 0.568 |
|
|
30
|
+
| Enformer<br><sub>Native outputs</sub> | 94.8 | — | — | 0.282 | 31.4 | 0.181 | 31.5 | 0.290 | 0.700 |
|
|
31
|
+
| NTv3 650M post<br><sub>Native outputs</sub> | 90.1 | 1.375 | 0.461 | 0.239 | 45.1 | 0.170 | 29.3 | 0.250 | 0.482 |
|
|
32
|
+
| NTv3 100M post<br><sub>Fitted readouts</sub> | 80.4 | 1.492 | 0.502 | 0.244 | 51.0 | 0.139 | 30.3 | 0.211 | 0.144 |
|
|
33
|
+
| NTv3 100M post<br><sub>Native outputs</sub> | 65.6 | 1.492 | 0.168 | 0.203 | 27.5 | 0.152 | 32.0 | 0.200 | 0.344 |
|
|
34
|
+
| Evo2 20B<br><sub>Fitted readouts</sub> | 40.4 | 1.388 | 0.367 | 0.041 | 21.6 | 0.050 | 43.5 | -0.350 | 0.275 |
|
|
35
|
+
| NTv3 100M pre<br><sub>Fitted readouts</sub> | 32.1 | 1.537 | 0.349 | 0.076 | 7.8 | 0.010 | 47.3 | -0.207 | 0.339 |
|
|
36
|
+
| GENERATORv2<br><sub>Fitted readouts</sub> | 31.5 | 1.711 | 0.388 | 0.066 | 17.6 | 0.046 | 44.7 | -0.256 | 0.234 |
|
|
37
|
+
| NTv3 650M pre<br><sub>Fitted readouts</sub> | 29.4 | 1.447 | 0.187 | 0.010 | 7.8 | 0.009 | 50.4 | -0.061 | 0.368 |
|
|
38
|
+
| CENO 1B<br><sub>Fitted readouts</sub> | 14.7 | 1.549 | 0.295 | 0.036 | 9.8 | 0.010 | 48.9 | -0.318 | -0.581 |
|
|
39
|
+
| DNABERT-2<br><sub>Fitted readouts</sub> | 11.3 | — | 0.198 | 0.076 | 5.9 | 0.020 | 48.6 | -0.302 | 0.233 |
|
|
40
|
+
| Experimental reference | — | — | 0.809 | 0.594 | 99.0 | 0.421 | 11.8 | 0.962 | 1.602 |
|
|
41
|
+
|
|
42
|
+
</details>
|
|
43
|
+
<!-- END GENERATED RESULTS -->
|
|
44
|
+
|
|
45
|
+
## Evaluate your model
|
|
46
|
+
|
|
47
|
+
BEND prepares the data and fits readouts to your model’s frozen embeddings.
|
|
48
|
+
Install in your model's environment with **Python 3.11 or later**:
|
|
49
|
+
|
|
50
|
+
```bash
|
|
51
|
+
python -m pip install bend2
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
The repository currently requires GitHub access; the package is available
|
|
55
|
+
independently from PyPI. Install your model's framework and weights separately.
|
|
56
|
+
|
|
57
|
+
Small datasets are included with the package. Larger files download from the
|
|
58
|
+
[BEND 2.0 data release on Hugging Face](https://huggingface.co/datasets/yhsure/BEND2)
|
|
59
|
+
when needed, with fixed versions and checksum checks. See
|
|
60
|
+
[data and offline use](docs/data.md) to download them in advance.
|
|
61
|
+
|
|
62
|
+
### Connect an encoder
|
|
63
|
+
|
|
64
|
+
Save this as `my_model.py`, replacing `your_encoder` with your model's inference:
|
|
65
|
+
|
|
66
|
+
```python
|
|
67
|
+
import numpy as np
|
|
68
|
+
from bend2 import Model
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def embed(sequences: list[str]) -> np.ndarray:
|
|
72
|
+
return your_encoder(sequences) # [batch, features]
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
model = Model("My model", embed=embed, revision="checkpoint-v1")
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
Return one finite, fixed-width CPU NumPy vector per complete sequence, in input
|
|
79
|
+
order. Use your actual checkpoint revision. The [model guide](docs/models.md)
|
|
80
|
+
covers pooling, model declarations and the nucleotide-probability interface.
|
|
81
|
+
|
|
82
|
+
### Run a first task
|
|
83
|
+
|
|
84
|
+
Start with Atlas: 7,657 short sequences, sixteen multi-cell design requests,
|
|
85
|
+
and data included with the package.
|
|
86
|
+
|
|
87
|
+
```bash
|
|
88
|
+
bend2 run my_model.py --tasks atlas_profile --output results/my-model
|
|
89
|
+
bend2 show results/my-model
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
<details>
|
|
93
|
+
<summary>Try an included example first</summary>
|
|
94
|
+
|
|
95
|
+
Run the base-composition baseline on CPU, without model weights:
|
|
96
|
+
|
|
97
|
+
```bash
|
|
98
|
+
bend2 run composition --tasks atlas_profile --output results/composition
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
`bend2 init my_model.py` writes an editable version of this example.
|
|
102
|
+
[Included adapters](docs/models.md#included-adapters) cover DNABERT-2, NTv3 and
|
|
103
|
+
HyenaDNA. The [model guide](docs/models.md#embeddings) also shows Python usage
|
|
104
|
+
with an encoder already in memory.
|
|
105
|
+
|
|
106
|
+
</details>
|
|
107
|
+
|
|
108
|
+
## Read the results
|
|
109
|
+
|
|
110
|
+
Reopen a saved run without loading the model:
|
|
111
|
+
|
|
112
|
+
```python
|
|
113
|
+
from bend2 import Run
|
|
114
|
+
|
|
115
|
+
run = Run("results/my-model")
|
|
116
|
+
scores = run.to_dataframe()
|
|
117
|
+
predictions = run.table("atlas_profile")
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
The result directory keeps scores and predictions as JSON and Parquet files.
|
|
121
|
+
`run.tables` lists the available tables; `run.table("atlas_profile", "requests")`
|
|
122
|
+
opens the design requests. Use `columns=[...]` to read selected columns.
|
|
123
|
+
|
|
124
|
+
## Choose the next task
|
|
125
|
+
|
|
126
|
+
Seven tasks use embeddings; sequence priors need nucleotide probabilities.
|
|
127
|
+
Run the subset your model supports.
|
|
128
|
+
|
|
129
|
+
| Task or group | What it tests |
|
|
130
|
+
| :--- | :--- |
|
|
131
|
+
| `atlas_profile` | Multi-cell design |
|
|
132
|
+
| `prior` | Hidden bases |
|
|
133
|
+
| `opensplice` | Splice effects |
|
|
134
|
+
| `siraj` | Four regulatory tasks |
|
|
135
|
+
| `tissue_identification` | Tissue identity |
|
|
136
|
+
|
|
137
|
+
`siraj` runs `context`, `background`, `repair_percentile` and `cell_selective`
|
|
138
|
+
together, sharing inference work. You can also pass individual task IDs,
|
|
139
|
+
a comma-separated list, or `all` for all compatible tasks.
|
|
140
|
+
|
|
141
|
+
Check input lengths and downloads before starting a larger run:
|
|
142
|
+
|
|
143
|
+
```bash
|
|
144
|
+
bend2 plan my_model.py --tasks siraj
|
|
145
|
+
bend2 run my_model.py --tasks siraj --output results/my-model
|
|
146
|
+
```
|
|
147
|
+
|
|
148
|
+
See [tasks & compute](docs/tasks.md) for requirements, scores and biological
|
|
149
|
+
holdouts. Long-context tissue identification needs the `genome` extra.
|
|
150
|
+
|
|
151
|
+
<details>
|
|
152
|
+
<summary>Resume or combine runs</summary>
|
|
153
|
+
|
|
154
|
+
Repeat a command to resume. If inference runs out of memory, lower `--batch-size`;
|
|
155
|
+
completed batches remain cached. A changed checkpoint, preprocessing, task
|
|
156
|
+
protocol or numeric runtime requires a new output directory. Compatible updates
|
|
157
|
+
to BEND and changes to descriptive metadata preserve completed work.
|
|
158
|
+
|
|
159
|
+
To combine tasks evaluated in separate jobs:
|
|
160
|
+
|
|
161
|
+
```bash
|
|
162
|
+
bend2 combine results/my-model results/my-model-siraj --output results/combined
|
|
163
|
+
```
|
|
164
|
+
|
|
165
|
+
Use the same adapter, model revision, declarations and inference settings in each
|
|
166
|
+
job. Duplicate tasks must agree. Continue evaluations in the source runs and
|
|
167
|
+
combine them again into a new directory. A combined run can be read or submitted.
|
|
168
|
+
The Python equivalent is `combine(["results/my-model", "results/my-model-siraj"],
|
|
169
|
+
output="results/combined")`, imported from `bend2`.
|
|
170
|
+
|
|
171
|
+
</details>
|
|
172
|
+
|
|
173
|
+
## Add your model to the leaderboard
|
|
174
|
+
|
|
175
|
+
Submit any subset of the eight tasks. Maintainers review the model setup and
|
|
176
|
+
rescore the saved predictions before accepting a submission. Runs stay local
|
|
177
|
+
until you choose to share them.
|
|
178
|
+
|
|
179
|
+
**[Prepare a submission →](docs/submitting.md)**
|
|
180
|
+
|
|
181
|
+
[Data on Hugging Face](https://huggingface.co/datasets/yhsure/BEND2) · [Data & licenses](docs/data.md) · [Contributing](CONTRIBUTING.md) · [Citation](CITATION.cff)
|