flowlayers 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flowlayers-0.1.0/.github/workflows/publish.yml +29 -0
- flowlayers-0.1.0/.github/workflows/tests.yml +15 -0
- flowlayers-0.1.0/.gitignore +7 -0
- flowlayers-0.1.0/CHANGELOG.md +8 -0
- flowlayers-0.1.0/LICENSE +21 -0
- flowlayers-0.1.0/PKG-INFO +114 -0
- flowlayers-0.1.0/README.md +90 -0
- flowlayers-0.1.0/benchmarks/bench_flywire.py +98 -0
- flowlayers-0.1.0/benchmarks/bench_navis.py +66 -0
- flowlayers-0.1.0/benchmarks/results/flywire_rtx5050.txt +30 -0
- flowlayers-0.1.0/benchmarks/results/random_graphs_rtx5050.md +6 -0
- flowlayers-0.1.0/pyproject.toml +35 -0
- flowlayers-0.1.0/src/flowlayers/__init__.py +6 -0
- flowlayers-0.1.0/src/flowlayers/exact.py +49 -0
- flowlayers-0.1.0/src/flowlayers/graph.py +116 -0
- flowlayers-0.1.0/src/flowlayers/model.py +263 -0
- flowlayers-0.1.0/tests/test_model.py +180 -0
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
name: publish
|
|
2
|
+
on:
|
|
3
|
+
release:
|
|
4
|
+
types: [published]
|
|
5
|
+
jobs:
|
|
6
|
+
build:
|
|
7
|
+
runs-on: ubuntu-latest
|
|
8
|
+
steps:
|
|
9
|
+
- uses: actions/checkout@v4
|
|
10
|
+
- uses: actions/setup-python@v5
|
|
11
|
+
with:
|
|
12
|
+
python-version: "3.12"
|
|
13
|
+
- run: pip install build && python -m build
|
|
14
|
+
- uses: actions/upload-artifact@v4
|
|
15
|
+
with:
|
|
16
|
+
name: dist
|
|
17
|
+
path: dist/
|
|
18
|
+
pypi:
|
|
19
|
+
needs: build
|
|
20
|
+
runs-on: ubuntu-latest
|
|
21
|
+
environment: pypi
|
|
22
|
+
permissions:
|
|
23
|
+
id-token: write
|
|
24
|
+
steps:
|
|
25
|
+
- uses: actions/download-artifact@v4
|
|
26
|
+
with:
|
|
27
|
+
name: dist
|
|
28
|
+
path: dist/
|
|
29
|
+
- uses: pypa/gh-action-pypi-publish@release/v1
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
name: tests
|
|
2
|
+
on: [push, pull_request]
|
|
3
|
+
jobs:
|
|
4
|
+
test:
|
|
5
|
+
runs-on: ubuntu-latest
|
|
6
|
+
strategy:
|
|
7
|
+
matrix:
|
|
8
|
+
python-version: ["3.10", "3.11", "3.12", "3.13"]
|
|
9
|
+
steps:
|
|
10
|
+
- uses: actions/checkout@v4
|
|
11
|
+
- uses: actions/setup-python@v5
|
|
12
|
+
with:
|
|
13
|
+
python-version: ${{ matrix.python-version }}
|
|
14
|
+
- run: pip install -e ".[test]"
|
|
15
|
+
- run: pytest -q
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## 0.1.0 (November 2026)
|
|
4
|
+
|
|
5
|
+
First release: `Graph`, `TraversalModel` (CPU and GPU), `TraversalResult` with the full first-traversal
|
|
6
|
+
distribution and navis-compatible `summary`, convergence-based run count, per-run layers for target
|
|
7
|
+
neurons, an exact solver for small graphs, tests against the exact solver and against `navis`, and
|
|
8
|
+
benchmarks on random graphs and on the whole FlyWire brain.
|
flowlayers-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Atlahua Quiahuitl Martínez Obregón
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: flowlayers
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Fast probabilistic traversal of connectomes (Schlegel et al. 2021 information-flow layers), on CPU or GPU
|
|
5
|
+
Project-URL: Repository, https://github.com/Quiwitl/flowlayers
|
|
6
|
+
Author: Atlahua Quiahuitl Martínez Obregón
|
|
7
|
+
License-Expression: MIT
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
Keywords: connectome,connectomics,drosophila,flywire,gpu,graph traversal,information flow
|
|
10
|
+
Classifier: Intended Audience :: Science/Research
|
|
11
|
+
Classifier: Operating System :: OS Independent
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
|
|
14
|
+
Requires-Python: >=3.10
|
|
15
|
+
Requires-Dist: numpy>=1.24
|
|
16
|
+
Requires-Dist: pandas>=2.0
|
|
17
|
+
Requires-Dist: scipy>=1.10
|
|
18
|
+
Provides-Extra: gpu
|
|
19
|
+
Requires-Dist: cupy-cuda12x>=13; extra == 'gpu'
|
|
20
|
+
Provides-Extra: test
|
|
21
|
+
Requires-Dist: navis>=1.10; extra == 'test'
|
|
22
|
+
Requires-Dist: pytest>=7; extra == 'test'
|
|
23
|
+
Description-Content-Type: text/markdown
|
|
24
|
+
|
|
25
|
+
# flowlayers
|
|
26
|
+
|
|
27
|
+
Fast probabilistic traversal of connectomes: the information-flow layers of [Schlegel et al. (2021)](https://doi.org/10.7554/eLife.66018), vectorized on the CPU and optionally on the GPU.
|
|
28
|
+
|
|
29
|
+
The model is the one in [`navis.models.TraversalModel`](https://navis-org.github.io/navis/): seed neurons are layer 1, and at each step every edge from a traversed neuron to a not-yet-traversed one fires with a probability proportional to its input fraction (100 % at 0.3). `flowlayers` runs **the same Monte Carlo model** (same rules, same distribution, no approximation) but draws whole batches of runs at once as sparse matrix products, so a whole brain takes seconds per seed set instead of hours.
|
|
30
|
+
|
|
31
|
+
On the whole FlyWire brain (138,625 neurons, 15.1 million connections), the 13 sensory seed sets of Dorkenwald et al. (2024), each run until the mean layers converge (32,100 runs in total), take **8.7 minutes on a laptop GPU**. At the 3.6 s per run that `navis` takes on the same graph, that is about 32 hours.
|
|
32
|
+
|
|
33
|
+
## Install
|
|
34
|
+
|
|
35
|
+
```bash
|
|
36
|
+
pip install flowlayers # CPU (numpy, scipy, pandas)
|
|
37
|
+
pip install "flowlayers[gpu]" # + CuPy for NVIDIA GPUs (CUDA 12)
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
## Use
|
|
41
|
+
|
|
42
|
+
```python
|
|
43
|
+
import flowlayers as fl
|
|
44
|
+
|
|
45
|
+
# edges: a DataFrame with presynaptic ID, postsynaptic ID and synapse count
|
|
46
|
+
model = fl.TraversalModel(edges, source="pre", target="post", weight="syn_count",
|
|
47
|
+
max_steps=15, device="auto") # "cpu", "gpu" or "auto"
|
|
48
|
+
res = model.run(olfactory_ids) # batches of 100 runs until the mean layer converges
|
|
49
|
+
|
|
50
|
+
res.layer_mean # pd.Series: mean layer of every neuron (NaN if never reached)
|
|
51
|
+
res.summary # layer_min, layer_max, layer_mean, layer_median, reached (navis' summary columns)
|
|
52
|
+
res.cmf() # probability of having been reached by each step
|
|
53
|
+
res.reached # fraction of runs in which each neuron was reached
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
- **Weights.** By default the weights are synapse counts and are turned into input fractions (`normalize=True`). Pass `normalize=False` if they already are input fractions, as `navis` expects. `min_weight=5` drops edges with fewer than 5 synapses *before* normalizing — this changes the layers noticeably (see below), so report it.
|
|
57
|
+
- **Runs.** `run(seeds, runs=1000)` does exactly 1,000 runs. Without `runs`, batches of 100 are added until the mean layer changes by less than `tol=0.01` in 99 % of the neurons (at least 5 batches; `max_runs=10_000`).
|
|
58
|
+
- **Per-run layers.** `run(seeds, targets=motor_neuron_ids)` also keeps the layer of those neurons in every run (`res.target_runs`), e.g. for bootstrap intervals.
|
|
59
|
+
- **Reuse.** Build the model once and call `run` for each seed set; the graph is prepared only once (and kept on the GPU).
|
|
60
|
+
- **Probability.** `max_w=0.3` and `min_w=0` as in `navis` (`linear_activation_p`).
|
|
61
|
+
|
|
62
|
+
## How it works
|
|
63
|
+
|
|
64
|
+
Drawing each edge independently and traversing a neuron if any of its edges fires is the same as drawing each neuron once with
|
|
65
|
+
|
|
66
|
+
P<sub>j</sub> = 1 − Π<sub>i</sub> (1 − p<sub>ij</sub>) over the traversed *i*,
|
|
67
|
+
|
|
68
|
+
which is `1 − exp(Σ log(1 − p_ij))`: one sparse matrix product with the indicator matrix of traversed neurons. Each run is a column of that matrix, so one step of a whole batch of runs is a single sparse × dense product (scipy on the CPU, cuSPARSE through CuPy on the GPU; the same code runs on both). Failed neurons are tried again at later steps, as in `navis`. Results are kept as the full distribution of the first-traversal step of every neuron, from which the mean, median, min, max and CMF are derived.
|
|
69
|
+
|
|
70
|
+
## Validation
|
|
71
|
+
|
|
72
|
+
- **Exact answer on small graphs.** `flowlayers.exact.exact_layers` enumerates every set of traversed nodes and gives the exact first-traversal distribution. On the diamond of [navis issue #194](https://github.com/navis-org/navis/issues/194) (0→1, 0→2, 1→3, 2→3, p = 0.5) it gives 3.96296 for node 3, the analytic value reported there; `flowlayers` matches it, and matches the full distribution node by node on random small graphs.
|
|
73
|
+
- **Same distribution as navis.** On random graphs the mean layers agree with `navis.models.TraversalModel` with no bias; the remaining differences are sampling noise.
|
|
74
|
+
- **Published layers.** This model, on the whole brain with no synapse threshold and a 15-step cap, reproduces the layers published by Dorkenwald et al. (2024) for FlyWire v783 in all 13 seed sets (Spearman 0.971–0.996, [fly-sensory-layers](https://github.com/Quiwitl/fly-sensory-layers)); flowlayers matches that implementation at ρ ≥ 0.998 (below).
|
|
75
|
+
- **CPU and GPU.** Same results on both devices (tested).
|
|
76
|
+
|
|
77
|
+
## Benchmarks
|
|
78
|
+
|
|
79
|
+
Laptop with an AMD Ryzen 7 260 (16 threads) and an NVIDIA RTX 5050 Laptop GPU (8 GB), Windows 11, navis 1.12.0, numpy 2.5, scipy 1.18, CuPy 14.2. Scripts in `benchmarks/`.
|
|
80
|
+
|
|
81
|
+
**Random graphs** (log-normal synapse counts, 20 seeds, 100 runs, `max_steps=15`; `bench_navis.py --gpu`). "Mean abs diff" and "bias" compare the mean layers of flowlayers and navis; with 100 runs each the absolute difference is sampling noise, and the bias stays below 0.01 layers.
|
|
82
|
+
|
|
83
|
+
| nodes | edges | navis | flowlayers CPU | speed-up | flowlayers GPU | speed-up | mean abs diff | bias |
|
|
84
|
+
|---:|---:|---:|---:|---:|---:|---:|---:|---:|
|
|
85
|
+
| 5,000 | 149,557 | 2.0 s | 0.14 s | ×14 | 0.02 s | ×86 | 0.092 | +0.001 |
|
|
86
|
+
| 20,000 | 998,750 | 12.2 s | 1.32 s | ×9 | 0.11 s | ×116 | 0.102 | −0.008 |
|
|
87
|
+
| 50,000 | 2,998,196 | 42.4 s | 8.57 s | ×5 | 0.25 s | ×168 | 0.107 | −0.006 |
|
|
88
|
+
| 140,000 | 13,994,879 | 143.6 s | 39.7 s | ×4 | 1.28 s | ×112 | 0.110 | −0.005 |
|
|
89
|
+
|
|
90
|
+
The CPU gain shrinks on big graphs (the sparse product becomes memory-bound); the GPU is the option for whole brains.
|
|
91
|
+
|
|
92
|
+
**Whole FlyWire brain** (v783, proofread connections, no synapse threshold, 15 steps; `bench_flywire.py --device gpu --navis-runs 2`):
|
|
93
|
+
|
|
94
|
+
| | navis | flowlayers GPU |
|
|
95
|
+
|---|---:|---:|
|
|
96
|
+
| 1,000 runs from the olfactory receptor neurons | ≈ 60 min (3.61 s per run) | 16.5 s (×219) |
|
|
97
|
+
| 13 seed sets to convergence (32,100 runs) | ≈ 32 h (extrapolated) | 8.7 min |
|
|
98
|
+
|
|
99
|
+
The mean layers agree with the independent implementation in [fly-sensory-layers](https://github.com/Quiwitl/fly-sensory-layers) at Spearman ρ = 0.998–0.9998 for every seed set (seeds excluded), the level of run-to-run noise.
|
|
100
|
+
|
|
101
|
+
## Differences from navis
|
|
102
|
+
|
|
103
|
+
- Seeds that never appear as a source are **kept** (as layer 1). `navis` drops them silently, which turns them into ordinary nodes.
|
|
104
|
+
- A run stops when no edge with p > 0 is left (or at `max_steps`); `navis` also waits on edges with p = 0, which changes when it stops but not the layers.
|
|
105
|
+
- Only the linear probability function (`min_w`, `max_w`) is supported; arbitrary per-edge Python callables cannot be vectorized.
|
|
106
|
+
- No `results` table with one row per node and run (it would have hundreds of millions of rows for a whole brain); the per-step counts hold the same information.
|
|
107
|
+
|
|
108
|
+
## Cite
|
|
109
|
+
|
|
110
|
+
If you use `flowlayers`, please cite the model:
|
|
111
|
+
|
|
112
|
+
> Schlegel P., Bates A. S., Stürner T., *et al.* (2021). Information flow, cell types and stereotypy in a full olfactory connectome. *eLife* 10, e66018.
|
|
113
|
+
|
|
114
|
+
and this repository. MIT license.
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
# flowlayers
|
|
2
|
+
|
|
3
|
+
Fast probabilistic traversal of connectomes: the information-flow layers of [Schlegel et al. (2021)](https://doi.org/10.7554/eLife.66018), vectorized on the CPU and optionally on the GPU.
|
|
4
|
+
|
|
5
|
+
The model is the one in [`navis.models.TraversalModel`](https://navis-org.github.io/navis/): seed neurons are layer 1, and at each step every edge from a traversed neuron to a not-yet-traversed one fires with a probability proportional to its input fraction (100 % at 0.3). `flowlayers` runs **the same Monte Carlo model** (same rules, same distribution, no approximation) but draws whole batches of runs at once as sparse matrix products, so a whole brain takes seconds per seed set instead of hours.
|
|
6
|
+
|
|
7
|
+
On the whole FlyWire brain (138,625 neurons, 15.1 million connections), the 13 sensory seed sets of Dorkenwald et al. (2024), each run until the mean layers converge (32,100 runs in total), take **8.7 minutes on a laptop GPU**. At the 3.6 s per run that `navis` takes on the same graph, that is about 32 hours.
|
|
8
|
+
|
|
9
|
+
## Install
|
|
10
|
+
|
|
11
|
+
```bash
|
|
12
|
+
pip install flowlayers # CPU (numpy, scipy, pandas)
|
|
13
|
+
pip install "flowlayers[gpu]" # + CuPy for NVIDIA GPUs (CUDA 12)
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
## Use
|
|
17
|
+
|
|
18
|
+
```python
|
|
19
|
+
import flowlayers as fl
|
|
20
|
+
|
|
21
|
+
# edges: a DataFrame with presynaptic ID, postsynaptic ID and synapse count
|
|
22
|
+
model = fl.TraversalModel(edges, source="pre", target="post", weight="syn_count",
|
|
23
|
+
max_steps=15, device="auto") # "cpu", "gpu" or "auto"
|
|
24
|
+
res = model.run(olfactory_ids) # batches of 100 runs until the mean layer converges
|
|
25
|
+
|
|
26
|
+
res.layer_mean # pd.Series: mean layer of every neuron (NaN if never reached)
|
|
27
|
+
res.summary # layer_min, layer_max, layer_mean, layer_median, reached (navis' summary columns)
|
|
28
|
+
res.cmf() # probability of having been reached by each step
|
|
29
|
+
res.reached # fraction of runs in which each neuron was reached
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
- **Weights.** By default the weights are synapse counts and are turned into input fractions (`normalize=True`). Pass `normalize=False` if they already are input fractions, as `navis` expects. `min_weight=5` drops edges with fewer than 5 synapses *before* normalizing — this changes the layers noticeably (see below), so report it.
|
|
33
|
+
- **Runs.** `run(seeds, runs=1000)` does exactly 1,000 runs. Without `runs`, batches of 100 are added until the mean layer changes by less than `tol=0.01` in 99 % of the neurons (at least 5 batches; `max_runs=10_000`).
|
|
34
|
+
- **Per-run layers.** `run(seeds, targets=motor_neuron_ids)` also keeps the layer of those neurons in every run (`res.target_runs`), e.g. for bootstrap intervals.
|
|
35
|
+
- **Reuse.** Build the model once and call `run` for each seed set; the graph is prepared only once (and kept on the GPU).
|
|
36
|
+
- **Probability.** `max_w=0.3` and `min_w=0` as in `navis` (`linear_activation_p`).
|
|
37
|
+
|
|
38
|
+
## How it works
|
|
39
|
+
|
|
40
|
+
Drawing each edge independently and traversing a neuron if any of its edges fires is the same as drawing each neuron once with
|
|
41
|
+
|
|
42
|
+
P<sub>j</sub> = 1 − Π<sub>i</sub> (1 − p<sub>ij</sub>) over the traversed *i*,
|
|
43
|
+
|
|
44
|
+
which is `1 − exp(Σ log(1 − p_ij))`: one sparse matrix product with the indicator matrix of traversed neurons. Each run is a column of that matrix, so one step of a whole batch of runs is a single sparse × dense product (scipy on the CPU, cuSPARSE through CuPy on the GPU; the same code runs on both). Failed neurons are tried again at later steps, as in `navis`. Results are kept as the full distribution of the first-traversal step of every neuron, from which the mean, median, min, max and CMF are derived.
|
|
45
|
+
|
|
46
|
+
## Validation
|
|
47
|
+
|
|
48
|
+
- **Exact answer on small graphs.** `flowlayers.exact.exact_layers` enumerates every set of traversed nodes and gives the exact first-traversal distribution. On the diamond of [navis issue #194](https://github.com/navis-org/navis/issues/194) (0→1, 0→2, 1→3, 2→3, p = 0.5) it gives 3.96296 for node 3, the analytic value reported there; `flowlayers` matches it, and matches the full distribution node by node on random small graphs.
|
|
49
|
+
- **Same distribution as navis.** On random graphs the mean layers agree with `navis.models.TraversalModel` with no bias; the remaining differences are sampling noise.
|
|
50
|
+
- **Published layers.** This model, on the whole brain with no synapse threshold and a 15-step cap, reproduces the layers published by Dorkenwald et al. (2024) for FlyWire v783 in all 13 seed sets (Spearman 0.971–0.996, [fly-sensory-layers](https://github.com/Quiwitl/fly-sensory-layers)); flowlayers matches that implementation at ρ ≥ 0.998 (below).
|
|
51
|
+
- **CPU and GPU.** Same results on both devices (tested).
|
|
52
|
+
|
|
53
|
+
## Benchmarks
|
|
54
|
+
|
|
55
|
+
Laptop with an AMD Ryzen 7 260 (16 threads) and an NVIDIA RTX 5050 Laptop GPU (8 GB), Windows 11, navis 1.12.0, numpy 2.5, scipy 1.18, CuPy 14.2. Scripts in `benchmarks/`.
|
|
56
|
+
|
|
57
|
+
**Random graphs** (log-normal synapse counts, 20 seeds, 100 runs, `max_steps=15`; `bench_navis.py --gpu`). "Mean abs diff" and "bias" compare the mean layers of flowlayers and navis; with 100 runs each the absolute difference is sampling noise, and the bias stays below 0.01 layers.
|
|
58
|
+
|
|
59
|
+
| nodes | edges | navis | flowlayers CPU | speed-up | flowlayers GPU | speed-up | mean abs diff | bias |
|
|
60
|
+
|---:|---:|---:|---:|---:|---:|---:|---:|---:|
|
|
61
|
+
| 5,000 | 149,557 | 2.0 s | 0.14 s | ×14 | 0.02 s | ×86 | 0.092 | +0.001 |
|
|
62
|
+
| 20,000 | 998,750 | 12.2 s | 1.32 s | ×9 | 0.11 s | ×116 | 0.102 | −0.008 |
|
|
63
|
+
| 50,000 | 2,998,196 | 42.4 s | 8.57 s | ×5 | 0.25 s | ×168 | 0.107 | −0.006 |
|
|
64
|
+
| 140,000 | 13,994,879 | 143.6 s | 39.7 s | ×4 | 1.28 s | ×112 | 0.110 | −0.005 |
|
|
65
|
+
|
|
66
|
+
The CPU gain shrinks on big graphs (the sparse product becomes memory-bound); the GPU is the option for whole brains.
|
|
67
|
+
|
|
68
|
+
**Whole FlyWire brain** (v783, proofread connections, no synapse threshold, 15 steps; `bench_flywire.py --device gpu --navis-runs 2`):
|
|
69
|
+
|
|
70
|
+
| | navis | flowlayers GPU |
|
|
71
|
+
|---|---:|---:|
|
|
72
|
+
| 1,000 runs from the olfactory receptor neurons | ≈ 60 min (3.61 s per run) | 16.5 s (×219) |
|
|
73
|
+
| 13 seed sets to convergence (32,100 runs) | ≈ 32 h (extrapolated) | 8.7 min |
|
|
74
|
+
|
|
75
|
+
The mean layers agree with the independent implementation in [fly-sensory-layers](https://github.com/Quiwitl/fly-sensory-layers) at Spearman ρ = 0.998–0.9998 for every seed set (seeds excluded), the level of run-to-run noise.
|
|
76
|
+
|
|
77
|
+
## Differences from navis
|
|
78
|
+
|
|
79
|
+
- Seeds that never appear as a source are **kept** (as layer 1). `navis` drops them silently, which turns them into ordinary nodes.
|
|
80
|
+
- A run stops when no edge with p > 0 is left (or at `max_steps`); `navis` also waits on edges with p = 0, which changes when it stops but not the layers.
|
|
81
|
+
- Only the linear probability function (`min_w`, `max_w`) is supported; arbitrary per-edge Python callables cannot be vectorized.
|
|
82
|
+
- No `results` table with one row per node and run (it would have hundreds of millions of rows for a whole brain); the per-step counts hold the same information.
|
|
83
|
+
|
|
84
|
+
## Cite
|
|
85
|
+
|
|
86
|
+
If you use `flowlayers`, please cite the model:
|
|
87
|
+
|
|
88
|
+
> Schlegel P., Bates A. S., Stürner T., *et al.* (2021). Information flow, cell types and stereotypy in a full olfactory connectome. *eLife* 10, e66018.
|
|
89
|
+
|
|
90
|
+
and this repository. MIT license.
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
"""Whole FlyWire brain (v783): the 13 seed sets of Dorkenwald et al. (2024), with and without navis.
|
|
2
|
+
|
|
3
|
+
Needs the FlyWire connections (Zenodo 10676866) and the annotations of Schlegel et al. (2024):
|
|
4
|
+
|
|
5
|
+
python benchmarks/bench_flywire.py --connections proofread_connections_783.feather \
|
|
6
|
+
--annotations neuron_annotations.tsv --device gpu [--compare layers.parquet] [--navis-runs 2]
|
|
7
|
+
|
|
8
|
+
``--compare``: a parquet with ``root_id`` and one column per seed set (e.g. from another implementation);
|
|
9
|
+
prints the Spearman correlation per seed set.
|
|
10
|
+
"""
|
|
11
|
+
import argparse
|
|
12
|
+
import contextlib
|
|
13
|
+
import io
|
|
14
|
+
import time
|
|
15
|
+
|
|
16
|
+
import numpy as np
|
|
17
|
+
import pandas as pd
|
|
18
|
+
|
|
19
|
+
import flowlayers as fl
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def seed_sets(anot):
|
|
23
|
+
c, s, sup = anot["cell_class"], anot["cell_sub_class"], anot["super_class"]
|
|
24
|
+
mech = c == "mechanosensory"
|
|
25
|
+
sets = {
|
|
26
|
+
"olfativas": c == "olfactory", "gustativas": c == "gustatory",
|
|
27
|
+
"johnston": mech & s.isin(["auditory", "wind_gravity"]), "cerdas_cabeza": mech & (s == "head bristle"),
|
|
28
|
+
"cerdas_ojo": mech & (s == "eye bristle"), "taste_peg": mech & (s == "taste peg"),
|
|
29
|
+
"termosensoriales": c == "thermosensory", "higrosensoriales": c == "hygrosensory",
|
|
30
|
+
"proyeccion_visual": sup == "visual_projection", "fotorreceptores": (c == "visual") & (s != "ocellar"),
|
|
31
|
+
"ocelos": (c == "visual") & (s == "ocellar"), "ascendentes": c == "AN",
|
|
32
|
+
}
|
|
33
|
+
sets["todas"] = (anot["flow"] == "afferent") | (sup == "visual_projection")
|
|
34
|
+
return {k: anot.loc[v.to_numpy(), "root_id"].to_numpy() for k, v in sets.items()}
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def main():
|
|
38
|
+
ap = argparse.ArgumentParser()
|
|
39
|
+
ap.add_argument("--connections", required=True)
|
|
40
|
+
ap.add_argument("--annotations", required=True)
|
|
41
|
+
ap.add_argument("--device", default="auto")
|
|
42
|
+
ap.add_argument("--max-steps", type=int, default=15)
|
|
43
|
+
ap.add_argument("--min-synapses", type=float, default=None)
|
|
44
|
+
ap.add_argument("--runs", type=int, default=None, help="fixed runs per set (default: until convergence)")
|
|
45
|
+
ap.add_argument("--compare", default=None)
|
|
46
|
+
ap.add_argument("--navis-runs", type=int, default=0)
|
|
47
|
+
a = ap.parse_args()
|
|
48
|
+
|
|
49
|
+
t0 = time.time()
|
|
50
|
+
t = pd.read_feather(a.connections, columns=["pre_pt_root_id", "post_pt_root_id", "syn_count"])
|
|
51
|
+
anot = pd.read_csv(a.annotations, sep="\t", usecols=["root_id", "flow", "super_class", "cell_class",
|
|
52
|
+
"cell_sub_class", "cell_type"])
|
|
53
|
+
keep = t["pre_pt_root_id"].isin(anot["root_id"]) & t["post_pt_root_id"].isin(anot["root_id"])
|
|
54
|
+
t = t[keep]
|
|
55
|
+
g = fl.Graph.from_edges(sources=t["pre_pt_root_id"].to_numpy(np.int64), targets=t["post_pt_root_id"].to_numpy(np.int64),
|
|
56
|
+
weights=t["syn_count"].to_numpy(np.float64), min_weight=a.min_synapses)
|
|
57
|
+
del t
|
|
58
|
+
model = fl.TraversalModel(g, max_steps=a.max_steps, device=a.device)
|
|
59
|
+
print(f"{g} , loaded and prepared in {time.time() - t0:.0f} s on {model.device}", flush=True)
|
|
60
|
+
|
|
61
|
+
sets = seed_sets(anot)
|
|
62
|
+
layers, total = {}, time.time()
|
|
63
|
+
for name, seeds in sets.items():
|
|
64
|
+
r = model.run(seeds, runs=a.runs, seed=783, missing="ignore")
|
|
65
|
+
layers[name] = r.layer_mean
|
|
66
|
+
print(f"{name:>18}: {len(r.seeds):6,} seeds, {r.runs:6,} runs, {r.seconds:6.1f} s", flush=True)
|
|
67
|
+
print(f"all {len(sets)} seed sets: {(time.time() - total) / 60:.1f} min", flush=True)
|
|
68
|
+
|
|
69
|
+
if a.compare:
|
|
70
|
+
from scipy.stats import spearmanr
|
|
71
|
+
ref = pd.read_parquet(a.compare)
|
|
72
|
+
ref = ref.set_index(ref["root_id"].astype(np.int64))
|
|
73
|
+
print("Spearman against", a.compare)
|
|
74
|
+
for name, s in layers.items():
|
|
75
|
+
if name in ref.columns:
|
|
76
|
+
x = pd.concat([s, ref[name]], axis=1, join="inner").dropna()
|
|
77
|
+
x = x[(x.iloc[:, 0] > 1) & (x.iloc[:, 1] > 1)] # seeds excluded
|
|
78
|
+
print(f"{name:>18}: rho = {spearmanr(x.iloc[:, 0], x.iloc[:, 1]).statistic:.4f} (n = {len(x):,})")
|
|
79
|
+
|
|
80
|
+
if a.navis_runs:
|
|
81
|
+
import navis
|
|
82
|
+
import navis.models as nm
|
|
83
|
+
navis.config.pbar_hide = True
|
|
84
|
+
A = g.A.tocoo()
|
|
85
|
+
edges = pd.DataFrame({"source": g.ids[A.row], "target": g.ids[A.col], "weight": A.data})
|
|
86
|
+
ref = nm.TraversalModel(edges, seeds=sets["olfativas"], max_steps=a.max_steps)
|
|
87
|
+
t = time.time()
|
|
88
|
+
with contextlib.redirect_stdout(io.StringIO()):
|
|
89
|
+
ref.run(iterations=a.navis_runs)
|
|
90
|
+
per_run = (time.time() - t) / a.navis_runs
|
|
91
|
+
ours = model.run(sets["olfativas"], runs=1000, seed=1, missing="ignore")
|
|
92
|
+
print(f"olfactory, 1,000 runs: navis about {per_run * 1000 / 60:.0f} min ({per_run:.2f} s per run, "
|
|
93
|
+
f"single process), flowlayers {ours.seconds:.1f} s on {model.device} "
|
|
94
|
+
f"(x{per_run * 1000 / ours.seconds:.0f})")
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
if __name__ == "__main__":
|
|
98
|
+
main()
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
"""Speed and agreement against navis.models.TraversalModel on random graphs.
|
|
2
|
+
|
|
3
|
+
python benchmarks/bench_navis.py # CPU
|
|
4
|
+
python benchmarks/bench_navis.py --gpu # also the GPU (needs CuPy)
|
|
5
|
+
"""
|
|
6
|
+
import argparse
|
|
7
|
+
import contextlib
|
|
8
|
+
import io
|
|
9
|
+
import time
|
|
10
|
+
|
|
11
|
+
import numpy as np
|
|
12
|
+
import pandas as pd
|
|
13
|
+
|
|
14
|
+
import flowlayers as fl
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def random_graph(n, degree, seed=0):
|
|
18
|
+
rng = np.random.default_rng(seed)
|
|
19
|
+
m = n * degree
|
|
20
|
+
# lognormal synapse counts, roughly like a connectome
|
|
21
|
+
return pd.DataFrame({"source": rng.integers(0, n, m), "target": rng.integers(0, n, m),
|
|
22
|
+
"weight": rng.lognormal(0, 1.2, m).round() + 1})
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def main():
|
|
26
|
+
ap = argparse.ArgumentParser()
|
|
27
|
+
ap.add_argument("--gpu", action="store_true")
|
|
28
|
+
ap.add_argument("--sizes", default="5000:30:100,20000:50:100,50000:60:50")
|
|
29
|
+
a = ap.parse_args()
|
|
30
|
+
import navis
|
|
31
|
+
import navis.models as nm
|
|
32
|
+
navis.config.pbar_hide = True
|
|
33
|
+
print("| nodes | edges | runs | navis (s) | flowlayers CPU (s) | speed-up | flowlayers GPU (s) | speed-up | mean abs diff | bias |")
|
|
34
|
+
print("|---|---|---|---|---|---|---|---|---|---|")
|
|
35
|
+
for spec in a.sizes.split(","):
|
|
36
|
+
n, degree, runs = map(int, spec.split(":"))
|
|
37
|
+
g = fl.Graph.from_edges(random_graph(n, degree))
|
|
38
|
+
A = g.A.tocoo()
|
|
39
|
+
edges = pd.DataFrame({"source": g.ids[A.row], "target": g.ids[A.col], "weight": A.data})
|
|
40
|
+
seeds = np.arange(100)
|
|
41
|
+
t = time.time()
|
|
42
|
+
ref = nm.TraversalModel(edges, seeds=seeds, max_steps=15)
|
|
43
|
+
with contextlib.redirect_stdout(io.StringIO()):
|
|
44
|
+
ref.run(iterations=runs)
|
|
45
|
+
t_navis = time.time() - t
|
|
46
|
+
s_ref = ref.summary.set_index("node")["layer_mean"] if "node" in ref.summary.columns else ref.summary["layer_mean"]
|
|
47
|
+
cpu = fl.TraversalModel(g, max_steps=15)
|
|
48
|
+
t = time.time()
|
|
49
|
+
r = cpu.run(seeds, runs=runs, seed=1)
|
|
50
|
+
t_cpu = time.time() - t
|
|
51
|
+
gpu_txt = "–", "–"
|
|
52
|
+
if a.gpu:
|
|
53
|
+
gm = fl.TraversalModel(g, max_steps=15, device="gpu")
|
|
54
|
+
gm.run(seeds, runs=min(runs, 100), seed=0) # warm-up (kernel compilation)
|
|
55
|
+
t = time.time()
|
|
56
|
+
gm.run(seeds, runs=runs, seed=1)
|
|
57
|
+
t_gpu = time.time() - t
|
|
58
|
+
gpu_txt = f"{t_gpu:.2f}", f"x{t_navis / t_gpu:.0f}"
|
|
59
|
+
common = s_ref.index.intersection(r.layer_mean.dropna().index)
|
|
60
|
+
d = r.layer_mean[common] - s_ref[common]
|
|
61
|
+
print(f"| {n:,} | {g.n_edges:,} | {runs} | {t_navis:.1f} | {t_cpu:.2f} | x{t_navis / t_cpu:.0f} | "
|
|
62
|
+
f"{gpu_txt[0]} | {gpu_txt[1]} | {d.abs().mean():.3f} | {d.mean():+.3f} |", flush=True)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
if __name__ == "__main__":
|
|
66
|
+
main()
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
<flowlayers.Graph: 138,625 nodes, 15,090,883 edges> · loaded and prepared in 10 s on gpu
|
|
2
|
+
olfativas: 2,279 seeds · 2,500 runs · 42.5 s
|
|
3
|
+
gustativas: 410 seeds · 2,600 runs · 42.3 s
|
|
4
|
+
johnston: 868 seeds · 2,500 runs · 40.6 s
|
|
5
|
+
cerdas_cabeza: 305 seeds · 2,600 runs · 42.2 s
|
|
6
|
+
cerdas_ojo: 1,112 seeds · 2,600 runs · 42.2 s
|
|
7
|
+
taste_peg: 75 seeds · 2,500 runs · 40.6 s
|
|
8
|
+
termosensoriales: 29 seeds · 2,600 runs · 42.2 s
|
|
9
|
+
higrosensoriales: 74 seeds · 2,600 runs · 42.2 s
|
|
10
|
+
proyeccion_visual: 8,038 seeds · 2,300 runs · 37.5 s
|
|
11
|
+
fotorreceptores: 10,582 seeds · 2,300 runs · 37.3 s
|
|
12
|
+
ocelos: 273 seeds · 2,800 runs · 45.4 s
|
|
13
|
+
ascendentes: 2,231 seeds · 2,300 runs · 37.3 s
|
|
14
|
+
todas: 26,702 seeds · 1,900 runs · 30.8 s
|
|
15
|
+
all 13 seed sets: 8.7 min
|
|
16
|
+
Spearman against fly-sensory-layers results/02_capas_completo_p15_u1_gpu.parquet
|
|
17
|
+
olfativas: rho = 0.9990 (n = 134,417)
|
|
18
|
+
gustativas: rho = 0.9985 (n = 136,286)
|
|
19
|
+
johnston: rho = 0.9983 (n = 135,828)
|
|
20
|
+
cerdas_cabeza: rho = 0.9980 (n = 136,391)
|
|
21
|
+
cerdas_ojo: rho = 0.9979 (n = 135,584)
|
|
22
|
+
taste_peg: rho = 0.9977 (n = 136,621)
|
|
23
|
+
termosensoriales: rho = 0.9991 (n = 136,667)
|
|
24
|
+
higrosensoriales: rho = 0.9990 (n = 136,622)
|
|
25
|
+
proyeccion_visual: rho = 0.9993 (n = 128,658)
|
|
26
|
+
fotorreceptores: rho = 0.9998 (n = 127,923)
|
|
27
|
+
ocelos: rho = 0.9977 (n = 136,431)
|
|
28
|
+
ascendentes: rho = 0.9981 (n = 134,558)
|
|
29
|
+
todas: rho = 0.9989 (n = 111,908)
|
|
30
|
+
olfactory, 1,000 runs: navis ≈ 60 min (3.61 s per run, single process) · flowlayers 16.5 s on gpu (×219)
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
| nodes | edges | runs | navis (s) | flowlayers CPU (s) | speed-up | flowlayers GPU (s) | speed-up | mean |Δ| | bias |
|
|
2
|
+
|---|---|---|---|---|---|---|---|---|---|
|
|
3
|
+
| 5,000 | 149,557 | 100 | 2.0 | 0.14 | ×14 | 0.02 | ×86 | 0.092 | +0.001 |
|
|
4
|
+
| 20,000 | 998,750 | 100 | 12.2 | 1.32 | ×9 | 0.11 | ×116 | 0.102 | -0.008 |
|
|
5
|
+
| 50,000 | 2,998,196 | 100 | 42.4 | 8.57 | ×5 | 0.25 | ×168 | 0.107 | -0.006 |
|
|
6
|
+
| 140,000 | 13,994,879 | 100 | 143.6 | 39.67 | ×4 | 1.28 | ×112 | 0.110 | -0.005 |
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "flowlayers"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Fast probabilistic traversal of connectomes (Schlegel et al. 2021 information-flow layers), on CPU or GPU"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "MIT"
|
|
11
|
+
license-files = ["LICENSE"]
|
|
12
|
+
authors = [{ name = "Atlahua Quiahuitl Martínez Obregón" }]
|
|
13
|
+
requires-python = ">=3.10"
|
|
14
|
+
dependencies = ["numpy>=1.24", "scipy>=1.10", "pandas>=2.0"]
|
|
15
|
+
keywords = ["connectome", "connectomics", "drosophila", "flywire", "graph traversal", "information flow", "gpu"]
|
|
16
|
+
classifiers = [
|
|
17
|
+
"Programming Language :: Python :: 3",
|
|
18
|
+
"Intended Audience :: Science/Research",
|
|
19
|
+
"Topic :: Scientific/Engineering :: Bio-Informatics",
|
|
20
|
+
"Operating System :: OS Independent",
|
|
21
|
+
]
|
|
22
|
+
|
|
23
|
+
[project.optional-dependencies]
|
|
24
|
+
gpu = ["cupy-cuda12x>=13"]
|
|
25
|
+
test = ["pytest>=7", "navis>=1.10"]
|
|
26
|
+
|
|
27
|
+
[project.urls]
|
|
28
|
+
Repository = "https://github.com/Quiwitl/flowlayers"
|
|
29
|
+
|
|
30
|
+
[tool.hatch.build.targets.wheel]
|
|
31
|
+
packages = ["src/flowlayers"]
|
|
32
|
+
|
|
33
|
+
[tool.pytest.ini_options]
|
|
34
|
+
testpaths = ["tests"]
|
|
35
|
+
filterwarnings = ["ignore::DeprecationWarning"]
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
"""flowlayers: fast probabilistic traversal of connectomes (Schlegel et al. 2021), on CPU or GPU."""
|
|
2
|
+
from .graph import Graph
|
|
3
|
+
from .model import TraversalModel, TraversalResult, linear_p, traverse
|
|
4
|
+
|
|
5
|
+
__version__ = "0.1.0"
|
|
6
|
+
__all__ = ["Graph", "TraversalModel", "TraversalResult", "linear_p", "traverse", "__version__"]
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
"""Exact first-traversal distribution for very small graphs, by enumerating every set of traversed nodes.
|
|
2
|
+
|
|
3
|
+
Used to test the Monte Carlo engine. The cost grows as 2**n, so keep n <= ~12.
|
|
4
|
+
"""
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
from itertools import product
|
|
8
|
+
|
|
9
|
+
import numpy as np
|
|
10
|
+
|
|
11
|
+
__all__ = ["exact_layers"]
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def exact_layers(P: np.ndarray, seeds, max_steps: int = 15) -> np.ndarray:
|
|
15
|
+
"""``out[k, t]`` = probability that node ``k`` is first reached at step ``t`` (``t = 1..max_steps``).
|
|
16
|
+
|
|
17
|
+
``P[i, j]`` is the edge probability ``i -> j`` (0 if there is no edge). Same rules as the model:
|
|
18
|
+
at each step every untraversed node ``j`` is reached with probability ``1 - prod_i (1 - P[i, j])``
|
|
19
|
+
over the traversed ``i``, independently of the other nodes; failed nodes are retried.
|
|
20
|
+
"""
|
|
21
|
+
P = np.asarray(P, dtype=np.float64)
|
|
22
|
+
n = P.shape[0]
|
|
23
|
+
start = frozenset(int(s) for s in seeds)
|
|
24
|
+
out = np.zeros((n, max_steps + 1))
|
|
25
|
+
for s in start:
|
|
26
|
+
out[s, 1] = 1.0
|
|
27
|
+
states = {start: 1.0}
|
|
28
|
+
for t in range(2, max_steps + 1):
|
|
29
|
+
nxt: dict[frozenset, float] = {}
|
|
30
|
+
for S, pr in states.items():
|
|
31
|
+
cand = [j for j in range(n) if j not in S]
|
|
32
|
+
pj = [1.0 - np.prod([1.0 - P[i, j] for i in S]) for j in cand]
|
|
33
|
+
live = [(j, q) for j, q in zip(cand, pj) if q > 0]
|
|
34
|
+
if not live:
|
|
35
|
+
nxt[S] = nxt.get(S, 0.0) + pr
|
|
36
|
+
continue
|
|
37
|
+
for hits in product((0, 1), repeat=len(live)):
|
|
38
|
+
w = pr
|
|
39
|
+
for (j, q), h in zip(live, hits):
|
|
40
|
+
w *= q if h else 1.0 - q
|
|
41
|
+
if w == 0.0:
|
|
42
|
+
continue
|
|
43
|
+
new = [j for (j, _), h in zip(live, hits) if h]
|
|
44
|
+
for j in new:
|
|
45
|
+
out[j, t] += w
|
|
46
|
+
T = S | frozenset(new)
|
|
47
|
+
nxt[T] = nxt.get(T, 0.0) + w
|
|
48
|
+
states = nxt
|
|
49
|
+
return out
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
"""Turn an edge list into the sparse matrices the traversal needs."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import warnings
|
|
5
|
+
|
|
6
|
+
import numpy as np
|
|
7
|
+
import pandas as pd
|
|
8
|
+
from scipy import sparse
|
|
9
|
+
|
|
10
|
+
__all__ = ["Graph"]
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class Graph:
|
|
14
|
+
"""A directed, weighted graph with an input-fraction matrix ``A``.
|
|
15
|
+
|
|
16
|
+
``A[i, j]`` is the fraction of the inputs of ``j`` that come from ``i``, which is what the
|
|
17
|
+
traversal model of Schlegel et al. (2021) turns into an edge probability.
|
|
18
|
+
|
|
19
|
+
Use :meth:`from_edges` to build it from a DataFrame (or arrays) of ``source, target, weight``.
|
|
20
|
+
Node IDs can be any hashable type; internally they are mapped to ``0..n-1``.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
def __init__(self, A: sparse.csr_matrix, ids: np.ndarray):
|
|
24
|
+
A = sparse.csr_matrix(A, dtype=np.float64)
|
|
25
|
+
if A.shape[0] != A.shape[1] or A.shape[0] != len(ids):
|
|
26
|
+
raise ValueError("A must be square with one row per id")
|
|
27
|
+
self.A = A
|
|
28
|
+
self.ids = np.asarray(ids)
|
|
29
|
+
self._index = None
|
|
30
|
+
|
|
31
|
+
# ------------------------------------------------------------------ build
|
|
32
|
+
@classmethod
|
|
33
|
+
def from_edges(cls, edges: pd.DataFrame | None = None, *, source: str = "source", target: str = "target",
|
|
34
|
+
weight: str | None = "weight", normalize: bool = True, min_weight: float | None = None,
|
|
35
|
+
sources=None, targets=None, weights=None) -> "Graph":
|
|
36
|
+
"""Build a graph from an edge list.
|
|
37
|
+
|
|
38
|
+
Parameters
|
|
39
|
+
----------
|
|
40
|
+
edges : DataFrame, optional
|
|
41
|
+
Edge list with ``source``, ``target`` and (optionally) ``weight`` columns. Alternatively pass
|
|
42
|
+
``sources``, ``targets`` and ``weights`` as arrays.
|
|
43
|
+
weight : str or None
|
|
44
|
+
Weight column. ``None`` gives every edge weight 1.
|
|
45
|
+
normalize : bool
|
|
46
|
+
If True (default), weights are synapse counts (or any non-negative strength) and are divided by
|
|
47
|
+
the total input of each target, giving input fractions. If False, weights are used as they are
|
|
48
|
+
(they should already be input fractions, as in ``navis.models.TraversalModel``).
|
|
49
|
+
min_weight : float, optional
|
|
50
|
+
Drop edges with weight below this value *before* normalizing (e.g. ``min_weight=5`` for the
|
|
51
|
+
usual 5-synapse threshold). The threshold changes the layers noticeably, so report it.
|
|
52
|
+
|
|
53
|
+
Duplicate edges are summed; self-loops are dropped (they never change a traversal).
|
|
54
|
+
"""
|
|
55
|
+
if edges is not None:
|
|
56
|
+
if not isinstance(edges, pd.DataFrame):
|
|
57
|
+
raise TypeError(f"edges must be a pandas DataFrame, got {type(edges)}")
|
|
58
|
+
for c in (source, target) + ((weight,) if weight else ()):
|
|
59
|
+
if c not in edges.columns:
|
|
60
|
+
raise KeyError(f'edges has no "{c}" column')
|
|
61
|
+
sources = edges[source].to_numpy()
|
|
62
|
+
targets = edges[target].to_numpy()
|
|
63
|
+
weights = edges[weight].to_numpy(dtype=np.float64) if weight else None
|
|
64
|
+
if sources is None or targets is None:
|
|
65
|
+
raise ValueError("pass an edge DataFrame or sources and targets arrays")
|
|
66
|
+
sources, targets = np.asarray(sources), np.asarray(targets)
|
|
67
|
+
weights = np.ones(len(sources)) if weights is None else np.asarray(weights, dtype=np.float64)
|
|
68
|
+
if not (len(sources) == len(targets) == len(weights)):
|
|
69
|
+
raise ValueError("sources, targets and weights must have the same length")
|
|
70
|
+
if np.any(weights < 0):
|
|
71
|
+
raise ValueError("weights must be non-negative")
|
|
72
|
+
|
|
73
|
+
keep = sources != targets
|
|
74
|
+
if min_weight is not None:
|
|
75
|
+
keep &= weights >= min_weight
|
|
76
|
+
sources, targets, weights = sources[keep], targets[keep], weights[keep]
|
|
77
|
+
|
|
78
|
+
ids, inv = np.unique(np.concatenate([sources, targets]), return_inverse=True)
|
|
79
|
+
n, m = len(ids), len(sources)
|
|
80
|
+
i, j = inv[:m], inv[m:]
|
|
81
|
+
W = sparse.csr_matrix((weights, (i, j)), shape=(n, n)) # duplicates are summed
|
|
82
|
+
W.eliminate_zeros()
|
|
83
|
+
if normalize:
|
|
84
|
+
W = W.tocsc()
|
|
85
|
+
entradas = np.asarray(W.sum(axis=0)).ravel()
|
|
86
|
+
inv_in = np.divide(1.0, entradas, out=np.zeros_like(entradas), where=entradas > 0)
|
|
87
|
+
W = sparse.csr_matrix(W @ sparse.diags(inv_in))
|
|
88
|
+
elif W.nnz and W.data.max() > 1:
|
|
89
|
+
warnings.warn("normalize=False but some weights are > 1: they should be input fractions", stacklevel=2)
|
|
90
|
+
return cls(W, ids)
|
|
91
|
+
|
|
92
|
+
# ------------------------------------------------------------------ ids
|
|
93
|
+
@property
|
|
94
|
+
def n_nodes(self) -> int:
|
|
95
|
+
return len(self.ids)
|
|
96
|
+
|
|
97
|
+
@property
|
|
98
|
+
def n_edges(self) -> int:
|
|
99
|
+
return int(self.A.nnz)
|
|
100
|
+
|
|
101
|
+
def index(self, nodes, missing: str = "warn") -> np.ndarray:
|
|
102
|
+
"""Indices of ``nodes`` in the graph. ``missing``: "warn" (drop with a warning), "raise" or "ignore"."""
|
|
103
|
+
nodes = np.unique(np.asarray(list(nodes) if not isinstance(nodes, np.ndarray) else nodes))
|
|
104
|
+
pos = np.searchsorted(self.ids, nodes)
|
|
105
|
+
pos = np.clip(pos, 0, max(len(self.ids) - 1, 0))
|
|
106
|
+
found = (len(self.ids) > 0) & (self.ids[pos] == nodes) if len(nodes) else np.zeros(0, bool)
|
|
107
|
+
if not found.all():
|
|
108
|
+
n_miss = int((~found).sum())
|
|
109
|
+
if missing == "raise":
|
|
110
|
+
raise KeyError(f"{n_miss} node(s) are not in the graph")
|
|
111
|
+
if missing == "warn":
|
|
112
|
+
warnings.warn(f"{n_miss} of {len(nodes)} node(s) are not in the graph and were dropped", stacklevel=3)
|
|
113
|
+
return pos[found]
|
|
114
|
+
|
|
115
|
+
def __repr__(self) -> str:
|
|
116
|
+
return f"<flowlayers.Graph: {self.n_nodes:,} nodes, {self.n_edges:,} edges>"
|
|
@@ -0,0 +1,263 @@
|
|
|
1
|
+
"""Probabilistic traversal model (Schlegel et al. 2021), vectorized on CPU and GPU."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import time
|
|
5
|
+
import warnings
|
|
6
|
+
from dataclasses import dataclass, field
|
|
7
|
+
|
|
8
|
+
import numpy as np
|
|
9
|
+
import pandas as pd
|
|
10
|
+
from scipy import sparse
|
|
11
|
+
|
|
12
|
+
from .graph import Graph
|
|
13
|
+
|
|
14
|
+
__all__ = ["TraversalModel", "TraversalResult", "traverse", "linear_p"]
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def linear_p(w, min_w: float = 0.0, max_w: float = 0.3):
|
|
18
|
+
"""Edge probability: 0 at ``min_w`` rising linearly to 1 at ``max_w`` (the ``navis`` default)."""
|
|
19
|
+
return np.clip((np.asarray(w, dtype=np.float64) - min_w) / (max_w - min_w), 0.0, 1.0)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _backend(device: str):
|
|
23
|
+
"""Array and sparse modules for ``device`` ("cpu", "gpu" or "auto")."""
|
|
24
|
+
if device not in ("cpu", "gpu", "auto"):
|
|
25
|
+
raise ValueError('device must be "cpu", "gpu" or "auto"')
|
|
26
|
+
if device in ("gpu", "auto"):
|
|
27
|
+
try:
|
|
28
|
+
import cupy as cp
|
|
29
|
+
import cupyx.scipy.sparse as cps
|
|
30
|
+
cp.cuda.runtime.getDeviceCount()
|
|
31
|
+
return cp, cps, "gpu"
|
|
32
|
+
except Exception as exc: # no CuPy, no driver or no device
|
|
33
|
+
if device == "gpu":
|
|
34
|
+
raise RuntimeError(f"device='gpu' needs CuPy and a CUDA device: {exc}") from exc
|
|
35
|
+
return np, sparse, "cpu"
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@dataclass
|
|
39
|
+
class TraversalResult:
|
|
40
|
+
"""First-traversal step of every node over all runs.
|
|
41
|
+
|
|
42
|
+
``counts[k, t]`` is the number of runs in which node ``k`` was first reached at step ``t``
|
|
43
|
+
(``t = 1`` are the seeds; column 0 is unused). Everything else is derived from it.
|
|
44
|
+
"""
|
|
45
|
+
|
|
46
|
+
ids: np.ndarray
|
|
47
|
+
counts: np.ndarray
|
|
48
|
+
runs: int
|
|
49
|
+
seeds: np.ndarray
|
|
50
|
+
max_steps: int
|
|
51
|
+
change: float = float("nan")
|
|
52
|
+
seconds: float = float("nan")
|
|
53
|
+
device: str = "cpu"
|
|
54
|
+
target_runs: pd.DataFrame | None = field(default=None, repr=False)
|
|
55
|
+
|
|
56
|
+
# -------------------------------------------------------------- per node
|
|
57
|
+
@property
|
|
58
|
+
def n_reached(self) -> np.ndarray:
|
|
59
|
+
return self.counts[:, 1:].sum(axis=1)
|
|
60
|
+
|
|
61
|
+
@property
|
|
62
|
+
def reached(self) -> pd.Series:
|
|
63
|
+
"""Fraction of runs in which each node was reached."""
|
|
64
|
+
return pd.Series(self.n_reached / self.runs, index=self.ids, name="reached")
|
|
65
|
+
|
|
66
|
+
@property
|
|
67
|
+
def layer_mean(self) -> pd.Series:
|
|
68
|
+
"""Mean first-traversal step over the runs that reached the node (NaN if never reached)."""
|
|
69
|
+
pasos = np.arange(self.counts.shape[1])
|
|
70
|
+
n = self.n_reached
|
|
71
|
+
media = np.divide(self.counts @ pasos, n, out=np.full(len(n), np.nan), where=n > 0)
|
|
72
|
+
return pd.Series(media, index=self.ids, name="layer_mean")
|
|
73
|
+
|
|
74
|
+
def cmf(self, conditional: bool = False) -> pd.DataFrame:
|
|
75
|
+
"""Cumulative probability of having been reached by each step (columns 1..max_steps).
|
|
76
|
+
|
|
77
|
+
With ``conditional=True`` it is conditioned on the node being reached at all."""
|
|
78
|
+
c = np.cumsum(self.counts[:, 1:], axis=1).astype(np.float64)
|
|
79
|
+
den = self.n_reached[:, None] if conditional else self.runs
|
|
80
|
+
c = np.divide(c, den, out=np.zeros_like(c), where=np.broadcast_to(den, c.shape) > 0)
|
|
81
|
+
return pd.DataFrame(c, index=self.ids, columns=np.arange(1, self.counts.shape[1]))
|
|
82
|
+
|
|
83
|
+
@property
|
|
84
|
+
def summary(self) -> pd.DataFrame:
|
|
85
|
+
"""``layer_min``, ``layer_max``, ``layer_mean``, ``layer_median`` and ``reached`` for every node
|
|
86
|
+
reached at least once: the columns of ``navis.models.TraversalModel.summary``."""
|
|
87
|
+
n = self.n_reached
|
|
88
|
+
ok = n > 0
|
|
89
|
+
c = self.counts[ok, 1:]
|
|
90
|
+
pasos = np.arange(1, self.counts.shape[1])
|
|
91
|
+
hay = c > 0
|
|
92
|
+
layer_min = pasos[hay.argmax(axis=1)]
|
|
93
|
+
layer_max = pasos[c.shape[1] - 1 - hay[:, ::-1].argmax(axis=1)]
|
|
94
|
+
acum = np.cumsum(c, axis=1)
|
|
95
|
+
half = n[ok] / 2.0
|
|
96
|
+
lo = pasos[(acum >= half[:, None]).argmax(axis=1)] # lower median
|
|
97
|
+
hi = pasos[(acum > half[:, None]).argmax(axis=1)] # upper median
|
|
98
|
+
median = np.where(n[ok] % 2 == 0, (lo + hi) / 2.0, hi).astype(np.float64)
|
|
99
|
+
return pd.DataFrame({"layer_min": layer_min, "layer_max": layer_max,
|
|
100
|
+
"layer_mean": self.layer_mean.to_numpy()[ok], "layer_median": median,
|
|
101
|
+
"reached": n[ok] / self.runs}, index=pd.Index(self.ids[ok], name="node"))
|
|
102
|
+
|
|
103
|
+
def __repr__(self) -> str:
|
|
104
|
+
return (f"<flowlayers.TraversalResult: {len(self.ids):,} nodes, {int((self.n_reached > 0).sum()):,} reached, "
|
|
105
|
+
f"{self.runs:,} runs on {self.device}, {self.seconds:.1f} s>")
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
class TraversalModel:
|
|
109
|
+
"""The traversal model of Schlegel et al. (2021), as in ``navis.models.TraversalModel``.
|
|
110
|
+
|
|
111
|
+
Seeds are layer 1. At each step, every edge ``i -> j`` from a traversed node ``i`` to a node ``j`` that
|
|
112
|
+
is not yet traversed succeeds with probability ``p_ij = clip((A_ij - min_w) / (max_w - min_w), 0, 1)``;
|
|
113
|
+
``j`` is traversed if any of its edges succeeds and gets that step as its layer. Nodes that fail are
|
|
114
|
+
tried again at later steps. A run ends when no candidate edges are left or at ``max_steps``.
|
|
115
|
+
|
|
116
|
+
Drawing once per node with ``P_j = 1 - prod_i (1 - p_ij)`` has the same distribution as drawing per
|
|
117
|
+
edge, and turns each step of a batch of runs into one sparse-times-dense product.
|
|
118
|
+
|
|
119
|
+
Parameters
|
|
120
|
+
----------
|
|
121
|
+
graph : Graph or DataFrame
|
|
122
|
+
A :class:`Graph`, or an edge list passed to :meth:`Graph.from_edges` with ``graph_kwargs``.
|
|
123
|
+
max_w, min_w : float
|
|
124
|
+
Input fraction at which an edge always / never fires (navis: 0.3 and 0).
|
|
125
|
+
max_steps : int
|
|
126
|
+
Step cap per run (navis default 15).
|
|
127
|
+
device : "cpu", "gpu" or "auto"
|
|
128
|
+
"gpu" needs CuPy; "auto" uses the GPU if it is available.
|
|
129
|
+
"""
|
|
130
|
+
|
|
131
|
+
def __init__(self, graph, *, max_w: float = 0.3, min_w: float = 0.0, max_steps: int = 15,
|
|
132
|
+
device: str = "cpu", **graph_kwargs):
|
|
133
|
+
if not isinstance(graph, Graph):
|
|
134
|
+
graph = Graph.from_edges(graph, **graph_kwargs)
|
|
135
|
+
elif graph_kwargs:
|
|
136
|
+
raise TypeError(f"unexpected arguments for an existing Graph: {sorted(graph_kwargs)}")
|
|
137
|
+
if not 0 <= min_w < max_w:
|
|
138
|
+
raise ValueError("need 0 <= min_w < max_w")
|
|
139
|
+
if max_steps < 1:
|
|
140
|
+
raise ValueError("max_steps must be >= 1")
|
|
141
|
+
self.graph, self.max_w, self.min_w, self.max_steps = graph, max_w, min_w, int(max_steps)
|
|
142
|
+
self.xp, self.sp, self.device = _backend(device)
|
|
143
|
+
|
|
144
|
+
A = graph.A
|
|
145
|
+
p = A.copy()
|
|
146
|
+
p.data = linear_p(p.data, min_w, max_w)
|
|
147
|
+
p.eliminate_zeros()
|
|
148
|
+
sure = p.copy()
|
|
149
|
+
sure.data = (sure.data >= 1.0).astype(np.float32)
|
|
150
|
+
sure.eliminate_zeros()
|
|
151
|
+
logq = p.copy()
|
|
152
|
+
logq.data = np.where(p.data < 1.0, np.log1p(-np.minimum(p.data, 1 - 1e-12)), 0.0)
|
|
153
|
+
logq.eliminate_zeros()
|
|
154
|
+
cand = p.copy()
|
|
155
|
+
cand.data = np.ones_like(cand.data, dtype=np.float32)
|
|
156
|
+
# transposed: row = target node; on the GPU the matrices live on the card
|
|
157
|
+
to = (lambda M: self.sp.csr_matrix(sparse.csr_matrix(M.T, dtype=np.float32))) if self.device == "gpu" \
|
|
158
|
+
else (lambda M: sparse.csr_matrix(M.T, dtype=np.float32))
|
|
159
|
+
self._logqT, self._sureT, self._candT = to(logq), to(sure), to(cand)
|
|
160
|
+
|
|
161
|
+
def __repr__(self) -> str:
|
|
162
|
+
return (f"<flowlayers.TraversalModel: {self.graph.n_nodes:,} nodes, {self.graph.n_edges:,} edges, "
|
|
163
|
+
f"max_w={self.max_w}, max_steps={self.max_steps}, device={self.device}>")
|
|
164
|
+
|
|
165
|
+
# -------------------------------------------------------------- one batch
|
|
166
|
+
def _batch(self, seeds_idx, runs: int, rng):
|
|
167
|
+
"""Layers of ``runs`` independent runs, as int16 (n, runs) on the device; 0 = not reached."""
|
|
168
|
+
xp = self.xp
|
|
169
|
+
n = self.graph.n_nodes
|
|
170
|
+
layers = xp.zeros((n, runs), dtype=xp.int16)
|
|
171
|
+
X = xp.zeros((n, runs), dtype=xp.float32)
|
|
172
|
+
layers[seeds_idx, :] = 1
|
|
173
|
+
X[seeds_idx, :] = 1.0
|
|
174
|
+
for t in range(2, self.max_steps + 1):
|
|
175
|
+
pending = (layers == 0) & ((self._candT @ X) > 0)
|
|
176
|
+
if not bool(pending.any()):
|
|
177
|
+
break
|
|
178
|
+
P = -xp.expm1(self._logqT @ X) # 1 - prod(1 - p) over edges with p < 1
|
|
179
|
+
P[(self._sureT @ X) > 0] = 1.0 # any edge with p = 1
|
|
180
|
+
new = pending & (rng.random((n, runs), dtype=xp.float32) < P)
|
|
181
|
+
del P
|
|
182
|
+
layers[new] = t
|
|
183
|
+
X[new] = 1.0
|
|
184
|
+
return layers
|
|
185
|
+
|
|
186
|
+
# -------------------------------------------------------------- run
|
|
187
|
+
def run(self, seeds, *, runs: int | None = None, batch: int = 100, tol: float = 0.01,
|
|
188
|
+
quantile: float = 0.99, max_runs: int = 10_000, seed=None, targets=None,
|
|
189
|
+
missing: str = "warn", progress=None) -> TraversalResult:
|
|
190
|
+
"""Traverse from ``seeds`` and return a :class:`TraversalResult`.
|
|
191
|
+
|
|
192
|
+
Parameters
|
|
193
|
+
----------
|
|
194
|
+
seeds : iterable of node IDs
|
|
195
|
+
Seeds missing from the graph are dropped with a warning (``missing``). Unlike navis, seeds
|
|
196
|
+
without outgoing edges are kept (they are layer 1 and reach nothing).
|
|
197
|
+
runs : int, optional
|
|
198
|
+
Exact number of runs. If None (default), batches of ``batch`` runs are added until the mean
|
|
199
|
+
layer changes by less than ``tol`` in the ``quantile`` of nodes (at least 5 batches), up to
|
|
200
|
+
``max_runs``.
|
|
201
|
+
seed : int or numpy/cupy Generator, optional
|
|
202
|
+
Random seed for reproducibility.
|
|
203
|
+
targets : iterable of node IDs, optional
|
|
204
|
+
Keep the layer of these nodes in every run (``result.target_runs``), e.g. for bootstraps.
|
|
205
|
+
progress : callable, optional
|
|
206
|
+
Called as ``progress(runs_done, change)`` after every batch.
|
|
207
|
+
"""
|
|
208
|
+
xp = self.xp
|
|
209
|
+
t0 = time.time()
|
|
210
|
+
idx = self.graph.index(seeds, missing=missing)
|
|
211
|
+
if len(idx) == 0:
|
|
212
|
+
raise ValueError("none of the seeds are in the graph")
|
|
213
|
+
seeds_idx = xp.asarray(idx)
|
|
214
|
+
tgt_idx = self.graph.index(targets, missing=missing) if targets is not None else None
|
|
215
|
+
rng = seed if hasattr(seed, "random") else xp.random.default_rng(seed)
|
|
216
|
+
n, S = self.graph.n_nodes, self.max_steps
|
|
217
|
+
counts = xp.zeros((n, S + 1), dtype=xp.int64)
|
|
218
|
+
kept = []
|
|
219
|
+
fixed = runs is not None
|
|
220
|
+
total = int(runs) if fixed else int(max_runs)
|
|
221
|
+
done, change, prev = 0, float("inf"), None
|
|
222
|
+
pasos = xp.arange(S + 1)
|
|
223
|
+
while done < total:
|
|
224
|
+
k = min(batch, total - done)
|
|
225
|
+
L = self._batch(seeds_idx, k, rng)
|
|
226
|
+
for t in range(1, S + 1):
|
|
227
|
+
counts[:, t] += (L == t).sum(axis=1)
|
|
228
|
+
if tgt_idx is not None:
|
|
229
|
+
kept.append(L[xp.asarray(tgt_idx)].get() if xp is not np else L[tgt_idx].copy())
|
|
230
|
+
del L
|
|
231
|
+
done += k
|
|
232
|
+
if not fixed:
|
|
233
|
+
c = counts[:, 1:].sum(axis=1)
|
|
234
|
+
mean = xp.where(c > 0, (counts @ pasos) / xp.maximum(c, 1), xp.nan)
|
|
235
|
+
mean = mean.get() if xp is not np else mean
|
|
236
|
+
if prev is not None:
|
|
237
|
+
ok = ~np.isnan(mean) & ~np.isnan(prev)
|
|
238
|
+
change = float(np.quantile(np.abs(mean[ok] - prev[ok]), quantile)) if ok.any() else 0.0
|
|
239
|
+
prev = mean
|
|
240
|
+
if progress:
|
|
241
|
+
progress(done, change)
|
|
242
|
+
if not fixed and done >= 5 * batch and change < tol:
|
|
243
|
+
break
|
|
244
|
+
counts = counts.get() if xp is not np else counts
|
|
245
|
+
if xp is not np:
|
|
246
|
+
xp.get_default_memory_pool().free_all_blocks()
|
|
247
|
+
if not fixed and change >= tol:
|
|
248
|
+
warnings.warn(f"did not converge in {done:,} runs (change {change:.4f} >= tol {tol})", stacklevel=2)
|
|
249
|
+
target_runs = None
|
|
250
|
+
if tgt_idx is not None:
|
|
251
|
+
target_runs = pd.DataFrame(np.concatenate(kept, axis=1).T, columns=self.graph.ids[tgt_idx])
|
|
252
|
+
target_runs.index.name = "run"
|
|
253
|
+
return TraversalResult(ids=self.graph.ids, counts=counts, runs=done, seeds=self.graph.ids[idx],
|
|
254
|
+
max_steps=S, change=change, seconds=time.time() - t0, device=self.device,
|
|
255
|
+
target_runs=target_runs)
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
def traverse(edges, seeds, *, max_w: float = 0.3, min_w: float = 0.0, max_steps: int = 15,
|
|
259
|
+
device: str = "cpu", graph_kwargs: dict | None = None, **run_kwargs) -> TraversalResult:
|
|
260
|
+
"""One-call shortcut: ``TraversalModel(edges, ...).run(seeds, ...)``."""
|
|
261
|
+
model = TraversalModel(edges, max_w=max_w, min_w=min_w, max_steps=max_steps, device=device,
|
|
262
|
+
**(graph_kwargs or {}))
|
|
263
|
+
return model.run(seeds, **run_kwargs)
|
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
import numpy as np
|
|
2
|
+
import pandas as pd
|
|
3
|
+
import pytest
|
|
4
|
+
|
|
5
|
+
import flowlayers as fl
|
|
6
|
+
from flowlayers.exact import exact_layers
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def edges_from_P(P):
|
|
10
|
+
i, j = np.nonzero(P)
|
|
11
|
+
# with max_w = 1 and normalize=False, the weight is the edge probability
|
|
12
|
+
return pd.DataFrame({"source": i, "target": j, "weight": P[i, j]})
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def mc_vs_exact(P, seeds, runs=60_000, max_steps=15, seed=0):
|
|
16
|
+
model = fl.TraversalModel(edges_from_P(P), normalize=False, max_w=1.0, max_steps=max_steps)
|
|
17
|
+
res = model.run(seeds, runs=runs, seed=seed, batch=5_000)
|
|
18
|
+
ex = exact_layers(P, seeds, max_steps)
|
|
19
|
+
return res, ex
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
# ------------------------------------------------------------------ exactness
|
|
23
|
+
def test_diamond_issue_194():
|
|
24
|
+
"""navis issue #194: diamond 0->1, 0->2, 1->3, 2->3, p = 0.5, seed 0. Analytic answer 3.963."""
|
|
25
|
+
P = np.zeros((4, 4))
|
|
26
|
+
for a, b in [(0, 1), (0, 2), (1, 3), (2, 3)]:
|
|
27
|
+
P[a, b] = 0.5
|
|
28
|
+
res, ex = mc_vs_exact(P, [0], runs=200_000, max_steps=40)
|
|
29
|
+
pasos = np.arange(ex.shape[1])
|
|
30
|
+
exact_mean = (ex[3] @ pasos) / ex[3, 1:].sum()
|
|
31
|
+
assert abs(exact_mean - 3.963) < 0.001
|
|
32
|
+
se = np.sqrt(((pasos - exact_mean) ** 2 @ ex[3]) / ex[3, 1:].sum() / res.runs)
|
|
33
|
+
assert abs(res.layer_mean[3] - exact_mean) < 4 * se
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@pytest.mark.parametrize("seed", [1, 2, 3, 4])
|
|
37
|
+
def test_random_small_graphs_match_exact(seed):
|
|
38
|
+
rng = np.random.default_rng(seed)
|
|
39
|
+
n = 7
|
|
40
|
+
P = np.where(rng.random((n, n)) < 0.35, rng.uniform(0.05, 1.0, (n, n)), 0.0)
|
|
41
|
+
np.fill_diagonal(P, 0)
|
|
42
|
+
P[(P > 0) & (rng.random((n, n)) < 0.2)] = 1.0 # some edges that always fire
|
|
43
|
+
np.fill_diagonal(P, 0)
|
|
44
|
+
res, ex = mc_vs_exact(P, [0], runs=60_000, max_steps=12)
|
|
45
|
+
reach_exact = ex[:, 1:].sum(axis=1)
|
|
46
|
+
np.testing.assert_allclose(res.reached.to_numpy(), reach_exact, atol=0.01)
|
|
47
|
+
# full first-step distribution, node by node
|
|
48
|
+
got = res.counts / res.runs
|
|
49
|
+
np.testing.assert_allclose(got[:, 1:], ex[:, 1:], atol=0.01)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def test_sure_edges_are_deterministic():
|
|
53
|
+
P = np.zeros((5, 5))
|
|
54
|
+
for a in range(4):
|
|
55
|
+
P[a, a + 1] = 1.0
|
|
56
|
+
res, _ = mc_vs_exact(P, [0], runs=50)
|
|
57
|
+
np.testing.assert_array_equal(res.layer_mean.to_numpy(), [1, 2, 3, 4, 5])
|
|
58
|
+
assert (res.reached == 1).all()
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def test_max_steps_caps_layers():
|
|
62
|
+
P = np.zeros((6, 6))
|
|
63
|
+
for a in range(5):
|
|
64
|
+
P[a, a + 1] = 1.0
|
|
65
|
+
model = fl.TraversalModel(edges_from_P(P), normalize=False, max_w=1.0, max_steps=3)
|
|
66
|
+
res = model.run([0], runs=10)
|
|
67
|
+
assert res.layer_mean.max() == 3
|
|
68
|
+
assert np.isnan(res.layer_mean[5])
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
# ------------------------------------------------------------------ graph
|
|
72
|
+
def test_normalize_and_threshold():
|
|
73
|
+
edges = pd.DataFrame({"pre": ["a", "b", "c", "a"], "post": ["d", "d", "d", "d"], "syn": [6, 2, 12, 4]})
|
|
74
|
+
g = fl.Graph.from_edges(edges, source="pre", target="post", weight="syn")
|
|
75
|
+
d = list(g.ids).index("d")
|
|
76
|
+
assert np.isclose(g.A[:, d].sum(), 1.0) # duplicates summed, columns sum to 1
|
|
77
|
+
assert np.isclose(g.A[list(g.ids).index("a"), d], 10 / 24)
|
|
78
|
+
g5 = fl.Graph.from_edges(edges, source="pre", target="post", weight="syn", min_weight=5)
|
|
79
|
+
assert g5.n_edges == 2 # b->d (2) and a->d (4) dropped before summing
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def test_string_ids_and_missing_seeds():
|
|
83
|
+
edges = pd.DataFrame({"source": ["x", "y"], "target": ["y", "z"], "weight": [1.0, 1.0]})
|
|
84
|
+
model = fl.TraversalModel(edges)
|
|
85
|
+
with pytest.warns(UserWarning, match="not in the graph"):
|
|
86
|
+
res = model.run(["x", "nope"], runs=20)
|
|
87
|
+
assert list(res.seeds) == ["x"]
|
|
88
|
+
assert res.layer_mean["z"] == 3
|
|
89
|
+
with pytest.raises(ValueError):
|
|
90
|
+
model.run(["nope"], runs=5, missing="ignore")
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def test_seed_without_out_edges_is_kept():
|
|
94
|
+
"""navis drops seeds that are never a source; here they stay as layer 1."""
|
|
95
|
+
edges = pd.DataFrame({"source": [0], "target": [1], "weight": [1.0]})
|
|
96
|
+
res = fl.TraversalModel(edges).run([0, 1], runs=5)
|
|
97
|
+
assert set(res.seeds) == {0, 1}
|
|
98
|
+
assert res.layer_mean[1] == 1
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
# ------------------------------------------------------------------ results
|
|
102
|
+
def test_summary_matches_target_runs():
|
|
103
|
+
rng = np.random.default_rng(7)
|
|
104
|
+
n, m = 300, 2_000
|
|
105
|
+
edges = pd.DataFrame({"source": rng.integers(0, n, m), "target": rng.integers(0, n, m),
|
|
106
|
+
"weight": rng.integers(1, 20, m)})
|
|
107
|
+
model = fl.TraversalModel(edges, max_steps=20)
|
|
108
|
+
res = model.run(np.arange(5), runs=400, seed=1, targets=np.arange(n))
|
|
109
|
+
lay = res.target_runs.replace(0, np.nan)
|
|
110
|
+
s = res.summary
|
|
111
|
+
common = s.index.intersection(lay.columns)
|
|
112
|
+
np.testing.assert_allclose(s.loc[common, "layer_mean"], lay[common].mean(), rtol=1e-9)
|
|
113
|
+
np.testing.assert_allclose(s.loc[common, "layer_median"], lay[common].median(), rtol=1e-9)
|
|
114
|
+
np.testing.assert_allclose(s.loc[common, "layer_min"], lay[common].min())
|
|
115
|
+
np.testing.assert_allclose(s.loc[common, "layer_max"], lay[common].max())
|
|
116
|
+
np.testing.assert_allclose(s.loc[common, "reached"], lay[common].notna().mean())
|
|
117
|
+
cmf = res.cmf()
|
|
118
|
+
assert np.all(np.diff(cmf.to_numpy(), axis=1) >= -1e-12)
|
|
119
|
+
np.testing.assert_allclose(cmf.iloc[:, -1].to_numpy(), res.reached.to_numpy())
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def test_reproducible_and_convergence():
|
|
123
|
+
rng = np.random.default_rng(3)
|
|
124
|
+
n, m = 500, 5_000
|
|
125
|
+
edges = pd.DataFrame({"source": rng.integers(0, n, m), "target": rng.integers(0, n, m),
|
|
126
|
+
"weight": rng.integers(1, 20, m)})
|
|
127
|
+
model = fl.TraversalModel(edges)
|
|
128
|
+
a = model.run(np.arange(10), runs=200, seed=42)
|
|
129
|
+
b = model.run(np.arange(10), runs=200, seed=42)
|
|
130
|
+
np.testing.assert_array_equal(a.counts, b.counts)
|
|
131
|
+
c = model.run(np.arange(10), seed=0, tol=0.02)
|
|
132
|
+
assert c.runs >= 500 and c.runs % 100 == 0 and c.change < 0.02
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def test_traverse_shortcut():
|
|
136
|
+
edges = pd.DataFrame({"source": [0, 1], "target": [1, 2], "weight": [1.0, 1.0]})
|
|
137
|
+
res = fl.traverse(edges, [0], runs=10)
|
|
138
|
+
assert res.layer_mean[2] == 3
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
# ------------------------------------------------------------------ optional backends
|
|
142
|
+
def test_gpu_matches_cpu():
|
|
143
|
+
cp = pytest.importorskip("cupy")
|
|
144
|
+
try:
|
|
145
|
+
cp.cuda.runtime.getDeviceCount()
|
|
146
|
+
except Exception:
|
|
147
|
+
pytest.skip("no CUDA device")
|
|
148
|
+
rng = np.random.default_rng(5)
|
|
149
|
+
n, m = 3_000, 40_000
|
|
150
|
+
edges = pd.DataFrame({"source": rng.integers(0, n, m), "target": rng.integers(0, n, m),
|
|
151
|
+
"weight": rng.integers(1, 20, m)})
|
|
152
|
+
cpu = fl.TraversalModel(edges, device="cpu").run(np.arange(20), runs=4_000, seed=1)
|
|
153
|
+
gpu = fl.TraversalModel(edges, device="gpu").run(np.arange(20), runs=4_000, seed=1)
|
|
154
|
+
x, y = cpu.layer_mean.to_numpy(), gpu.layer_mean.to_numpy()
|
|
155
|
+
ok = ~np.isnan(x) & ~np.isnan(y)
|
|
156
|
+
assert np.corrcoef(x[ok], y[ok])[0, 1] > 0.99
|
|
157
|
+
assert abs(np.mean(y[ok] - x[ok])) < 0.01
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def test_same_distribution_as_navis():
|
|
161
|
+
nm = pytest.importorskip("navis.models")
|
|
162
|
+
import navis
|
|
163
|
+
navis.config.pbar_hide = True
|
|
164
|
+
rng = np.random.default_rng(11)
|
|
165
|
+
n, m = 1_500, 15_000
|
|
166
|
+
raw = pd.DataFrame({"source": rng.integers(0, n, m), "target": rng.integers(0, n, m),
|
|
167
|
+
"weight": rng.integers(1, 20, m)})
|
|
168
|
+
g = fl.Graph.from_edges(raw)
|
|
169
|
+
A = g.A.tocoo()
|
|
170
|
+
edges = pd.DataFrame({"source": g.ids[A.row], "target": g.ids[A.col], "weight": A.data})
|
|
171
|
+
seeds = np.arange(10)
|
|
172
|
+
ref = nm.TraversalModel(edges, seeds=seeds, max_steps=15)
|
|
173
|
+
ref.run(iterations=1_500)
|
|
174
|
+
ours = fl.TraversalModel(g, max_steps=15).run(seeds, runs=1_500, seed=2)
|
|
175
|
+
s_ref = ref.summary.set_index("node") if "node" in ref.summary.columns else ref.summary
|
|
176
|
+
common = s_ref.index.intersection(ours.summary.index)
|
|
177
|
+
d = ours.summary.loc[common, "layer_mean"] - s_ref.loc[common, "layer_mean"]
|
|
178
|
+
assert abs(d.mean()) < 0.01 # no bias
|
|
179
|
+
assert np.abs(d).mean() < 0.05 # differences are sampling noise
|
|
180
|
+
assert set(s_ref.index) <= set(ours.summary.index)
|