flowlayers 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
flowlayers/__init__.py ADDED
@@ -0,0 +1,6 @@
1
+ """flowlayers: fast probabilistic traversal of connectomes (Schlegel et al. 2021), on CPU or GPU."""
2
+ from .graph import Graph
3
+ from .model import TraversalModel, TraversalResult, linear_p, traverse
4
+
5
+ __version__ = "0.1.0"
6
+ __all__ = ["Graph", "TraversalModel", "TraversalResult", "linear_p", "traverse", "__version__"]
flowlayers/exact.py ADDED
@@ -0,0 +1,49 @@
1
+ """Exact first-traversal distribution for very small graphs, by enumerating every set of traversed nodes.
2
+
3
+ Used to test the Monte Carlo engine. The cost grows as 2**n, so keep n <= ~12.
4
+ """
5
+ from __future__ import annotations
6
+
7
+ from itertools import product
8
+
9
+ import numpy as np
10
+
11
+ __all__ = ["exact_layers"]
12
+
13
+
14
+ def exact_layers(P: np.ndarray, seeds, max_steps: int = 15) -> np.ndarray:
15
+ """``out[k, t]`` = probability that node ``k`` is first reached at step ``t`` (``t = 1..max_steps``).
16
+
17
+ ``P[i, j]`` is the edge probability ``i -> j`` (0 if there is no edge). Same rules as the model:
18
+ at each step every untraversed node ``j`` is reached with probability ``1 - prod_i (1 - P[i, j])``
19
+ over the traversed ``i``, independently of the other nodes; failed nodes are retried.
20
+ """
21
+ P = np.asarray(P, dtype=np.float64)
22
+ n = P.shape[0]
23
+ start = frozenset(int(s) for s in seeds)
24
+ out = np.zeros((n, max_steps + 1))
25
+ for s in start:
26
+ out[s, 1] = 1.0
27
+ states = {start: 1.0}
28
+ for t in range(2, max_steps + 1):
29
+ nxt: dict[frozenset, float] = {}
30
+ for S, pr in states.items():
31
+ cand = [j for j in range(n) if j not in S]
32
+ pj = [1.0 - np.prod([1.0 - P[i, j] for i in S]) for j in cand]
33
+ live = [(j, q) for j, q in zip(cand, pj) if q > 0]
34
+ if not live:
35
+ nxt[S] = nxt.get(S, 0.0) + pr
36
+ continue
37
+ for hits in product((0, 1), repeat=len(live)):
38
+ w = pr
39
+ for (j, q), h in zip(live, hits):
40
+ w *= q if h else 1.0 - q
41
+ if w == 0.0:
42
+ continue
43
+ new = [j for (j, _), h in zip(live, hits) if h]
44
+ for j in new:
45
+ out[j, t] += w
46
+ T = S | frozenset(new)
47
+ nxt[T] = nxt.get(T, 0.0) + w
48
+ states = nxt
49
+ return out
flowlayers/graph.py ADDED
@@ -0,0 +1,116 @@
1
+ """Turn an edge list into the sparse matrices the traversal needs."""
2
+ from __future__ import annotations
3
+
4
+ import warnings
5
+
6
+ import numpy as np
7
+ import pandas as pd
8
+ from scipy import sparse
9
+
10
+ __all__ = ["Graph"]
11
+
12
+
13
+ class Graph:
14
+ """A directed, weighted graph with an input-fraction matrix ``A``.
15
+
16
+ ``A[i, j]`` is the fraction of the inputs of ``j`` that come from ``i``, which is what the
17
+ traversal model of Schlegel et al. (2021) turns into an edge probability.
18
+
19
+ Use :meth:`from_edges` to build it from a DataFrame (or arrays) of ``source, target, weight``.
20
+ Node IDs can be any hashable type; internally they are mapped to ``0..n-1``.
21
+ """
22
+
23
+ def __init__(self, A: sparse.csr_matrix, ids: np.ndarray):
24
+ A = sparse.csr_matrix(A, dtype=np.float64)
25
+ if A.shape[0] != A.shape[1] or A.shape[0] != len(ids):
26
+ raise ValueError("A must be square with one row per id")
27
+ self.A = A
28
+ self.ids = np.asarray(ids)
29
+ self._index = None
30
+
31
+ # ------------------------------------------------------------------ build
32
+ @classmethod
33
+ def from_edges(cls, edges: pd.DataFrame | None = None, *, source: str = "source", target: str = "target",
34
+ weight: str | None = "weight", normalize: bool = True, min_weight: float | None = None,
35
+ sources=None, targets=None, weights=None) -> "Graph":
36
+ """Build a graph from an edge list.
37
+
38
+ Parameters
39
+ ----------
40
+ edges : DataFrame, optional
41
+ Edge list with ``source``, ``target`` and (optionally) ``weight`` columns. Alternatively pass
42
+ ``sources``, ``targets`` and ``weights`` as arrays.
43
+ weight : str or None
44
+ Weight column. ``None`` gives every edge weight 1.
45
+ normalize : bool
46
+ If True (default), weights are synapse counts (or any non-negative strength) and are divided by
47
+ the total input of each target, giving input fractions. If False, weights are used as they are
48
+ (they should already be input fractions, as in ``navis.models.TraversalModel``).
49
+ min_weight : float, optional
50
+ Drop edges with weight below this value *before* normalizing (e.g. ``min_weight=5`` for the
51
+ usual 5-synapse threshold). The threshold changes the layers noticeably, so report it.
52
+
53
+ Duplicate edges are summed; self-loops are dropped (they never change a traversal).
54
+ """
55
+ if edges is not None:
56
+ if not isinstance(edges, pd.DataFrame):
57
+ raise TypeError(f"edges must be a pandas DataFrame, got {type(edges)}")
58
+ for c in (source, target) + ((weight,) if weight else ()):
59
+ if c not in edges.columns:
60
+ raise KeyError(f'edges has no "{c}" column')
61
+ sources = edges[source].to_numpy()
62
+ targets = edges[target].to_numpy()
63
+ weights = edges[weight].to_numpy(dtype=np.float64) if weight else None
64
+ if sources is None or targets is None:
65
+ raise ValueError("pass an edge DataFrame or sources and targets arrays")
66
+ sources, targets = np.asarray(sources), np.asarray(targets)
67
+ weights = np.ones(len(sources)) if weights is None else np.asarray(weights, dtype=np.float64)
68
+ if not (len(sources) == len(targets) == len(weights)):
69
+ raise ValueError("sources, targets and weights must have the same length")
70
+ if np.any(weights < 0):
71
+ raise ValueError("weights must be non-negative")
72
+
73
+ keep = sources != targets
74
+ if min_weight is not None:
75
+ keep &= weights >= min_weight
76
+ sources, targets, weights = sources[keep], targets[keep], weights[keep]
77
+
78
+ ids, inv = np.unique(np.concatenate([sources, targets]), return_inverse=True)
79
+ n, m = len(ids), len(sources)
80
+ i, j = inv[:m], inv[m:]
81
+ W = sparse.csr_matrix((weights, (i, j)), shape=(n, n)) # duplicates are summed
82
+ W.eliminate_zeros()
83
+ if normalize:
84
+ W = W.tocsc()
85
+ entradas = np.asarray(W.sum(axis=0)).ravel()
86
+ inv_in = np.divide(1.0, entradas, out=np.zeros_like(entradas), where=entradas > 0)
87
+ W = sparse.csr_matrix(W @ sparse.diags(inv_in))
88
+ elif W.nnz and W.data.max() > 1:
89
+ warnings.warn("normalize=False but some weights are > 1: they should be input fractions", stacklevel=2)
90
+ return cls(W, ids)
91
+
92
+ # ------------------------------------------------------------------ ids
93
+ @property
94
+ def n_nodes(self) -> int:
95
+ return len(self.ids)
96
+
97
+ @property
98
+ def n_edges(self) -> int:
99
+ return int(self.A.nnz)
100
+
101
+ def index(self, nodes, missing: str = "warn") -> np.ndarray:
102
+ """Indices of ``nodes`` in the graph. ``missing``: "warn" (drop with a warning), "raise" or "ignore"."""
103
+ nodes = np.unique(np.asarray(list(nodes) if not isinstance(nodes, np.ndarray) else nodes))
104
+ pos = np.searchsorted(self.ids, nodes)
105
+ pos = np.clip(pos, 0, max(len(self.ids) - 1, 0))
106
+ found = (len(self.ids) > 0) & (self.ids[pos] == nodes) if len(nodes) else np.zeros(0, bool)
107
+ if not found.all():
108
+ n_miss = int((~found).sum())
109
+ if missing == "raise":
110
+ raise KeyError(f"{n_miss} node(s) are not in the graph")
111
+ if missing == "warn":
112
+ warnings.warn(f"{n_miss} of {len(nodes)} node(s) are not in the graph and were dropped", stacklevel=3)
113
+ return pos[found]
114
+
115
+ def __repr__(self) -> str:
116
+ return f"<flowlayers.Graph: {self.n_nodes:,} nodes, {self.n_edges:,} edges>"
flowlayers/model.py ADDED
@@ -0,0 +1,263 @@
1
+ """Probabilistic traversal model (Schlegel et al. 2021), vectorized on CPU and GPU."""
2
+ from __future__ import annotations
3
+
4
+ import time
5
+ import warnings
6
+ from dataclasses import dataclass, field
7
+
8
+ import numpy as np
9
+ import pandas as pd
10
+ from scipy import sparse
11
+
12
+ from .graph import Graph
13
+
14
+ __all__ = ["TraversalModel", "TraversalResult", "traverse", "linear_p"]
15
+
16
+
17
+ def linear_p(w, min_w: float = 0.0, max_w: float = 0.3):
18
+ """Edge probability: 0 at ``min_w`` rising linearly to 1 at ``max_w`` (the ``navis`` default)."""
19
+ return np.clip((np.asarray(w, dtype=np.float64) - min_w) / (max_w - min_w), 0.0, 1.0)
20
+
21
+
22
+ def _backend(device: str):
23
+ """Array and sparse modules for ``device`` ("cpu", "gpu" or "auto")."""
24
+ if device not in ("cpu", "gpu", "auto"):
25
+ raise ValueError('device must be "cpu", "gpu" or "auto"')
26
+ if device in ("gpu", "auto"):
27
+ try:
28
+ import cupy as cp
29
+ import cupyx.scipy.sparse as cps
30
+ cp.cuda.runtime.getDeviceCount()
31
+ return cp, cps, "gpu"
32
+ except Exception as exc: # no CuPy, no driver or no device
33
+ if device == "gpu":
34
+ raise RuntimeError(f"device='gpu' needs CuPy and a CUDA device: {exc}") from exc
35
+ return np, sparse, "cpu"
36
+
37
+
38
+ @dataclass
39
+ class TraversalResult:
40
+ """First-traversal step of every node over all runs.
41
+
42
+ ``counts[k, t]`` is the number of runs in which node ``k`` was first reached at step ``t``
43
+ (``t = 1`` are the seeds; column 0 is unused). Everything else is derived from it.
44
+ """
45
+
46
+ ids: np.ndarray
47
+ counts: np.ndarray
48
+ runs: int
49
+ seeds: np.ndarray
50
+ max_steps: int
51
+ change: float = float("nan")
52
+ seconds: float = float("nan")
53
+ device: str = "cpu"
54
+ target_runs: pd.DataFrame | None = field(default=None, repr=False)
55
+
56
+ # -------------------------------------------------------------- per node
57
+ @property
58
+ def n_reached(self) -> np.ndarray:
59
+ return self.counts[:, 1:].sum(axis=1)
60
+
61
+ @property
62
+ def reached(self) -> pd.Series:
63
+ """Fraction of runs in which each node was reached."""
64
+ return pd.Series(self.n_reached / self.runs, index=self.ids, name="reached")
65
+
66
+ @property
67
+ def layer_mean(self) -> pd.Series:
68
+ """Mean first-traversal step over the runs that reached the node (NaN if never reached)."""
69
+ pasos = np.arange(self.counts.shape[1])
70
+ n = self.n_reached
71
+ media = np.divide(self.counts @ pasos, n, out=np.full(len(n), np.nan), where=n > 0)
72
+ return pd.Series(media, index=self.ids, name="layer_mean")
73
+
74
+ def cmf(self, conditional: bool = False) -> pd.DataFrame:
75
+ """Cumulative probability of having been reached by each step (columns 1..max_steps).
76
+
77
+ With ``conditional=True`` it is conditioned on the node being reached at all."""
78
+ c = np.cumsum(self.counts[:, 1:], axis=1).astype(np.float64)
79
+ den = self.n_reached[:, None] if conditional else self.runs
80
+ c = np.divide(c, den, out=np.zeros_like(c), where=np.broadcast_to(den, c.shape) > 0)
81
+ return pd.DataFrame(c, index=self.ids, columns=np.arange(1, self.counts.shape[1]))
82
+
83
+ @property
84
+ def summary(self) -> pd.DataFrame:
85
+ """``layer_min``, ``layer_max``, ``layer_mean``, ``layer_median`` and ``reached`` for every node
86
+ reached at least once: the columns of ``navis.models.TraversalModel.summary``."""
87
+ n = self.n_reached
88
+ ok = n > 0
89
+ c = self.counts[ok, 1:]
90
+ pasos = np.arange(1, self.counts.shape[1])
91
+ hay = c > 0
92
+ layer_min = pasos[hay.argmax(axis=1)]
93
+ layer_max = pasos[c.shape[1] - 1 - hay[:, ::-1].argmax(axis=1)]
94
+ acum = np.cumsum(c, axis=1)
95
+ half = n[ok] / 2.0
96
+ lo = pasos[(acum >= half[:, None]).argmax(axis=1)] # lower median
97
+ hi = pasos[(acum > half[:, None]).argmax(axis=1)] # upper median
98
+ median = np.where(n[ok] % 2 == 0, (lo + hi) / 2.0, hi).astype(np.float64)
99
+ return pd.DataFrame({"layer_min": layer_min, "layer_max": layer_max,
100
+ "layer_mean": self.layer_mean.to_numpy()[ok], "layer_median": median,
101
+ "reached": n[ok] / self.runs}, index=pd.Index(self.ids[ok], name="node"))
102
+
103
+ def __repr__(self) -> str:
104
+ return (f"<flowlayers.TraversalResult: {len(self.ids):,} nodes, {int((self.n_reached > 0).sum()):,} reached, "
105
+ f"{self.runs:,} runs on {self.device}, {self.seconds:.1f} s>")
106
+
107
+
108
+ class TraversalModel:
109
+ """The traversal model of Schlegel et al. (2021), as in ``navis.models.TraversalModel``.
110
+
111
+ Seeds are layer 1. At each step, every edge ``i -> j`` from a traversed node ``i`` to a node ``j`` that
112
+ is not yet traversed succeeds with probability ``p_ij = clip((A_ij - min_w) / (max_w - min_w), 0, 1)``;
113
+ ``j`` is traversed if any of its edges succeeds and gets that step as its layer. Nodes that fail are
114
+ tried again at later steps. A run ends when no candidate edges are left or at ``max_steps``.
115
+
116
+ Drawing once per node with ``P_j = 1 - prod_i (1 - p_ij)`` has the same distribution as drawing per
117
+ edge, and turns each step of a batch of runs into one sparse-times-dense product.
118
+
119
+ Parameters
120
+ ----------
121
+ graph : Graph or DataFrame
122
+ A :class:`Graph`, or an edge list passed to :meth:`Graph.from_edges` with ``graph_kwargs``.
123
+ max_w, min_w : float
124
+ Input fraction at which an edge always / never fires (navis: 0.3 and 0).
125
+ max_steps : int
126
+ Step cap per run (navis default 15).
127
+ device : "cpu", "gpu" or "auto"
128
+ "gpu" needs CuPy; "auto" uses the GPU if it is available.
129
+ """
130
+
131
+ def __init__(self, graph, *, max_w: float = 0.3, min_w: float = 0.0, max_steps: int = 15,
132
+ device: str = "cpu", **graph_kwargs):
133
+ if not isinstance(graph, Graph):
134
+ graph = Graph.from_edges(graph, **graph_kwargs)
135
+ elif graph_kwargs:
136
+ raise TypeError(f"unexpected arguments for an existing Graph: {sorted(graph_kwargs)}")
137
+ if not 0 <= min_w < max_w:
138
+ raise ValueError("need 0 <= min_w < max_w")
139
+ if max_steps < 1:
140
+ raise ValueError("max_steps must be >= 1")
141
+ self.graph, self.max_w, self.min_w, self.max_steps = graph, max_w, min_w, int(max_steps)
142
+ self.xp, self.sp, self.device = _backend(device)
143
+
144
+ A = graph.A
145
+ p = A.copy()
146
+ p.data = linear_p(p.data, min_w, max_w)
147
+ p.eliminate_zeros()
148
+ sure = p.copy()
149
+ sure.data = (sure.data >= 1.0).astype(np.float32)
150
+ sure.eliminate_zeros()
151
+ logq = p.copy()
152
+ logq.data = np.where(p.data < 1.0, np.log1p(-np.minimum(p.data, 1 - 1e-12)), 0.0)
153
+ logq.eliminate_zeros()
154
+ cand = p.copy()
155
+ cand.data = np.ones_like(cand.data, dtype=np.float32)
156
+ # transposed: row = target node; on the GPU the matrices live on the card
157
+ to = (lambda M: self.sp.csr_matrix(sparse.csr_matrix(M.T, dtype=np.float32))) if self.device == "gpu" \
158
+ else (lambda M: sparse.csr_matrix(M.T, dtype=np.float32))
159
+ self._logqT, self._sureT, self._candT = to(logq), to(sure), to(cand)
160
+
161
+ def __repr__(self) -> str:
162
+ return (f"<flowlayers.TraversalModel: {self.graph.n_nodes:,} nodes, {self.graph.n_edges:,} edges, "
163
+ f"max_w={self.max_w}, max_steps={self.max_steps}, device={self.device}>")
164
+
165
+ # -------------------------------------------------------------- one batch
166
+ def _batch(self, seeds_idx, runs: int, rng):
167
+ """Layers of ``runs`` independent runs, as int16 (n, runs) on the device; 0 = not reached."""
168
+ xp = self.xp
169
+ n = self.graph.n_nodes
170
+ layers = xp.zeros((n, runs), dtype=xp.int16)
171
+ X = xp.zeros((n, runs), dtype=xp.float32)
172
+ layers[seeds_idx, :] = 1
173
+ X[seeds_idx, :] = 1.0
174
+ for t in range(2, self.max_steps + 1):
175
+ pending = (layers == 0) & ((self._candT @ X) > 0)
176
+ if not bool(pending.any()):
177
+ break
178
+ P = -xp.expm1(self._logqT @ X) # 1 - prod(1 - p) over edges with p < 1
179
+ P[(self._sureT @ X) > 0] = 1.0 # any edge with p = 1
180
+ new = pending & (rng.random((n, runs), dtype=xp.float32) < P)
181
+ del P
182
+ layers[new] = t
183
+ X[new] = 1.0
184
+ return layers
185
+
186
+ # -------------------------------------------------------------- run
187
+ def run(self, seeds, *, runs: int | None = None, batch: int = 100, tol: float = 0.01,
188
+ quantile: float = 0.99, max_runs: int = 10_000, seed=None, targets=None,
189
+ missing: str = "warn", progress=None) -> TraversalResult:
190
+ """Traverse from ``seeds`` and return a :class:`TraversalResult`.
191
+
192
+ Parameters
193
+ ----------
194
+ seeds : iterable of node IDs
195
+ Seeds missing from the graph are dropped with a warning (``missing``). Unlike navis, seeds
196
+ without outgoing edges are kept (they are layer 1 and reach nothing).
197
+ runs : int, optional
198
+ Exact number of runs. If None (default), batches of ``batch`` runs are added until the mean
199
+ layer changes by less than ``tol`` in the ``quantile`` of nodes (at least 5 batches), up to
200
+ ``max_runs``.
201
+ seed : int or numpy/cupy Generator, optional
202
+ Random seed for reproducibility.
203
+ targets : iterable of node IDs, optional
204
+ Keep the layer of these nodes in every run (``result.target_runs``), e.g. for bootstraps.
205
+ progress : callable, optional
206
+ Called as ``progress(runs_done, change)`` after every batch.
207
+ """
208
+ xp = self.xp
209
+ t0 = time.time()
210
+ idx = self.graph.index(seeds, missing=missing)
211
+ if len(idx) == 0:
212
+ raise ValueError("none of the seeds are in the graph")
213
+ seeds_idx = xp.asarray(idx)
214
+ tgt_idx = self.graph.index(targets, missing=missing) if targets is not None else None
215
+ rng = seed if hasattr(seed, "random") else xp.random.default_rng(seed)
216
+ n, S = self.graph.n_nodes, self.max_steps
217
+ counts = xp.zeros((n, S + 1), dtype=xp.int64)
218
+ kept = []
219
+ fixed = runs is not None
220
+ total = int(runs) if fixed else int(max_runs)
221
+ done, change, prev = 0, float("inf"), None
222
+ pasos = xp.arange(S + 1)
223
+ while done < total:
224
+ k = min(batch, total - done)
225
+ L = self._batch(seeds_idx, k, rng)
226
+ for t in range(1, S + 1):
227
+ counts[:, t] += (L == t).sum(axis=1)
228
+ if tgt_idx is not None:
229
+ kept.append(L[xp.asarray(tgt_idx)].get() if xp is not np else L[tgt_idx].copy())
230
+ del L
231
+ done += k
232
+ if not fixed:
233
+ c = counts[:, 1:].sum(axis=1)
234
+ mean = xp.where(c > 0, (counts @ pasos) / xp.maximum(c, 1), xp.nan)
235
+ mean = mean.get() if xp is not np else mean
236
+ if prev is not None:
237
+ ok = ~np.isnan(mean) & ~np.isnan(prev)
238
+ change = float(np.quantile(np.abs(mean[ok] - prev[ok]), quantile)) if ok.any() else 0.0
239
+ prev = mean
240
+ if progress:
241
+ progress(done, change)
242
+ if not fixed and done >= 5 * batch and change < tol:
243
+ break
244
+ counts = counts.get() if xp is not np else counts
245
+ if xp is not np:
246
+ xp.get_default_memory_pool().free_all_blocks()
247
+ if not fixed and change >= tol:
248
+ warnings.warn(f"did not converge in {done:,} runs (change {change:.4f} >= tol {tol})", stacklevel=2)
249
+ target_runs = None
250
+ if tgt_idx is not None:
251
+ target_runs = pd.DataFrame(np.concatenate(kept, axis=1).T, columns=self.graph.ids[tgt_idx])
252
+ target_runs.index.name = "run"
253
+ return TraversalResult(ids=self.graph.ids, counts=counts, runs=done, seeds=self.graph.ids[idx],
254
+ max_steps=S, change=change, seconds=time.time() - t0, device=self.device,
255
+ target_runs=target_runs)
256
+
257
+
258
+ def traverse(edges, seeds, *, max_w: float = 0.3, min_w: float = 0.0, max_steps: int = 15,
259
+ device: str = "cpu", graph_kwargs: dict | None = None, **run_kwargs) -> TraversalResult:
260
+ """One-call shortcut: ``TraversalModel(edges, ...).run(seeds, ...)``."""
261
+ model = TraversalModel(edges, max_w=max_w, min_w=min_w, max_steps=max_steps, device=device,
262
+ **(graph_kwargs or {}))
263
+ return model.run(seeds, **run_kwargs)
@@ -0,0 +1,114 @@
1
+ Metadata-Version: 2.5
2
+ Name: flowlayers
3
+ Version: 0.1.0
4
+ Summary: Fast probabilistic traversal of connectomes (Schlegel et al. 2021 information-flow layers), on CPU or GPU
5
+ Project-URL: Repository, https://github.com/Quiwitl/flowlayers
6
+ Author: Atlahua Quiahuitl Martínez Obregón
7
+ License-Expression: MIT
8
+ License-File: LICENSE
9
+ Keywords: connectome,connectomics,drosophila,flywire,gpu,graph traversal,information flow
10
+ Classifier: Intended Audience :: Science/Research
11
+ Classifier: Operating System :: OS Independent
12
+ Classifier: Programming Language :: Python :: 3
13
+ Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
14
+ Requires-Python: >=3.10
15
+ Requires-Dist: numpy>=1.24
16
+ Requires-Dist: pandas>=2.0
17
+ Requires-Dist: scipy>=1.10
18
+ Provides-Extra: gpu
19
+ Requires-Dist: cupy-cuda12x>=13; extra == 'gpu'
20
+ Provides-Extra: test
21
+ Requires-Dist: navis>=1.10; extra == 'test'
22
+ Requires-Dist: pytest>=7; extra == 'test'
23
+ Description-Content-Type: text/markdown
24
+
25
+ # flowlayers
26
+
27
+ Fast probabilistic traversal of connectomes: the information-flow layers of [Schlegel et al. (2021)](https://doi.org/10.7554/eLife.66018), vectorized on the CPU and optionally on the GPU.
28
+
29
+ The model is the one in [`navis.models.TraversalModel`](https://navis-org.github.io/navis/): seed neurons are layer 1, and at each step every edge from a traversed neuron to a not-yet-traversed one fires with a probability proportional to its input fraction (100 % at 0.3). `flowlayers` runs **the same Monte Carlo model** (same rules, same distribution, no approximation) but draws whole batches of runs at once as sparse matrix products, so a whole brain takes seconds per seed set instead of hours.
30
+
31
+ On the whole FlyWire brain (138,625 neurons, 15.1 million connections), the 13 sensory seed sets of Dorkenwald et al. (2024), each run until the mean layers converge (32,100 runs in total), take **8.7 minutes on a laptop GPU**. At the 3.6 s per run that `navis` takes on the same graph, that is about 32 hours.
32
+
33
+ ## Install
34
+
35
+ ```bash
36
+ pip install flowlayers # CPU (numpy, scipy, pandas)
37
+ pip install "flowlayers[gpu]" # + CuPy for NVIDIA GPUs (CUDA 12)
38
+ ```
39
+
40
+ ## Use
41
+
42
+ ```python
43
+ import flowlayers as fl
44
+
45
+ # edges: a DataFrame with presynaptic ID, postsynaptic ID and synapse count
46
+ model = fl.TraversalModel(edges, source="pre", target="post", weight="syn_count",
47
+ max_steps=15, device="auto") # "cpu", "gpu" or "auto"
48
+ res = model.run(olfactory_ids) # batches of 100 runs until the mean layer converges
49
+
50
+ res.layer_mean # pd.Series: mean layer of every neuron (NaN if never reached)
51
+ res.summary # layer_min, layer_max, layer_mean, layer_median, reached (navis' summary columns)
52
+ res.cmf() # probability of having been reached by each step
53
+ res.reached # fraction of runs in which each neuron was reached
54
+ ```
55
+
56
+ - **Weights.** By default the weights are synapse counts and are turned into input fractions (`normalize=True`). Pass `normalize=False` if they already are input fractions, as `navis` expects. `min_weight=5` drops edges with fewer than 5 synapses *before* normalizing — this changes the layers noticeably (see below), so report it.
57
+ - **Runs.** `run(seeds, runs=1000)` does exactly 1,000 runs. Without `runs`, batches of 100 are added until the mean layer changes by less than `tol=0.01` in 99 % of the neurons (at least 5 batches; `max_runs=10_000`).
58
+ - **Per-run layers.** `run(seeds, targets=motor_neuron_ids)` also keeps the layer of those neurons in every run (`res.target_runs`), e.g. for bootstrap intervals.
59
+ - **Reuse.** Build the model once and call `run` for each seed set; the graph is prepared only once (and kept on the GPU).
60
+ - **Probability.** `max_w=0.3` and `min_w=0` as in `navis` (`linear_activation_p`).
61
+
62
+ ## How it works
63
+
64
+ Drawing each edge independently and traversing a neuron if any of its edges fires is the same as drawing each neuron once with
65
+
66
+ P<sub>j</sub> = 1 − Π<sub>i</sub> (1 − p<sub>ij</sub>) over the traversed *i*,
67
+
68
+ which is `1 − exp(Σ log(1 − p_ij))`: one sparse matrix product with the indicator matrix of traversed neurons. Each run is a column of that matrix, so one step of a whole batch of runs is a single sparse × dense product (scipy on the CPU, cuSPARSE through CuPy on the GPU; the same code runs on both). Failed neurons are tried again at later steps, as in `navis`. Results are kept as the full distribution of the first-traversal step of every neuron, from which the mean, median, min, max and CMF are derived.
69
+
70
+ ## Validation
71
+
72
+ - **Exact answer on small graphs.** `flowlayers.exact.exact_layers` enumerates every set of traversed nodes and gives the exact first-traversal distribution. On the diamond of [navis issue #194](https://github.com/navis-org/navis/issues/194) (0→1, 0→2, 1→3, 2→3, p = 0.5) it gives 3.96296 for node 3, the analytic value reported there; `flowlayers` matches it, and matches the full distribution node by node on random small graphs.
73
+ - **Same distribution as navis.** On random graphs the mean layers agree with `navis.models.TraversalModel` with no bias; the remaining differences are sampling noise.
74
+ - **Published layers.** This model, on the whole brain with no synapse threshold and a 15-step cap, reproduces the layers published by Dorkenwald et al. (2024) for FlyWire v783 in all 13 seed sets (Spearman 0.971–0.996, [fly-sensory-layers](https://github.com/Quiwitl/fly-sensory-layers)); flowlayers matches that implementation at ρ ≥ 0.998 (below).
75
+ - **CPU and GPU.** Same results on both devices (tested).
76
+
77
+ ## Benchmarks
78
+
79
+ Laptop with an AMD Ryzen 7 260 (16 threads) and an NVIDIA RTX 5050 Laptop GPU (8 GB), Windows 11, navis 1.12.0, numpy 2.5, scipy 1.18, CuPy 14.2. Scripts in `benchmarks/`.
80
+
81
+ **Random graphs** (log-normal synapse counts, 20 seeds, 100 runs, `max_steps=15`; `bench_navis.py --gpu`). "Mean abs diff" and "bias" compare the mean layers of flowlayers and navis; with 100 runs each the absolute difference is sampling noise, and the bias stays below 0.01 layers.
82
+
83
+ | nodes | edges | navis | flowlayers CPU | speed-up | flowlayers GPU | speed-up | mean abs diff | bias |
84
+ |---:|---:|---:|---:|---:|---:|---:|---:|---:|
85
+ | 5,000 | 149,557 | 2.0 s | 0.14 s | ×14 | 0.02 s | ×86 | 0.092 | +0.001 |
86
+ | 20,000 | 998,750 | 12.2 s | 1.32 s | ×9 | 0.11 s | ×116 | 0.102 | −0.008 |
87
+ | 50,000 | 2,998,196 | 42.4 s | 8.57 s | ×5 | 0.25 s | ×168 | 0.107 | −0.006 |
88
+ | 140,000 | 13,994,879 | 143.6 s | 39.7 s | ×4 | 1.28 s | ×112 | 0.110 | −0.005 |
89
+
90
+ The CPU gain shrinks on big graphs (the sparse product becomes memory-bound); the GPU is the option for whole brains.
91
+
92
+ **Whole FlyWire brain** (v783, proofread connections, no synapse threshold, 15 steps; `bench_flywire.py --device gpu --navis-runs 2`):
93
+
94
+ | | navis | flowlayers GPU |
95
+ |---|---:|---:|
96
+ | 1,000 runs from the olfactory receptor neurons | ≈ 60 min (3.61 s per run) | 16.5 s (×219) |
97
+ | 13 seed sets to convergence (32,100 runs) | ≈ 32 h (extrapolated) | 8.7 min |
98
+
99
+ The mean layers agree with the independent implementation in [fly-sensory-layers](https://github.com/Quiwitl/fly-sensory-layers) at Spearman ρ = 0.998–0.9998 for every seed set (seeds excluded), the level of run-to-run noise.
100
+
101
+ ## Differences from navis
102
+
103
+ - Seeds that never appear as a source are **kept** (as layer 1). `navis` drops them silently, which turns them into ordinary nodes.
104
+ - A run stops when no edge with p > 0 is left (or at `max_steps`); `navis` also waits on edges with p = 0, which changes when it stops but not the layers.
105
+ - Only the linear probability function (`min_w`, `max_w`) is supported; arbitrary per-edge Python callables cannot be vectorized.
106
+ - No `results` table with one row per node and run (it would have hundreds of millions of rows for a whole brain); the per-step counts hold the same information.
107
+
108
+ ## Cite
109
+
110
+ If you use `flowlayers`, please cite the model:
111
+
112
+ > Schlegel P., Bates A. S., Stürner T., *et al.* (2021). Information flow, cell types and stereotypy in a full olfactory connectome. *eLife* 10, e66018.
113
+
114
+ and this repository. MIT license.
@@ -0,0 +1,8 @@
1
+ flowlayers/__init__.py,sha256=mz06yGjVu9iWFDGTfOGr-VtqFcF4zlmMl0gwurGHVzs,316
2
+ flowlayers/exact.py,sha256=BjLo_sxpWrycjY7stRkQWKaVfQnJSaW1Rd2Z6lnsJ3M,1892
3
+ flowlayers/graph.py,sha256=gYF7T26pXlKYaiP2XQMviltWzHBX5lXcoilf_6Oo_ks,5593
4
+ flowlayers/model.py,sha256=TXMXTogAeXDfwJ5UB8EoEwgVUlBCFXK9MFj5V5UXFW8,12524
5
+ flowlayers-0.1.0.dist-info/METADATA,sha256=z9Z2-cJhXy0o7Lmh3PtQqiss_Hp4PS5SOgKd_pG5IqA,8066
6
+ flowlayers-0.1.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
7
+ flowlayers-0.1.0.dist-info/licenses/LICENSE,sha256=oM9UoDmZI8k1vQOp9uNrssAD9nAB5DGTeYYHz5I7LyQ,1093
8
+ flowlayers-0.1.0.dist-info/RECORD,,
@@ -0,0 +1,4 @@
1
+ Wheel-Version: 1.0
2
+ Generator: hatchling 1.33.0
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Atlahua Quiahuitl Martínez Obregón
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.