datanuggets-flow 0.1.0a3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- datanuggets_flow/__init__.py +32 -0
- datanuggets_flow/_defaults.py +120 -0
- datanuggets_flow/_pick/__init__.py +3 -0
- datanuggets_flow/_pick/args.py +351 -0
- datanuggets_flow/_pick/prompt.py +78 -0
- datanuggets_flow/_pipeline/__init__.py +2 -0
- datanuggets_flow/_pipeline/clustering.py +397 -0
- datanuggets_flow/_pipeline/cofactor.py +113 -0
- datanuggets_flow/_pipeline/fcs.py +93 -0
- datanuggets_flow/_pipeline/gating.py +102 -0
- datanuggets_flow/_pipeline/records.py +232 -0
- datanuggets_flow/_pipeline/selection.py +89 -0
- datanuggets_flow/_pipeline/stages.py +260 -0
- datanuggets_flow/_pipeline/tables.py +101 -0
- datanuggets_flow/_validate.py +33 -0
- datanuggets_flow/colors.py +22 -0
- datanuggets_flow/compare.py +183 -0
- datanuggets_flow/exceptions.py +22 -0
- datanuggets_flow/figures.py +166 -0
- datanuggets_flow/gate.py +173 -0
- datanuggets_flow/grid.py +75 -0
- datanuggets_flow/livecells.py +432 -0
- datanuggets_flow/outputs.py +212 -0
- datanuggets_flow/peaks.py +84 -0
- datanuggets_flow/pick.py +296 -0
- datanuggets_flow/pipeline.py +55 -0
- datanuggets_flow/plot.py +137 -0
- datanuggets_flow/settings.py +323 -0
- datanuggets_flow/state.py +429 -0
- datanuggets_flow-0.1.0a3.dist-info/METADATA +233 -0
- datanuggets_flow-0.1.0a3.dist-info/RECORD +35 -0
- datanuggets_flow-0.1.0a3.dist-info/WHEEL +4 -0
- datanuggets_flow-0.1.0a3.dist-info/entry_points.txt +2 -0
- datanuggets_flow-0.1.0a3.dist-info/licenses/LICENSE +338 -0
- datanuggets_flow-0.1.0a3.dist-info/licenses/THIRD_PARTY.md +104 -0
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
"""Cabrera's gate finder for the lymphocyte and macrophage populations on FSC-A against SSC-A,
|
|
2
|
+
from Prof. Cabrera's Python port of the original R code, checked piece by piece against it."""
|
|
3
|
+
|
|
4
|
+
from importlib.metadata import version as _version
|
|
5
|
+
|
|
6
|
+
from datanuggets_flow.gate import (
|
|
7
|
+
FinderPeaks,
|
|
8
|
+
finder_peaks,
|
|
9
|
+
gate_peaks,
|
|
10
|
+
hull_gate,
|
|
11
|
+
lymphmacro_finder,
|
|
12
|
+
)
|
|
13
|
+
from datanuggets_flow.grid import DensityGrid, f_plot
|
|
14
|
+
from datanuggets_flow.peaks import find_area, local_maxima, mirrored_area, refine_peaks, search_ind
|
|
15
|
+
|
|
16
|
+
# pyproject.toml holds the version; this reads it back from the installed metadata
|
|
17
|
+
__version__ = _version("datanuggets-flow")
|
|
18
|
+
|
|
19
|
+
__all__ = [
|
|
20
|
+
"DensityGrid",
|
|
21
|
+
"FinderPeaks",
|
|
22
|
+
"f_plot",
|
|
23
|
+
"find_area",
|
|
24
|
+
"finder_peaks",
|
|
25
|
+
"gate_peaks",
|
|
26
|
+
"hull_gate",
|
|
27
|
+
"local_maxima",
|
|
28
|
+
"lymphmacro_finder",
|
|
29
|
+
"mirrored_area",
|
|
30
|
+
"refine_peaks",
|
|
31
|
+
"search_ind",
|
|
32
|
+
]
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
"""datanuggets-flow's default values, kept in one place: the flow's own, and the library
|
|
2
|
+
defaults its settings start from (SEED to CENTER_METHODS, TSNE_PERPLEXITY and TSNE_SEED), copied
|
|
3
|
+
because the library keeps them private. tests/flow/test_flow_library_pins.py checks each copy
|
|
4
|
+
against the library's public behaviour."""
|
|
5
|
+
|
|
6
|
+
SEED = 291102 # WCluster's Wkmeans and DN.Wkmeans default to it too
|
|
7
|
+
SEED_MAX = 2**31 - 1 # R's .Machine$integer.max, the largest seed set.seed and RDraws take
|
|
8
|
+
DN_NUM1 = 10_000
|
|
9
|
+
CENTER_METHODS = ("mean", "random", "original")
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
# the flow gate finder's tunable values, as cfc.py calls it: the n_row by n_col window its local
|
|
13
|
+
# maxima are found in, and the area each peak grows, the cells closer than t1 grid cells with more
|
|
14
|
+
# than t2 times the peak's density; t11 and t21 for the first peak, t12 and t22 for the others
|
|
15
|
+
FINDER_N_ROW = 11
|
|
16
|
+
FINDER_N_COL = 11
|
|
17
|
+
FINDER_T11 = 22
|
|
18
|
+
FINDER_T21 = 0.25
|
|
19
|
+
FINDER_T12 = 20
|
|
20
|
+
FINDER_T22 = 0.5
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
# datanuggets-flow's defaults: the scatter channels it gates on; reference, cfc.py's mirrored_area,
|
|
24
|
+
# as the area rule, where the library's lymphmacro_finder defaults to find_area; the channels it
|
|
25
|
+
# clusters, every -A channel but the scatter ones; the arcsinh cofactor; the Scale stage, each
|
|
26
|
+
# channel centred and scaled to SD 1 by its mean and SD pooled over the files; nuggets per file,
|
|
27
|
+
# where create_dn's own default is DN_NUM2; every core for create_dn, which doesn't change its
|
|
28
|
+
# results; the clusters; and the output folder, named by the UTC time a run starts
|
|
29
|
+
FLOW_FSC = "FSC-A"
|
|
30
|
+
FLOW_SSC = "SSC-A"
|
|
31
|
+
FLOW_DISC = "reference"
|
|
32
|
+
FLOW_CHANNEL_SUFFIX = "-A"
|
|
33
|
+
FLOW_SCATTER_PREFIXES = ("FSC", "SSC")
|
|
34
|
+
# the Transform stage's cofactor. "auto" picks one per clustered channel, once per run and before
|
|
35
|
+
# any arcsinh, from the raw values of the cells the run clusters (the populations kept, after the
|
|
36
|
+
# row filters, in the files it keeps): a channel with at least FLOW_COFACTOR_MIN_NEGATIVE of those
|
|
37
|
+
# cells below 0 takes the SD of its cloud below 0, the median of |x| over them divided by
|
|
38
|
+
# FLOW_COFACTOR_MAD_Z, as it is for a zero-centred Gaussian, but no less than FLOW_COFACTOR_FLOOR;
|
|
39
|
+
# any other channel takes FLOW_COFACTOR_FLOOR. arcsinh(x / c) splits a zero-centred Gaussian of SD
|
|
40
|
+
# s into two humps exactly when c < s, so a cofactor at least the SD of the cells below 0 keeps
|
|
41
|
+
# that cloud one population, and the floor keeps the rest of the channel logarithmic, where 6000
|
|
42
|
+
# for every channel left the long right tails linear. The median is read from a histogram of |x|
|
|
43
|
+
# in log-spaced bins, FLOW_COFACTOR_HIST's low and high edges at its bins per decade (about 0.58%
|
|
44
|
+
# wide), with one bin below and one above, so the files are pooled in bounded memory and in any
|
|
45
|
+
# order. On whole public files with no gate, FlowCAP's FACSAria-type files (0092, 0094, 0253 and
|
|
46
|
+
# 0255) have 6% to 21% of their cells below 0 on every marker channel, with an SD of 16 to 36, so
|
|
47
|
+
# every channel takes 150; on the Aurora demo files (190, 193, 199 and 223) six of the 32 do not:
|
|
48
|
+
# CD8 338, CD14 350, ACE2 210, CD19 204, CD123 198 and CCR4 175. The guard keeps a few stray
|
|
49
|
+
# negatives from setting a channel's cofactor: CCR7's spread there is 698 from 0.12% of the cells,
|
|
50
|
+
# and eFluor 450's comes from about ten events. FLOW_COFACTOR_WIDE is the one number a run used
|
|
51
|
+
# before the Transform stage was recorded, which a record without one is read back with.
|
|
52
|
+
FLOW_COFACTOR_AUTO = "auto"
|
|
53
|
+
FLOW_COFACTOR = FLOW_COFACTOR_AUTO
|
|
54
|
+
FLOW_COFACTOR_FLOOR = 150.0
|
|
55
|
+
FLOW_COFACTOR_MIN_NEGATIVE = 0.01
|
|
56
|
+
FLOW_COFACTOR_MAD_Z = 0.6745 # the median of |z| for a standard normal z
|
|
57
|
+
FLOW_COFACTOR_HIST = (1e-2, 1e8, 400) # low and high edges of |x|, and bins per decade
|
|
58
|
+
FLOW_COFACTOR_WIDE = 6000.0
|
|
59
|
+
FLOW_TIME_PATTERN = r"^(time|hdr-t)\b" # matched ignoring case: Time, Time (s), HDR-T
|
|
60
|
+
FLOW_STANDARDIZE = True
|
|
61
|
+
# the Scale stage warns when a channel's skewness after the Transform stage is beyond this either
|
|
62
|
+
# way: mean and SD then follow its tail, so most cells take a narrow band of the scaled axis
|
|
63
|
+
FLOW_SKEW_WARN = 5.0
|
|
64
|
+
FLOW_DN_NUM2 = 1000
|
|
65
|
+
FLOW_N_JOBS = -1
|
|
66
|
+
FLOW_K = 8
|
|
67
|
+
# the flow's clustering of the pooled nuggets: weighted k-means, WCluster 1.3.0's Wkmeans, R's
|
|
68
|
+
# implementation of Algorithm 3 of Beavers et al. (JCGS 34(1), 2025), or the hierarchical tree
|
|
69
|
+
# (Whclust, cut by cut_tree), still available. R starts each run from a random partition and adds
|
|
70
|
+
# 20 weighted Lloyd steps to each round, where the paper starts from K nugget centers and moves
|
|
71
|
+
# single nuggets. num_init is the paper's ten starts (Sec 4). max_iter is ours: R's default is 10,
|
|
72
|
+
# and the paper repeats its single-nugget step until no nugget moves, allowing a cap it gives no
|
|
73
|
+
# value for. The nugget centers default to "original", the stage-2 points that build the nuggets,
|
|
74
|
+
# which Cabrera's group prefers to the means (create.DN's own default is "mean").
|
|
75
|
+
FLOW_METHODS = ("wkmeans", "tree")
|
|
76
|
+
FLOW_METHOD = "wkmeans"
|
|
77
|
+
FLOW_CENTER_METHOD = "original"
|
|
78
|
+
FLOW_NUM_INIT = 10
|
|
79
|
+
FLOW_MAX_ITER = 100
|
|
80
|
+
# the random draws behind each file's create_dn: "r", R's set.seed(seed) draws (RDraws), so the
|
|
81
|
+
# nuggets are create.DN's; or "numpy", NumPy's Generator from the int seed, which every run
|
|
82
|
+
# recorded before draws existed used. Weighted k-means always draws R's, as every run of it has.
|
|
83
|
+
DRAWS = ("r", "numpy")
|
|
84
|
+
FLOW_DRAWS = "r"
|
|
85
|
+
# a run record or job written before method existed ran the tree on mean centers, and one written
|
|
86
|
+
# before draws existed drew its nuggets with NumPy
|
|
87
|
+
FLOW_LEGACY_METHOD = "tree"
|
|
88
|
+
FLOW_LEGACY_CENTER_METHOD = "mean"
|
|
89
|
+
FLOW_LEGACY_DRAWS = "numpy"
|
|
90
|
+
FLOW_OUT_DIR = "outputs"
|
|
91
|
+
FLOW_RUN_NAME = "flow-%Y%m%d-%H%M%S-%fZ"
|
|
92
|
+
FLOW_GATES_NAME = "flow-gates-%Y%m%d-%H%M%S-%fZ"
|
|
93
|
+
FLOW_SPILL_TOL = 1e-3 # how far from the identity a file's spillover matrix may be before a warning
|
|
94
|
+
FLOW_GATE_DPI = 110 # local_maxima.png and gates.png
|
|
95
|
+
FLOW_RESULT_DPI = 150 # the clustering's figures
|
|
96
|
+
|
|
97
|
+
# the t-SNE of the clustered nuggets: its perplexity, lowered to (n - 1) / 3 for n nuggets when
|
|
98
|
+
# that's smaller, as scikit-learn needs it below n, and its seed
|
|
99
|
+
TSNE_PERPLEXITY = 30.0
|
|
100
|
+
TSNE_SEED = 0
|
|
101
|
+
PCA_VIEW_COMPONENTS = 5 # the PCs cells.csv gives each sampled row, fewer with fewer variables
|
|
102
|
+
|
|
103
|
+
# datanuggets-flow's sample comparison: the cells per file sampled into cells.csv (0 writes
|
|
104
|
+
# none), the pseudocount of group_summary.csv's log2 fold change in cells of each kept file (half
|
|
105
|
+
# a cell: 100 * 0.5 / N_f percentage points for a file of N_f kept cells, averaged over the run's
|
|
106
|
+
# kept files), and the names a group may take
|
|
107
|
+
FLOW_CELLS_PER_FILE = 2000
|
|
108
|
+
FLOW_LOG2FC_CELLS = 0.5
|
|
109
|
+
FLOW_GROUP_NAME = r"^[A-Za-z0-9][A-Za-z0-9 _.+-]{0,39}$"
|
|
110
|
+
FLOW_LEGEND_ROWS = 25 # clusters_tsne.png legend rows per column, so k = 100 fits beside the plot
|
|
111
|
+
|
|
112
|
+
# the live-cells step's display defaults (datanuggets_flow.livecells), none of them a threshold:
|
|
113
|
+
# bins per histogram, the pooled quantiles a channel's range spans, the margin added to each side
|
|
114
|
+
# of it (a share of its width, cut back to the data's own extremes), the width a range of one
|
|
115
|
+
# value is opened to, and the schema of the kept values' saved file
|
|
116
|
+
LIVECELLS_BINS = 128
|
|
117
|
+
LIVECELLS_RANGE_QUANTILES = (0.001, 0.999)
|
|
118
|
+
LIVECELLS_RANGE_PAD = 0.05
|
|
119
|
+
LIVECELLS_FLAT_WIDTH = 1.0
|
|
120
|
+
LIVECELLS_SCHEMA = "datanuggets-livecells/1"
|
|
@@ -0,0 +1,351 @@
|
|
|
1
|
+
"""datanuggets-flow's arguments: the flags, the FCS paths they name, the settings they give,
|
|
2
|
+
checked before any file is read, and the rules each mode holds them to."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import argparse
|
|
7
|
+
from dataclasses import fields, replace
|
|
8
|
+
from datetime import UTC, datetime
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
|
|
11
|
+
from datanuggets_flow import pipeline
|
|
12
|
+
from datanuggets_flow._defaults import (
|
|
13
|
+
CENTER_METHODS,
|
|
14
|
+
DRAWS,
|
|
15
|
+
FLOW_COFACTOR_AUTO,
|
|
16
|
+
FLOW_COFACTOR_FLOOR,
|
|
17
|
+
FLOW_GATES_NAME,
|
|
18
|
+
FLOW_METHODS,
|
|
19
|
+
FLOW_OUT_DIR,
|
|
20
|
+
FLOW_RUN_NAME,
|
|
21
|
+
)
|
|
22
|
+
from datanuggets_flow.settings import (
|
|
23
|
+
AREAS,
|
|
24
|
+
DISC_ALIASES,
|
|
25
|
+
ClusterSettings,
|
|
26
|
+
GateSettings,
|
|
27
|
+
check_groups,
|
|
28
|
+
cli_flag,
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def per_file(text: str | None, name: str) -> tuple[float, ...] | None:
|
|
33
|
+
"""--t11 or --t21 as GateSettings takes it: one number for every file, or one per file
|
|
34
|
+
separated by commas, or None when it is left out. GateSettings.check counts them against the
|
|
35
|
+
files, so the rule is kept in one place."""
|
|
36
|
+
if text is None:
|
|
37
|
+
return None
|
|
38
|
+
try:
|
|
39
|
+
return tuple(float(t) for t in str(text).split(","))
|
|
40
|
+
except ValueError as e:
|
|
41
|
+
raise SystemExit(f"{name} takes numbers like 22 or 22,20,18,18, got {text!r}") from e
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _cofactor(text: str) -> float | str:
|
|
45
|
+
"""--cofactor as ClusterSettings takes it: auto, or a number. Anything else is passed on as
|
|
46
|
+
given, for ClusterSettings.check to refuse with the rule."""
|
|
47
|
+
if text == FLOW_COFACTOR_AUTO:
|
|
48
|
+
return text
|
|
49
|
+
try:
|
|
50
|
+
return float(text)
|
|
51
|
+
except ValueError:
|
|
52
|
+
return text
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _fcs_paths(items) -> list[Path]:
|
|
56
|
+
paths = []
|
|
57
|
+
for item in map(Path, items):
|
|
58
|
+
if item.is_dir(): # a folder's .fcs files by name, leaving out macOS's ._ copies
|
|
59
|
+
paths += sorted(
|
|
60
|
+
p
|
|
61
|
+
for p in item.iterdir()
|
|
62
|
+
if p.suffix.lower() == ".fcs" and not p.name.startswith(".")
|
|
63
|
+
)
|
|
64
|
+
elif item.is_file():
|
|
65
|
+
paths.append(item)
|
|
66
|
+
else:
|
|
67
|
+
raise SystemExit(f"{item}: no such file or folder")
|
|
68
|
+
if not paths:
|
|
69
|
+
raise SystemExit(f"no .fcs files in {', '.join(map(str, items))}")
|
|
70
|
+
return paths
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _parser() -> argparse.ArgumentParser:
|
|
74
|
+
"""The flags. One that sets a GateSettings or ClusterSettings field, under the field's name,
|
|
75
|
+
defaults to None, so a run without it takes the settings' default, which its help shows."""
|
|
76
|
+
gate, cluster = GateSettings(), ClusterSettings()
|
|
77
|
+
p = argparse.ArgumentParser(
|
|
78
|
+
prog="datanuggets-flow",
|
|
79
|
+
description="Gate FCS files on FSC-A vs SSC-A, keep the populations you pick, and cluster "
|
|
80
|
+
"their cells with data nuggets and weighted k-means (or the hierarchical tree).",
|
|
81
|
+
)
|
|
82
|
+
p.add_argument("paths", nargs="*", help="FCS files, or folders of them; none with --from-gates")
|
|
83
|
+
stage = p.add_mutually_exclusive_group()
|
|
84
|
+
stage.add_argument(
|
|
85
|
+
"--gates-only",
|
|
86
|
+
action="store_true",
|
|
87
|
+
help="gate the files, draw local_maxima.png and gates.png, save the gates in --out, and "
|
|
88
|
+
"stop, without asking",
|
|
89
|
+
)
|
|
90
|
+
stage.add_argument(
|
|
91
|
+
"--from-gates",
|
|
92
|
+
metavar="GATES",
|
|
93
|
+
help="choose populations over the figures of GATES, a --gates-only folder, and cluster "
|
|
94
|
+
"its files, which must not have changed",
|
|
95
|
+
)
|
|
96
|
+
p.add_argument(
|
|
97
|
+
"--pops",
|
|
98
|
+
help='populations to keep, e.g. "1,2" for every file or "1,2;1;1" '
|
|
99
|
+
"file by file; without it you are asked",
|
|
100
|
+
)
|
|
101
|
+
p.add_argument(
|
|
102
|
+
"--out",
|
|
103
|
+
help="new output folder (default: outputs/flow-<timestamp>, or "
|
|
104
|
+
"outputs/flow-gates-<timestamp> with --gates-only); existing paths are refused",
|
|
105
|
+
)
|
|
106
|
+
p.add_argument("--k", type=int, help=f"clusters (default: {cluster.k})")
|
|
107
|
+
p.add_argument(
|
|
108
|
+
"--method",
|
|
109
|
+
choices=FLOW_METHODS,
|
|
110
|
+
help="wkmeans: WCluster's weighted k-means of the pooled nuggets, as Wkmeans clusters "
|
|
111
|
+
"them; tree: the hierarchical tree, Whclust cut into --k clusters, or into as many as "
|
|
112
|
+
f"there are nuggets when that's fewer (default: {cluster.method})",
|
|
113
|
+
)
|
|
114
|
+
p.add_argument(
|
|
115
|
+
"--center-method",
|
|
116
|
+
choices=CENTER_METHODS,
|
|
117
|
+
help="each nugget's center, create.DN's center.method: original, the point the nugget "
|
|
118
|
+
"was built around; mean, its cells' mean; random, one of its cells at random (default: "
|
|
119
|
+
f"{cluster.center_method})",
|
|
120
|
+
)
|
|
121
|
+
p.add_argument(
|
|
122
|
+
"--num-init",
|
|
123
|
+
type=int,
|
|
124
|
+
metavar="N",
|
|
125
|
+
help=f"weighted k-means' random starts, the best kept (default: {cluster.num_init})",
|
|
126
|
+
)
|
|
127
|
+
p.add_argument(
|
|
128
|
+
"--max-iter",
|
|
129
|
+
type=int,
|
|
130
|
+
metavar="N",
|
|
131
|
+
help=f"weighted k-means' rounds per start at most (default: {cluster.max_iter})",
|
|
132
|
+
)
|
|
133
|
+
p.add_argument(
|
|
134
|
+
"--draws",
|
|
135
|
+
choices=DRAWS,
|
|
136
|
+
help="create_dn's draws from --seed: r, R's set.seed draws, so the nuggets are "
|
|
137
|
+
"create.DN's; numpy, the draws runs made before this flag (default: "
|
|
138
|
+
f"{cluster.draws})",
|
|
139
|
+
)
|
|
140
|
+
p.add_argument(
|
|
141
|
+
"--cofactor",
|
|
142
|
+
type=_cofactor,
|
|
143
|
+
metavar="COFACTOR",
|
|
144
|
+
help="the Transform stage: fluorescence is clustered as arcsinh(x / cofactor), or as "
|
|
145
|
+
f"stored with 0; {cluster.cofactor} (the default) picks one per channel from the spread "
|
|
146
|
+
f"of its cells below 0, at least {FLOW_COFACTOR_FLOOR:g}; a number applies to every "
|
|
147
|
+
"channel; 0 keeps the channels linear",
|
|
148
|
+
)
|
|
149
|
+
p.add_argument(
|
|
150
|
+
"--no-standardize",
|
|
151
|
+
dest="standardize",
|
|
152
|
+
action="store_false",
|
|
153
|
+
default=None,
|
|
154
|
+
help="leave out the Scale stage: keep each channel in the transform's units instead of "
|
|
155
|
+
"centring it and scaling it to SD 1 by the mean and SD pooled over the files",
|
|
156
|
+
)
|
|
157
|
+
p.add_argument(
|
|
158
|
+
"--dn-num1",
|
|
159
|
+
type=int,
|
|
160
|
+
help="create_dn's dn_num1, the nuggets its first pass makes from a file's kept cells "
|
|
161
|
+
f"before they are reduced to --dn-num2 (default: {cluster.dn_num1})",
|
|
162
|
+
)
|
|
163
|
+
p.add_argument("--dn-num2", type=int, help=f"nuggets per file (default: {cluster.dn_num2})")
|
|
164
|
+
p.add_argument(
|
|
165
|
+
"--seed",
|
|
166
|
+
type=int,
|
|
167
|
+
help=f"the seed of create_dn and of weighted k-means (default: {cluster.seed})",
|
|
168
|
+
)
|
|
169
|
+
p.add_argument(
|
|
170
|
+
"--jobs",
|
|
171
|
+
type=int,
|
|
172
|
+
dest="n_jobs",
|
|
173
|
+
metavar="JOBS",
|
|
174
|
+
help="cores create_dn and weighted k-means use, -1 for every one; the results are the "
|
|
175
|
+
f"same for any (default: {cluster.n_jobs})",
|
|
176
|
+
)
|
|
177
|
+
channels = p.add_mutually_exclusive_group()
|
|
178
|
+
channels.add_argument(
|
|
179
|
+
"--channels",
|
|
180
|
+
help="comma-separated channels to cluster (default: every -A channel but FSC and SSC)",
|
|
181
|
+
)
|
|
182
|
+
channels.add_argument(
|
|
183
|
+
"--exclude-channels", help="comma-separated channels to leave out of the default ones"
|
|
184
|
+
)
|
|
185
|
+
p.add_argument("--fsc", help=f"the channel gated as FSC (default: {gate.fsc})")
|
|
186
|
+
p.add_argument("--ssc", help=f"the channel gated as SSC (default: {gate.ssc})")
|
|
187
|
+
p.add_argument(
|
|
188
|
+
"--disc",
|
|
189
|
+
type=lambda d: DISC_ALIASES.get(d, d), # his, the old name, still works
|
|
190
|
+
choices=tuple(AREAS),
|
|
191
|
+
help="reference: the reference code's gates, so the populations match cfc.py's figures; "
|
|
192
|
+
"peak: datanuggets_flow.find_area, the disc centred on the peak "
|
|
193
|
+
f"(default: {gate.disc})",
|
|
194
|
+
)
|
|
195
|
+
p.add_argument(
|
|
196
|
+
"--t11",
|
|
197
|
+
help="the first peak's distance threshold, one for all files or one per file, e.g. "
|
|
198
|
+
f"22,20,18,18 as in cfc.py (default: {gate.t11[0]})",
|
|
199
|
+
)
|
|
200
|
+
p.add_argument(
|
|
201
|
+
"--t21",
|
|
202
|
+
help="the first peak's density threshold, the same way, e.g. 0.25,0.5,0.5,0.5 "
|
|
203
|
+
f"(default: {gate.t21[0]})",
|
|
204
|
+
)
|
|
205
|
+
p.add_argument(
|
|
206
|
+
"--t12",
|
|
207
|
+
type=float,
|
|
208
|
+
help=f"every other peak's distance threshold, in every file (default: {gate.t12})",
|
|
209
|
+
)
|
|
210
|
+
p.add_argument(
|
|
211
|
+
"--t22",
|
|
212
|
+
type=float,
|
|
213
|
+
help=f"every other peak's density threshold, in every file (default: {gate.t22})",
|
|
214
|
+
)
|
|
215
|
+
p.add_argument(
|
|
216
|
+
"--n-row",
|
|
217
|
+
type=int,
|
|
218
|
+
help=f"rows of the window the finder's local maxima are found in (default: {gate.n_row})",
|
|
219
|
+
)
|
|
220
|
+
p.add_argument("--n-col", type=int, help=f"columns of that window (default: {gate.n_col})")
|
|
221
|
+
p.add_argument(
|
|
222
|
+
"--no-tsne",
|
|
223
|
+
dest="tsne",
|
|
224
|
+
action="store_false",
|
|
225
|
+
default=None,
|
|
226
|
+
help="leave out the t-SNE of the nuggets, clusters_tsne.png, and the need for scikit-learn",
|
|
227
|
+
)
|
|
228
|
+
p.add_argument(
|
|
229
|
+
"--tsne-perplexity",
|
|
230
|
+
type=float,
|
|
231
|
+
help="the t-SNE's perplexity, lowered to (nuggets - 1) / 3 when that's smaller "
|
|
232
|
+
f"(default: {cluster.tsne_perplexity:g})",
|
|
233
|
+
)
|
|
234
|
+
p.add_argument("--tsne-seed", type=int, help=f"the t-SNE's seed (default: {cluster.tsne_seed})")
|
|
235
|
+
p.add_argument(
|
|
236
|
+
"--filter",
|
|
237
|
+
dest="row_filters",
|
|
238
|
+
action="append",
|
|
239
|
+
metavar="SPEC",
|
|
240
|
+
help='keep only kept cells that pass, on raw $PnN values, e.g. "[FJComp-BV510-A] > 1000"; '
|
|
241
|
+
"bracket a channel name with spaces; repeat to AND several",
|
|
242
|
+
)
|
|
243
|
+
p.add_argument(
|
|
244
|
+
"--groups",
|
|
245
|
+
help='one group name per file, in file order, separated by ;, e.g. "ctrl;ctrl;treated"; '
|
|
246
|
+
"adds groups.csv, group_summary.csv and group_markers.csv",
|
|
247
|
+
)
|
|
248
|
+
p.add_argument(
|
|
249
|
+
"--cells-per-file",
|
|
250
|
+
type=int,
|
|
251
|
+
help="kept cells per file sampled into cells.csv, 0 for none "
|
|
252
|
+
f"(default: {cluster.cells_per_file})",
|
|
253
|
+
)
|
|
254
|
+
p.add_argument("--no-show", action="store_true", help="save the figures without opening them")
|
|
255
|
+
return p
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
def _given(args, cls) -> list[str]:
|
|
259
|
+
"""The settings of cls whose flags were given."""
|
|
260
|
+
return [f.name for f in fields(cls) if getattr(args, f.name, None) is not None]
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def _mode_rules(args) -> None:
|
|
264
|
+
"""Refuses a flag the mode can't use, before any setting is checked or file read: FCS paths
|
|
265
|
+
or a gate setting with --from-gates, which reuses its folder's files and gates, and a
|
|
266
|
+
clustering setting or --pops with --gates-only, which stops before them."""
|
|
267
|
+
if args.from_gates is None and not args.paths:
|
|
268
|
+
raise SystemExit("give FCS files or folders, or --from-gates GATES")
|
|
269
|
+
if args.from_gates is not None and args.paths:
|
|
270
|
+
raise SystemExit(
|
|
271
|
+
"--from-gates reads the FCS files its gate folder records; give no FCS paths"
|
|
272
|
+
)
|
|
273
|
+
gate = _given(args, GateSettings) if args.from_gates is not None else []
|
|
274
|
+
if gate:
|
|
275
|
+
raise SystemExit(
|
|
276
|
+
f"{cli_flag(gate[0])} sets the gate stage, and --from-gates reuses the gates in "
|
|
277
|
+
f"{args.from_gates}; run --gates-only again to change it"
|
|
278
|
+
)
|
|
279
|
+
cluster = _given(args, ClusterSettings) + ["pops"] * (args.pops is not None)
|
|
280
|
+
if args.gates_only and cluster:
|
|
281
|
+
raise SystemExit(
|
|
282
|
+
f"{cli_flag(cluster[0])} sets the clustering stage, which --gates-only stops before"
|
|
283
|
+
)
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
def _flags(cls, args, **values):
|
|
287
|
+
"""cls's settings from the flags named for them, values standing in for flags that are read
|
|
288
|
+
another way first. A flag left out, or a value of None, keeps cls's default."""
|
|
289
|
+
given = {f.name: getattr(args, f.name, None) for f in fields(cls)} | values
|
|
290
|
+
return cls(**{name: v for name, v in given.items() if v is not None})
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
def _settings(args) -> ClusterSettings:
|
|
294
|
+
"""The settings the flags give, checked before any file is read, so a bad value stops the run
|
|
295
|
+
first; the clustering's are returned. --t11 and --t21 wait for the files to be counted, and
|
|
296
|
+
--groups, whose names are checked here, for groups_for to count them against the files. A
|
|
297
|
+
flag that would have no effect is refused rather than ignored: the t-SNE's with --no-tsne,
|
|
298
|
+
and weighted k-means' starts and rounds with --method tree."""
|
|
299
|
+
for name in ("tsne_perplexity", "tsne_seed"):
|
|
300
|
+
if args.tsne is False and getattr(args, name) is not None:
|
|
301
|
+
raise SystemExit(f"{cli_flag(name)} has no effect with --no-tsne")
|
|
302
|
+
if args.method == "tree" and (args.num_init is not None or args.max_iter is not None):
|
|
303
|
+
raise SystemExit(
|
|
304
|
+
"--num-init and --max-iter set weighted k-means, not the hierarchical tree"
|
|
305
|
+
)
|
|
306
|
+
_flags(GateSettings, args, t11=None, t21=None).check()
|
|
307
|
+
if args.groups is not None:
|
|
308
|
+
check_groups([name.strip() for name in args.groups.split(";")])
|
|
309
|
+
cluster = _flags(
|
|
310
|
+
ClusterSettings,
|
|
311
|
+
args,
|
|
312
|
+
channels=_channel_names(args.channels),
|
|
313
|
+
exclude_channels=_channel_names(args.exclude_channels),
|
|
314
|
+
row_filters=tuple(args.row_filters) if args.row_filters else None,
|
|
315
|
+
groups=None,
|
|
316
|
+
)
|
|
317
|
+
return cluster.check()
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
def groups_for(args, cluster: ClusterSettings, n_files: int) -> ClusterSettings:
|
|
321
|
+
"""cluster with --groups, once the files are counted: one name per file, by
|
|
322
|
+
pipeline.parse_groups. Without --groups, cluster as it is."""
|
|
323
|
+
if args.groups is None:
|
|
324
|
+
return cluster
|
|
325
|
+
return replace(cluster, groups=pipeline.parse_groups(args.groups, n_files))
|
|
326
|
+
|
|
327
|
+
|
|
328
|
+
def _out(args, name: str) -> Path:
|
|
329
|
+
"""--out, or a folder under FLOW_OUT_DIR named for the UTC time by the pattern name, so each
|
|
330
|
+
run has its own."""
|
|
331
|
+
return Path(args.out) if args.out else Path(FLOW_OUT_DIR) / datetime.now(UTC).strftime(name)
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
def _check_args(args) -> tuple[list[Path], GateSettings, Path]:
|
|
335
|
+
"""Validate arguments without writing files; return inputs, the gate settings and output.
|
|
336
|
+
The gate settings are built once, here, with --t11 and --t21 as given and checked against the
|
|
337
|
+
files; one left out gives the settings' one value for every file. --pops is counted against
|
|
338
|
+
the files too, by the rule parse_selection reads it with."""
|
|
339
|
+
paths = _fcs_paths(args.paths)
|
|
340
|
+
t11, t21 = per_file(args.t11, "--t11"), per_file(args.t21, "--t21")
|
|
341
|
+
gate = _flags(GateSettings, args, t11=t11, t21=t21).check(len(paths))
|
|
342
|
+
out = _out(args, FLOW_GATES_NAME if args.gates_only else FLOW_RUN_NAME)
|
|
343
|
+
if args.pops is not None:
|
|
344
|
+
pipeline.selection_choices(args.pops, len(paths))
|
|
345
|
+
return paths, gate, out
|
|
346
|
+
|
|
347
|
+
|
|
348
|
+
def _channel_names(names: str | None) -> tuple[str, ...] | None:
|
|
349
|
+
"""--channels or --exclude-channels as ClusterSettings takes it: its names, or None when it
|
|
350
|
+
is left out."""
|
|
351
|
+
return tuple(c.strip() for c in names.split(",")) if names else None
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
"""datanuggets-flow's question: which populations to keep in each file, asked at the
|
|
2
|
+
terminal while the figures stay responsive, or read from --pops."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import sys
|
|
7
|
+
|
|
8
|
+
from datanuggets_flow import pipeline
|
|
9
|
+
from datanuggets_flow.pipeline import parse_pops
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def _live_input(prompt, plt, show):
|
|
13
|
+
"""input(), with the figure window kept responsive (zoom, pan, resize) until something is
|
|
14
|
+
typed in the terminal: input() alone stops the window's event loop while it waits."""
|
|
15
|
+
print(prompt, end="", flush=True)
|
|
16
|
+
if show:
|
|
17
|
+
try:
|
|
18
|
+
import msvcrt # Windows: wait for the first key press
|
|
19
|
+
|
|
20
|
+
while not msvcrt.kbhit():
|
|
21
|
+
plt.pause(0.1)
|
|
22
|
+
except ImportError: # macOS and Linux: wait for a line
|
|
23
|
+
import select
|
|
24
|
+
|
|
25
|
+
while not select.select([sys.stdin], [], [], 0)[0]:
|
|
26
|
+
plt.pause(0.1)
|
|
27
|
+
try:
|
|
28
|
+
return input()
|
|
29
|
+
except (EOFError, KeyboardInterrupt):
|
|
30
|
+
raise SystemExit("\ncancelled; nothing was clustered") from None
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _ask(samples, plt, show):
|
|
34
|
+
"""Asks, file by file, which populations to keep. Enter keeps the previous file's choice."""
|
|
35
|
+
if not sys.stdin.isatty():
|
|
36
|
+
raise SystemExit('no terminal to ask in; pass --pops, e.g. --pops "1,2"')
|
|
37
|
+
if show:
|
|
38
|
+
plt.show(block=False)
|
|
39
|
+
plt.pause(0.5)
|
|
40
|
+
print(
|
|
41
|
+
"\nWhich populations hold the cells you want? Each file's Pop numbers label the peaks "
|
|
42
|
+
"in its own panel of the local maxima figure, and the same number can be a different "
|
|
43
|
+
'peak in another file. Type numbers like 1 or 1,2 or 1-3, "all", or "none" to leave the file '
|
|
44
|
+
"out; Enter repeats the previous file's choice, Ctrl+C stops. Click this terminal "
|
|
45
|
+
"before typing."
|
|
46
|
+
)
|
|
47
|
+
prev, chosen = None, []
|
|
48
|
+
for i, s in enumerate(samples):
|
|
49
|
+
n = s["masks"].shape[1]
|
|
50
|
+
print(f"\nFile {i + 1}, {s['path'].name}:")
|
|
51
|
+
print("\n".join(s["table"]))
|
|
52
|
+
if n == 0:
|
|
53
|
+
print(" no populations found, so this file is left out")
|
|
54
|
+
chosen.append([])
|
|
55
|
+
continue
|
|
56
|
+
while True:
|
|
57
|
+
hint = f" [{','.join(map(str, prev)) or 'none'}]" if prev is not None else ""
|
|
58
|
+
text = _live_input(f"Keep which, Pop 1 to {n}{hint}: ", plt, show)
|
|
59
|
+
if not text.strip() and prev is not None:
|
|
60
|
+
text = ",".join(map(str, prev)) if prev else "none"
|
|
61
|
+
try:
|
|
62
|
+
pops = parse_pops(text, n)
|
|
63
|
+
break
|
|
64
|
+
except ValueError as e:
|
|
65
|
+
print(f" {e}")
|
|
66
|
+
chosen.append(pops)
|
|
67
|
+
prev = pops
|
|
68
|
+
return chosen
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _choose_pops(samples, spec, plt, show) -> list[list[int]]:
|
|
72
|
+
"""The populations to keep in each file: --pops, or asked file by file, the answers then
|
|
73
|
+
printed as the --pops that repeats them without the prompt."""
|
|
74
|
+
if spec is not None:
|
|
75
|
+
return pipeline.parse_selection(spec, [s["masks"].shape[1] for s in samples])
|
|
76
|
+
chosen = _ask(samples, plt, show)
|
|
77
|
+
print(f'\nTo repeat this choice: --pops "{pipeline.pops_spec(chosen)}"')
|
|
78
|
+
return chosen
|