flowcellar 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (159) hide show
  1. flowcellar/__init__.py +9 -0
  2. flowcellar/__main__.py +4 -0
  3. flowcellar/_version.py +1 -0
  4. flowcellar/aggregates/__init__.py +11 -0
  5. flowcellar/aggregates/build.py +151 -0
  6. flowcellar/aggregates/derive.py +109 -0
  7. flowcellar/aggregates/merge.py +362 -0
  8. flowcellar/catalogue/__init__.py +63 -0
  9. flowcellar/catalogue/metrics_nanopore.json +1 -0
  10. flowcellar/cli/__init__.py +97 -0
  11. flowcellar/cli/_options.py +25 -0
  12. flowcellar/cli/analysis.py +317 -0
  13. flowcellar/cli/discover.py +168 -0
  14. flowcellar/cli/doctor.py +31 -0
  15. flowcellar/cli/hub.py +47 -0
  16. flowcellar/cli/init.py +41 -0
  17. flowcellar/cli/ls.py +94 -0
  18. flowcellar/cli/recompute.py +52 -0
  19. flowcellar/cli/rm.py +21 -0
  20. flowcellar/cli/run.py +333 -0
  21. flowcellar/cli/scan.py +60 -0
  22. flowcellar/cli/schema.py +25 -0
  23. flowcellar/cli/serve.py +46 -0
  24. flowcellar/cli/show.py +109 -0
  25. flowcellar/cli/validate.py +44 -0
  26. flowcellar/config/__init__.py +4 -0
  27. flowcellar/config/defaults.toml +51 -0
  28. flowcellar/config/loader.py +369 -0
  29. flowcellar/contracts.py +159 -0
  30. flowcellar/discovery/__init__.py +141 -0
  31. flowcellar/discovery/guard.py +208 -0
  32. flowcellar/discovery/patterns.py +209 -0
  33. flowcellar/discovery/walk.py +348 -0
  34. flowcellar/engine/__init__.py +1 -0
  35. flowcellar/engine/api.py +119 -0
  36. flowcellar/engine/assemble.py +363 -0
  37. flowcellar/engine/baselines.py +128 -0
  38. flowcellar/engine/card.py +250 -0
  39. flowcellar/engine/copy_pending.py +110 -0
  40. flowcellar/engine/flowcell_use.py +85 -0
  41. flowcellar/engine/identity.py +351 -0
  42. flowcellar/engine/limits.py +135 -0
  43. flowcellar/engine/passes.py +213 -0
  44. flowcellar/engine/status.py +728 -0
  45. flowcellar/library/__init__.py +4 -0
  46. flowcellar/library/api.py +61 -0
  47. flowcellar/library/cards.py +69 -0
  48. flowcellar/library/draft.py +247 -0
  49. flowcellar/library/index.py +323 -0
  50. flowcellar/library/layout.py +69 -0
  51. flowcellar/library/lease.py +248 -0
  52. flowcellar/library/maintain.py +347 -0
  53. flowcellar/library/projects.py +84 -0
  54. flowcellar/library/query.py +382 -0
  55. flowcellar/library/scan.py +1770 -0
  56. flowcellar/library/store.py +225 -0
  57. flowcellar/library/trash.py +63 -0
  58. flowcellar/model/__init__.py +1 -0
  59. flowcellar/model/axes.py +200 -0
  60. flowcellar/model/bundle.py +143 -0
  61. flowcellar/model/partial.py +96 -0
  62. flowcellar/model/runfolder.py +83 -0
  63. flowcellar/model/runkey.py +98 -0
  64. flowcellar/model/schema.py +46 -0
  65. flowcellar/model/schemas/bundle-0.1.1.schema.json +3713 -0
  66. flowcellar/model/status.py +97 -0
  67. flowcellar/modules/__init__.py +0 -0
  68. flowcellar/modules/common.py +73 -0
  69. flowcellar/modules/dyn.py +124 -0
  70. flowcellar/modules/fc.py +390 -0
  71. flowcellar/modules/len_est.py +74 -0
  72. flowcellar/modules/run.py +88 -0
  73. flowcellar/modules/yield_.py +120 -0
  74. flowcellar/readdata/__init__.py +1 -0
  75. flowcellar/readdata/_pool.py +87 -0
  76. flowcellar/readdata/acc.py +161 -0
  77. flowcellar/readdata/alnacc.py +572 -0
  78. flowcellar/readdata/alnpass.py +85 -0
  79. flowcellar/readdata/api.py +189 -0
  80. flowcellar/readdata/bam.py +851 -0
  81. flowcellar/readdata/bamindex.py +149 -0
  82. flowcellar/readdata/cards.py +82 -0
  83. flowcellar/readdata/checks.py +312 -0
  84. flowcellar/readdata/copy_pending.py +73 -0
  85. flowcellar/readdata/fate.py +341 -0
  86. flowcellar/readdata/finder.py +147 -0
  87. flowcellar/readdata/libprep.py +97 -0
  88. flowcellar/readdata/link.py +486 -0
  89. flowcellar/readdata/passes.py +149 -0
  90. flowcellar/readdata/patterns.py +230 -0
  91. flowcellar/readdata/pychopper.py +431 -0
  92. flowcellar/readdata/readtable.py +416 -0
  93. flowcellar/readdata/record.py +159 -0
  94. flowcellar/readdata/reference.py +139 -0
  95. flowcellar/readdata/rnacomp.py +329 -0
  96. flowcellar/readdata/sample_sheet.py +153 -0
  97. flowcellar/readdata/saturation.py +110 -0
  98. flowcellar/readdata/sections.py +892 -0
  99. flowcellar/readdata/sections_align.py +644 -0
  100. flowcellar/readdata/tables.py +262 -0
  101. flowcellar/report/__init__.py +1 -0
  102. flowcellar/report/api.py +54 -0
  103. flowcellar/report/html.py +342 -0
  104. flowcellar/report/hub.py +159 -0
  105. flowcellar/report/index_one.py +144 -0
  106. flowcellar/report/served.py +67 -0
  107. flowcellar/sc/__init__.py +57 -0
  108. flowcellar/sc/axes.py +211 -0
  109. flowcellar/sc/catalogue.json +1 -0
  110. flowcellar/sc/catalogue.py +51 -0
  111. flowcellar/sc/check.py +57 -0
  112. flowcellar/sc/collect.py +299 -0
  113. flowcellar/sc/collect_sr.py +450 -0
  114. flowcellar/sc/collect_wf.py +479 -0
  115. flowcellar/sc/detect.py +303 -0
  116. flowcellar/sc/judge.py +269 -0
  117. flowcellar/sc/link.py +326 -0
  118. flowcellar/sc/listing.py +144 -0
  119. flowcellar/sc/maps.py +181 -0
  120. flowcellar/sc/readpass.py +416 -0
  121. flowcellar/sc/report.py +230 -0
  122. flowcellar/sc/scan.py +176 -0
  123. flowcellar/sc/spaceranger.py +483 -0
  124. flowcellar/sc/store.py +537 -0
  125. flowcellar/sc/wfsc.py +419 -0
  126. flowcellar/serve/__init__.py +4 -0
  127. flowcellar/serve/server.py +153 -0
  128. flowcellar/sources/__init__.py +0 -0
  129. flowcellar/sources/minknow.py +455 -0
  130. flowcellar/sources/porescan.py +218 -0
  131. flowcellar/sources/seqsummary.py +450 -0
  132. flowcellar/sources/workers.py +244 -0
  133. flowcellar/testing/__init__.py +1 -0
  134. flowcellar/testing/demo.py +1217 -0
  135. flowcellar/testing/demo_spatial.py +370 -0
  136. flowcellar/testing/fixtures.py +459 -0
  137. flowcellar/testing/libprep.py +252 -0
  138. flowcellar/testing/minknow_variants.py +384 -0
  139. flowcellar/testing/readdata.py +744 -0
  140. flowcellar/testing/sc.py +817 -0
  141. flowcellar/testing/trees.py +175 -0
  142. flowcellar/util/__init__.py +1 -0
  143. flowcellar/util/atomic.py +112 -0
  144. flowcellar/util/errors.py +72 -0
  145. flowcellar/util/fingerprint.py +49 -0
  146. flowcellar/util/log.py +50 -0
  147. flowcellar/util/text.py +43 -0
  148. flowcellar/util/timeutil.py +52 -0
  149. flowcellar/viewer_dist/favicon.svg +1 -0
  150. flowcellar/viewer_dist/manifest.json +37 -0
  151. flowcellar/viewer_dist/template.html +12 -0
  152. flowcellar/viewer_dist/viewer.core.js +3 -0
  153. flowcellar/viewer_dist/viewer.css +2151 -0
  154. flowcellar-0.1.0.dist-info/METADATA +506 -0
  155. flowcellar-0.1.0.dist-info/RECORD +159 -0
  156. flowcellar-0.1.0.dist-info/WHEEL +4 -0
  157. flowcellar-0.1.0.dist-info/entry_points.txt +2 -0
  158. flowcellar-0.1.0.dist-info/licenses/LICENSE +674 -0
  159. flowcellar-0.1.0.dist-info/licenses/THIRD_PARTY_NOTICES.md +85 -0
flowcellar/__init__.py ADDED
@@ -0,0 +1,9 @@
1
+ """FlowCellar: quality control for Oxford Nanopore sequencing runs."""
2
+ from flowcellar._version import __version__
3
+
4
+ #: Version of the bundle schema this package writes (src/flowcellar/model/schemas/).
5
+ SCHEMA_VERSION = "0.1.1"
6
+ #: Layout version of a library folder (the LIBRARY marker file).
7
+ LAYOUT_VERSION = 1
8
+
9
+ __all__ = ["__version__", "SCHEMA_VERSION", "LAYOUT_VERSION"]
flowcellar/__main__.py ADDED
@@ -0,0 +1,4 @@
1
+ from flowcellar.cli import main
2
+
3
+ if __name__ == "__main__":
4
+ main()
flowcellar/_version.py ADDED
@@ -0,0 +1 @@
1
+ __version__ = "0.1.0"
@@ -0,0 +1,11 @@
1
+ """The aggregate kinds of a bundle: building them, merging them, deriving numbers from them.
2
+
3
+ Every stored number is one of nine kinds; each kind has one merge rule. Only integers are accumulated, so
4
+ merging in any order is exact.
5
+
6
+ from flowcellar.aggregates import build, merge, derive
7
+ """
8
+ from flowcellar.aggregates import build, derive, merge # noqa: F401
9
+ from flowcellar.aggregates.merge import NotMergeable, merge_cell, merge_cells, merge_kind, rollup # noqa: F401
10
+
11
+ KINDS = ("scalars", "hists", "hists2d", "series", "counts", "channels", "tables", "curves", "grids")
@@ -0,0 +1,151 @@
1
+ """From numpy accumulators to the stored (JSON) form of each aggregate kind.
2
+
3
+ Internal histogram arrays are ``(under, bin 0 ... bin n-1, over)``: ``numpy.bincount`` of ``axes.bin_index``.
4
+ """
5
+ from __future__ import annotations
6
+
7
+ import math
8
+
9
+ import numpy as np
10
+
11
+ from flowcellar.model.axes import EDGES
12
+
13
+ B64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"
14
+ B64URL = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"
15
+
16
+
17
+ def hist(axis: str, full, w_full=None, *, w_name: str | None = None, missing: int | None = None,
18
+ vmin=None, vmax=None) -> dict:
19
+ """Internal array -> stored histogram with empty ends trimmed. ``w_full`` is the per-cell sum of the binned
20
+ quantity (bases for length); its under- and overflow cells are kept as ``w_under`` and ``w_over``."""
21
+ full = np.asarray(full)
22
+ if len(full) != len(EDGES[axis]) + 1:
23
+ raise ValueError(f"{axis}: expected {len(EDGES[axis]) + 1} cells, got {len(full)}")
24
+ inner = full[1:-1]
25
+ nz = np.nonzero(inner)[0]
26
+ a, b = (int(nz[0]), int(nz[-1]) + 1) if len(nz) else (0, 0)
27
+ h: dict = {"axis": axis, "start": a, "n": [int(v) for v in inner[a:b]]}
28
+ if w_full is not None:
29
+ w_full = np.asarray(w_full)
30
+ h["w"] = [int(v) for v in w_full[1:-1][a:b]]
31
+ if w_name:
32
+ h["w_name"] = w_name
33
+ h["w_under"] = int(w_full[0])
34
+ h["w_over"] = int(w_full[-1])
35
+ h["under"] = int(full[0])
36
+ h["over"] = int(full[-1])
37
+ if missing is not None:
38
+ h["missing"] = int(missing)
39
+ if vmin is not None:
40
+ h["min"] = vmin
41
+ if vmax is not None:
42
+ h["max"] = vmax
43
+ return h
44
+
45
+
46
+ def hist_full(h: dict, key: str = "n") -> np.ndarray:
47
+ """Stored histogram -> internal array (under, bins..., over) of ``n`` or ``w``."""
48
+ out = np.zeros(len(EDGES[h["axis"]]) + 1, np.int64)
49
+ vals = h.get(key) or []
50
+ out[1 + h["start"]: 1 + h["start"] + len(vals)] = vals
51
+ if key == "n":
52
+ out[0], out[-1] = h.get("under", 0), h.get("over", 0)
53
+ else:
54
+ out[0], out[-1] = h.get("w_under", 0), h.get("w_over", 0)
55
+ return out
56
+
57
+
58
+ def hist2d(x: str, y: str, matrix, w_matrix=None, *, outside: int = 0, missing: int | None = None,
59
+ w_name: str | None = None) -> dict:
60
+ """Matrix of the INNER bins (rows = x bins, columns = y bins) -> stored sparse 2-D histogram."""
61
+ m = np.asarray(matrix)
62
+ i, j = np.nonzero(m)
63
+ h: dict = {"x": x, "y": y, "enc": "coo", "i": [int(v) for v in i], "j": [int(v) for v in j],
64
+ "n": [int(v) for v in m[i, j]]}
65
+ if w_matrix is not None:
66
+ h["w"] = [int(v) for v in np.asarray(w_matrix)[i, j]]
67
+ if w_name:
68
+ h["w_name"] = w_name
69
+ h["outside"] = int(outside)
70
+ if missing is not None:
71
+ h["missing"] = int(missing)
72
+ return h
73
+
74
+
75
+ def series(axis: str, values: dict, *, unit: str | None = None, levels: dict[str, str] | None = None,
76
+ n_bins: int | None = None, start: int = 0) -> dict:
77
+ """Named arrays on one axis. Increments are cut after the last bin in which any of them is non-zero (never
78
+ before ``n_bins``); ``levels`` maps the names that are levels to their merge rule and may hold None values."""
79
+ levels = levels or {}
80
+ inc = {k: np.asarray(v) for k, v in values.items() if k not in levels}
81
+ last = 0
82
+ for v in inc.values():
83
+ nz = np.nonzero(v)[0]
84
+ if len(nz):
85
+ last = max(last, int(nz[-1]) + 1)
86
+ for k in levels:
87
+ last = max(last, len(values[k]))
88
+ if n_bins is not None:
89
+ last = max(last, n_bins)
90
+ out_vals: dict = {}
91
+ for k, v in values.items():
92
+ if k in levels:
93
+ vv = [None if x is None else (x if isinstance(x, (int, float)) else float(x)) for x in v]
94
+ out_vals[k] = vv + [None] * (last - len(vv))
95
+ else:
96
+ vv = [int(x) for x in np.asarray(v)[:last]]
97
+ out_vals[k] = vv + [0] * (last - len(vv))
98
+ s: dict = {"axis": axis, "start": start}
99
+ if unit:
100
+ s["unit"] = unit
101
+ s["values"] = out_vals
102
+ if levels:
103
+ s["levels"] = sorted(levels)
104
+ s["merge"] = dict(levels)
105
+ return s
106
+
107
+
108
+ def counts(cats: list[str], n: list[int], *, w: list[int] | None = None, role: str | None = None,
109
+ missing: list[str] | None = None, sources: dict | None = None) -> dict:
110
+ c: dict = {"cats": list(cats), "n": [int(v) for v in n]}
111
+ if w is not None:
112
+ c["w"] = [int(v) for v in w]
113
+ if role:
114
+ c["role"] = role
115
+ if missing:
116
+ c["missing"] = list(missing)
117
+ if sources:
118
+ c["sources"] = sources
119
+ return c
120
+
121
+
122
+ def largest_remainder(vals, total: int) -> list[int]:
123
+ """Integers that sum to ``total``, each within 1 of its exact value."""
124
+ fl = [int(math.floor(v)) for v in vals]
125
+ rem = int(total - sum(fl))
126
+ order = sorted(range(len(vals)), key=lambda i: vals[i] - fl[i], reverse=True)
127
+ for i in order[: max(rem, 0)]:
128
+ fl[i] += 1
129
+ return fl
130
+
131
+
132
+ def u6_encode(levels, alphabet: str = B64) -> str:
133
+ """One character per cell (levels 0..63), row-major: grids ``enc: u6`` and the index sparklines (B64URL)."""
134
+ lut = np.frombuffer(alphabet.encode(), dtype=np.uint8)
135
+ return lut[np.clip(np.asarray(levels), 0, 63).astype(np.int64).ravel()].tobytes().decode()
136
+
137
+
138
+ def u6_decode(text: str, alphabet: str = B64) -> np.ndarray:
139
+ lut = np.full(256, 0, np.uint8)
140
+ lut[np.frombuffer(alphabet.encode(), dtype=np.uint8)] = np.arange(64, dtype=np.uint8)
141
+ return lut[np.frombuffer(text.encode(), dtype=np.uint8)]
142
+
143
+
144
+ def sparkline(values, *, points: int = 32) -> tuple[str, float]:
145
+ """(32 characters of the base64url alphabet, the row's maximum): 6 bits per point, scaled to the row's maximum."""
146
+ v = np.zeros(points)
147
+ vals = np.asarray(values, dtype=np.float64)[:points]
148
+ v[: len(vals)] = vals
149
+ mx = float(v.max()) if len(v) else 0.0
150
+ lv = np.rint(v / mx * 63).astype(np.int64) if mx > 0 else np.zeros(points, np.int64)
151
+ return u6_encode(lv, B64URL), mx
@@ -0,0 +1,109 @@
1
+ """The fixed set of derivations: numbers read from stored aggregates.
2
+
3
+ Quantiles and N50 are read from histograms by rank interpolation inside a bin (geometric on log axes); they are
4
+ exact to bin resolution. The viewer implements the same functions; ``tests/vectors/derive`` holds them equal.
5
+ """
6
+ from __future__ import annotations
7
+
8
+ import numpy as np
9
+
10
+ from flowcellar.aggregates.build import hist_full
11
+ from flowcellar.model.axes import AXES, EDGES
12
+
13
+
14
+ def total(h: dict, key: str = "n") -> int:
15
+ """All values of a histogram, under- and overflow included (not ``missing``)."""
16
+ if key == "n":
17
+ return int(sum(h.get("n", [])) + h.get("under", 0) + h.get("over", 0))
18
+ return int(sum(h.get("w", [])) + h.get("w_under", 0) + h.get("w_over", 0))
19
+
20
+
21
+ def ratio(num, den):
22
+ return None if num is None or not den else num / den
23
+
24
+
25
+ def mean(h: dict) -> float | None:
26
+ """Exact mean of the binned quantity: sum of weights over count (needs ``w``)."""
27
+ n = total(h, "n")
28
+ return None if "w" not in h or not n else total(h, "w") / n
29
+
30
+
31
+ def _interp(axis: str, b: int, frac: float) -> float:
32
+ e = EDGES[axis]
33
+ if b <= 0:
34
+ return float(e[0])
35
+ if b >= len(e):
36
+ return float(e[-1])
37
+ lo, hi = e[b - 1], e[b]
38
+ if AXES[axis]["type"] == "log10":
39
+ return float(lo * (hi / lo) ** frac)
40
+ return float(lo + (hi - lo) * frac)
41
+
42
+
43
+ def quantile(h: dict, q: float) -> float | None:
44
+ """The q-quantile of the binned values (0 < q < 1); None for an empty histogram."""
45
+ c = hist_full(h, "n").astype(np.float64)
46
+ tot = c.sum()
47
+ if tot <= 0:
48
+ return None
49
+ cum = np.cumsum(c)
50
+ b = int(np.searchsorted(cum, q * tot, side="left"))
51
+ frac = (q * tot - (cum[b - 1] if b else 0.0)) / c[b] if c[b] else 0.0
52
+ return _interp(h["axis"], b, frac)
53
+
54
+
55
+ def median(h: dict) -> float | None:
56
+ return quantile(h, 0.5)
57
+
58
+
59
+ def nx(h: dict, x: float = 0.5) -> float | None:
60
+ """N50-type statistic from the per-bin sums ``w``: the value at which the bins above it hold the share x of the
61
+ weighted total (N50: x = 0.5)."""
62
+ if "w" not in h:
63
+ return None
64
+ w = hist_full(h, "w").astype(np.float64)
65
+ tot = w.sum()
66
+ if tot <= 0:
67
+ return None
68
+ cum = np.cumsum(w)
69
+ e = EDGES[h["axis"]]
70
+ b = int(np.searchsorted(cum, (1.0 - x) * tot, side="left"))
71
+ b = min(max(b, 1), len(e) - 1)
72
+ frac = ((1.0 - x) * tot - cum[b - 1]) / w[b] if w[b] > 0 else 0.0
73
+ return _interp(h["axis"], b, frac)
74
+
75
+
76
+ def frac_above(h: dict, x: float, key: str = "n") -> float | None:
77
+ """Share of the values (or of the weight) at or above ``x``; ``x`` is snapped down to the edge of its bin."""
78
+ c = hist_full(h, key).astype(np.float64)
79
+ tot = c.sum()
80
+ if tot <= 0:
81
+ return None
82
+ b = int(np.searchsorted(EDGES[h["axis"]], x, side="right"))
83
+ return float(c[b:].sum() / tot)
84
+
85
+
86
+ def mode(h: dict) -> float | None:
87
+ """Centre of the fullest bin (geometric on log axes)."""
88
+ n = h.get("n") or []
89
+ if not n or max(n) <= 0:
90
+ return None
91
+ return _interp(h["axis"], 1 + h["start"] + int(np.argmax(n)), 0.5)
92
+
93
+
94
+ def cumulative(values) -> list:
95
+ return [int(v) if float(v).is_integer() else float(v) for v in np.cumsum(np.asarray(values, dtype=np.float64))]
96
+
97
+
98
+ def rate(values, step_s: float) -> list[float]:
99
+ """Per-second rate of per-bin increments."""
100
+ return [float(v) / step_s for v in values]
101
+
102
+
103
+ def gini(x) -> float | None:
104
+ """Gini coefficient of per-channel output (0 = even)."""
105
+ x = np.sort(np.asarray(x, dtype=np.float64))
106
+ n, s = len(x), x.sum()
107
+ if n == 0 or s <= 0:
108
+ return None
109
+ return float((2 * np.sum(np.arange(1, n + 1) * x) / (n * s)) - (n + 1) / n)
@@ -0,0 +1,362 @@
1
+ """Merging the stored form of the aggregate kinds, and rolling a cube up.
2
+
3
+ All functions return a new object and leave their arguments untouched. Merging is associative and
4
+ commutative for every kind that is mergeable (``curves`` and ``grids`` are not: they raise NotMergeable).
5
+ """
6
+ from __future__ import annotations
7
+
8
+ import copy
9
+
10
+ from flowcellar.model import axes as _axes
11
+
12
+ MERGE_RULES = ("sum", "min", "max", "mean", "first", "last", "same")
13
+
14
+
15
+ class NotMergeable(ValueError):
16
+ """Two aggregates that cannot be added: different axes outside one family, a curve, a grid."""
17
+
18
+
19
+ # ---------------------------------------------------------------------------------------- scalars
20
+ def _merge_value(a, b, rule: str):
21
+ """One value from two under a merge rule. None is "not known": a sum with an unknown part is unknown; for
22
+ every other rule the known side is kept."""
23
+ if rule == "sum":
24
+ return None if a is None or b is None else a + b
25
+ if a is None:
26
+ return b
27
+ if b is None:
28
+ return a
29
+ if rule == "min":
30
+ return min(a, b)
31
+ if rule == "max":
32
+ return max(a, b)
33
+ if rule == "first":
34
+ return a
35
+ if rule == "last":
36
+ return b
37
+ if rule == "same":
38
+ return a if a == b else None
39
+ raise NotMergeable(f"rule {rule!r} needs a weight and cannot be applied to two values")
40
+
41
+
42
+ def merge_scalars(a: dict, b: dict, rules: dict | None = None) -> dict:
43
+ """Add; other rules per id in ``rules`` (the section's ``scalar_merge``). An id present on one side only is
44
+ kept as it is. A sum with an unknown (None) part is unknown."""
45
+ rules = rules or {}
46
+ out = dict(a)
47
+ for k, v in b.items():
48
+ out[k] = _merge_value(a[k], v, rules.get(k, "sum")) if k in a else v
49
+ return out
50
+
51
+
52
+ # ------------------------------------------------------------------------------------- histograms
53
+ def _to_axis(h: dict, axis: str) -> dict:
54
+ """Re-bin a stored histogram exactly onto a coarser axis of its family."""
55
+ if h["axis"] == axis:
56
+ return h
57
+ f = _axes.coarsen_factor(h["axis"], axis)
58
+ out = {k: v for k, v in h.items() if k not in ("n", "w", "start", "axis")}
59
+ out["axis"] = axis
60
+ start = h["start"] // f
61
+ for key in ("n", "w"):
62
+ if key not in h:
63
+ continue
64
+ vals = [0] * (h["start"] - start * f) + list(h[key])
65
+ vals += [0] * (-len(vals) % f)
66
+ out[key] = [sum(vals[i:i + f]) for i in range(0, len(vals), f)]
67
+ out["start"] = start
68
+ out.setdefault("n", [])
69
+ return out
70
+
71
+
72
+ def _coarser_common(a: str, b: str) -> str:
73
+ if a == b:
74
+ return a
75
+ if b in _axes.AXES[a].get("coarsens_to", []):
76
+ return b
77
+ if a in _axes.AXES[b].get("coarsens_to", []):
78
+ return a
79
+ raise NotMergeable(f"axes {a} and {b} are not levels of one family")
80
+
81
+
82
+ def _add_offset(sa: int, va: list, sb: int, vb: list) -> tuple[int, list]:
83
+ if not va:
84
+ return (sb, list(vb)) if vb else (0, [])
85
+ if not vb:
86
+ return sa, list(va)
87
+ s = min(sa, sb)
88
+ e = max(sa + len(va), sb + len(vb))
89
+ out = [0] * (e - s)
90
+ for i, v in enumerate(va):
91
+ out[sa - s + i] += v
92
+ for i, v in enumerate(vb):
93
+ out[sb - s + i] += v
94
+ return s, out
95
+
96
+
97
+ def merge_hist(a: dict, b: dict) -> dict:
98
+ """Add bin by bin. Two levels of one axis family merge exactly on the coarser one."""
99
+ axis = _coarser_common(a["axis"], b["axis"])
100
+ a, b = _to_axis(a, axis), _to_axis(b, axis)
101
+ start, n = _add_offset(a["start"], a.get("n", []), b["start"], b.get("n", []))
102
+ out: dict = {"axis": axis, "start": start, "n": n}
103
+ if "w" in a or "w" in b:
104
+ if ("w" in a) != ("w" in b) and (a.get("n") and b.get("n")):
105
+ raise NotMergeable("one histogram has weights and the other has none")
106
+ _, out["w"] = _add_offset(a["start"], a.get("w", []), b["start"], b.get("w", []))
107
+ if len(out["w"]) != len(n):
108
+ out["w"] = out["w"] + [0] * (len(n) - len(out["w"]))
109
+ if a.get("w_name") or b.get("w_name"):
110
+ out["w_name"] = a.get("w_name") or b.get("w_name")
111
+ for k in ("under", "over", "missing", "w_under", "w_over"):
112
+ if k in a or k in b:
113
+ out[k] = a.get(k, 0) + b.get(k, 0)
114
+ for k, fn in (("min", min), ("max", max)):
115
+ vals = [x for x in (a.get(k), b.get(k)) if x is not None]
116
+ if vals:
117
+ out[k] = fn(vals)
118
+ # trim empty ends so that the result does not depend on the order of merging
119
+ while out["n"] and out["n"][0] == 0 and (not out.get("w") or out["w"][0] == 0):
120
+ out["n"].pop(0); out["start"] += 1
121
+ if "w" in out:
122
+ out["w"].pop(0)
123
+ while out["n"] and out["n"][-1] == 0 and (not out.get("w") or out["w"][-1] == 0):
124
+ out["n"].pop()
125
+ if "w" in out:
126
+ out["w"].pop()
127
+ if not out["n"]:
128
+ out["start"] = 0
129
+ return out
130
+
131
+
132
+ def merge_hist2d(a: dict, b: dict) -> dict:
133
+ """Add cell by cell (sparse ``coo`` encoding; cells stay sorted by (i, j))."""
134
+ if (a["x"], a["y"]) != (b["x"], b["y"]):
135
+ raise NotMergeable(f"2-D histograms on different axes: {a['x']} x {a['y']} and {b['x']} x {b['y']}")
136
+ if a.get("enc", "coo") != "coo" or b.get("enc", "coo") != "coo":
137
+ raise NotMergeable("only the coo encoding is merged")
138
+ has_w = "w" in a or "w" in b
139
+ cells: dict[tuple[int, int], list[int]] = {}
140
+ for h in (a, b):
141
+ w = h.get("w") or [0] * len(h["n"])
142
+ for i, j, n, ww in zip(h.get("i", []), h.get("j", []), h["n"], w):
143
+ c = cells.setdefault((i, j), [0, 0])
144
+ c[0] += n
145
+ c[1] += ww
146
+ keys = sorted(k for k, v in cells.items() if v[0] or v[1])
147
+ out: dict = {"x": a["x"], "y": a["y"], "enc": "coo", "i": [k[0] for k in keys], "j": [k[1] for k in keys],
148
+ "n": [cells[k][0] for k in keys]}
149
+ if has_w:
150
+ out["w"] = [cells[k][1] for k in keys]
151
+ if a.get("w_name") or b.get("w_name"):
152
+ out["w_name"] = a.get("w_name") or b.get("w_name")
153
+ for k in ("outside", "w_outside", "missing"):
154
+ if k in a or k in b:
155
+ out[k] = a.get(k, 0) + b.get(k, 0)
156
+ return out
157
+
158
+
159
+ # ----------------------------------------------------------------------------------------- series
160
+ def merge_series(a: dict, b: dict) -> dict:
161
+ """Increments add bin by bin. A level (listed in ``levels``) follows its rule in ``merge``; ``mean`` needs the
162
+ array named by ``weights``."""
163
+ if a["axis"] != b["axis"]:
164
+ raise NotMergeable(f"series on different axes: {a['axis']} and {b['axis']}")
165
+ levels = set(a.get("levels", [])) | set(b.get("levels", []))
166
+ rules = {**(b.get("merge") or {}), **(a.get("merge") or {})}
167
+ s = min(a["start"], b["start"])
168
+ la = max((len(v) for v in a["values"].values()), default=0)
169
+ lb = max((len(v) for v in b["values"].values()), default=0)
170
+ n = max(a["start"] + la, b["start"] + lb) - s
171
+
172
+ def spread(src: dict, name: str, fill):
173
+ vals = src["values"].get(name)
174
+ out = [fill] * n
175
+ if vals is not None:
176
+ off = src["start"] - s
177
+ out[off: off + len(vals)] = vals
178
+ return out
179
+ wname = a.get("weights") or b.get("weights")
180
+ values: dict = {}
181
+ for name in list(a["values"]) + [k for k in b["values"] if k not in a["values"]]:
182
+ if name not in levels:
183
+ va, vb = spread(a, name, 0), spread(b, name, 0)
184
+ values[name] = [x + y for x, y in zip(va, vb)]
185
+ continue
186
+ rule = rules.get(name)
187
+ va, vb = spread(a, name, None), spread(b, name, None)
188
+ if rule == "mean":
189
+ if not wname:
190
+ raise NotMergeable(f"level {name!r} has rule mean and the series names no weights")
191
+ wa, wb = spread(a, wname, 0), spread(b, wname, 0)
192
+ values[name] = [y if x is None else x if y is None else ((x * p + y * q) / (p + q) if p + q else None)
193
+ for x, y, p, q in zip(va, vb, wa, wb)]
194
+ elif rule in MERGE_RULES and rule != "sum":
195
+ values[name] = [_merge_value(x, y, rule) for x, y in zip(va, vb)]
196
+ else:
197
+ raise NotMergeable(f"level {name!r} has no merge rule")
198
+ out: dict = {"axis": a["axis"], "start": s}
199
+ if a.get("unit") or b.get("unit"):
200
+ out["unit"] = a.get("unit") or b.get("unit")
201
+ out["values"] = values
202
+ if levels:
203
+ out["levels"] = sorted(levels)
204
+ if rules:
205
+ out["merge"] = rules
206
+ if wname:
207
+ out["weights"] = wname
208
+ return out
209
+
210
+
211
+ # ----------------------------------------------------------------------------------------- counts
212
+ def merge_counts(a: dict, b: dict) -> dict:
213
+ """Union of the categories in order of first appearance, counts added. A category that is not known on one
214
+ side (listed in ``missing``) stays not known: unknown plus a number is unknown."""
215
+ cats = list(a["cats"]) + [c for c in b["cats"] if c not in a["cats"]]
216
+ missing = [c for c in cats if c in (a.get("missing") or []) or c in (b.get("missing") or [])]
217
+ ia = {c: k for k, c in enumerate(a["cats"])}
218
+ ib = {c: k for k, c in enumerate(b["cats"])}
219
+
220
+ def add(key):
221
+ return [(a[key][ia[c]] if c in ia else 0) + (b[key][ib[c]] if c in ib else 0) for c in cats]
222
+ out: dict = {"cats": cats, "n": add("n")}
223
+ if "w" in a and "w" in b:
224
+ out["w"] = add("w")
225
+ if a.get("w_name") or b.get("w_name"):
226
+ out["w_name"] = a.get("w_name") or b.get("w_name")
227
+ if a.get("other") is not None or b.get("other") is not None:
228
+ out["other"] = int(a.get("other") or 0) + int(b.get("other") or 0) # what was folded away adds up too
229
+ if a.get("role") or b.get("role"):
230
+ out["role"] = a.get("role") or b.get("role")
231
+ if missing:
232
+ out["missing"] = missing
233
+ if a.get("sources") or b.get("sources"):
234
+ out["sources"] = {**(b.get("sources") or {}), **(a.get("sources") or {})}
235
+ return out
236
+
237
+
238
+ # --------------------------------------------------------------------------------------- channels
239
+ def merge_channels(a: dict, b: dict) -> dict:
240
+ """Per-channel arrays of one layout: add, or the per-value rule in ``merge`` (max for the last read time)."""
241
+ if a.get("layout") != b.get("layout") or a.get("wells") != b.get("wells"):
242
+ raise NotMergeable("channel maps of different layouts")
243
+ rules = {**(b.get("merge") or {}), **(a.get("merge") or {})}
244
+ values: dict = {}
245
+ for name in list(a["values"]) + [k for k in b["values"] if k not in a["values"]]:
246
+ va, vb = a["values"].get(name), b["values"].get(name)
247
+ if va is None or vb is None:
248
+ values[name] = list(va if vb is None else vb)
249
+ continue
250
+ if len(va) != len(vb):
251
+ raise NotMergeable(f"channel value {name!r} has {len(va)} and {len(vb)} entries")
252
+ rule = rules.get(name, "sum")
253
+ values[name] = [_merge_value(x, y, rule) for x, y in zip(va, vb)]
254
+ out = {k: v for k, v in a.items() if k not in ("values", "merge")}
255
+ out["values"] = values
256
+ if rules:
257
+ out["merge"] = rules
258
+ return out
259
+
260
+
261
+ # ----------------------------------------------------------------------------------------- tables
262
+ def merge_table(a: dict, b: dict) -> dict:
263
+ """``merge``: ``concat`` (rows of both), ``by_key`` (rows with an equal key are added column by column where
264
+ numeric), ``replace`` (the later one wins)."""
265
+ how = a.get("merge", "concat")
266
+ if how == "replace":
267
+ return copy.deepcopy(b)
268
+ if [c["id"] for c in a["columns"]] != [c["id"] for c in b["columns"]]:
269
+ raise NotMergeable("tables with different columns")
270
+ out = {k: copy.deepcopy(v) for k, v in a.items() if k != "rows"}
271
+ if how == "concat":
272
+ out["rows"] = [list(r) for r in a["rows"]] + [list(r) for r in b["rows"]]
273
+ if "total_rows" in a or "total_rows" in b:
274
+ out["total_rows"] = a.get("total_rows", len(a["rows"])) + b.get("total_rows", len(b["rows"]))
275
+ return out
276
+ if how == "by_key":
277
+ ids = [c["id"] for c in a["columns"]]
278
+ kidx = [ids.index(k) for k in a.get("key", [])]
279
+ rows: dict[tuple, list] = {}
280
+ for r in list(a["rows"]) + list(b["rows"]):
281
+ key = tuple(r[i] for i in kidx)
282
+ if key not in rows:
283
+ rows[key] = list(r)
284
+ else:
285
+ cur = rows[key]
286
+ for i, v in enumerate(r):
287
+ if i not in kidx and isinstance(v, (int, float)) and not isinstance(v, bool) \
288
+ and isinstance(cur[i], (int, float)) and not isinstance(cur[i], bool):
289
+ cur[i] = cur[i] + v
290
+ out["rows"] = list(rows.values())
291
+ return out
292
+ raise NotMergeable(f"table merge rule {how!r}")
293
+
294
+
295
+ _MERGERS = {"hists": merge_hist, "hists2d": merge_hist2d, "series": merge_series, "counts": merge_counts,
296
+ "channels": merge_channels, "tables": merge_table}
297
+
298
+
299
+ def merge_kind(kind: str, a: dict, b: dict):
300
+ """Merge two aggregates of one kind (not ``scalars``, which need the section's rules)."""
301
+ if kind not in _MERGERS:
302
+ raise NotMergeable(f"aggregates of kind {kind!r} are not mergeable")
303
+ return _MERGERS[kind](a, b)
304
+
305
+
306
+ # ------------------------------------------------------------------------------------------ cells
307
+ def merge_cell(a: dict, b: dict, scalar_merge: dict | None = None, *, at: list | None = None) -> dict:
308
+ """Two cells (or two ``run_level`` objects) added member by member. ``at`` defaults to that of ``a``."""
309
+ out: dict = {}
310
+ if "at" in a or at is not None:
311
+ out["at"] = list(at if at is not None else a["at"])
312
+ if "scalars" in a or "scalars" in b:
313
+ out["scalars"] = merge_scalars(a.get("scalars", {}), b.get("scalars", {}), scalar_merge)
314
+ for kind in ("hists", "hists2d", "series", "counts", "channels", "tables"):
315
+ if kind not in a and kind not in b:
316
+ continue
317
+ ka, kb = a.get(kind, {}), b.get(kind, {})
318
+ out[kind] = {name: (merge_kind(kind, ka[name], kb[name]) if name in ka and name in kb
319
+ else copy.deepcopy(ka.get(name, kb.get(name))))
320
+ for name in list(ka) + [n for n in kb if n not in ka]}
321
+ for kind in ("curves", "grids"):
322
+ if kind in a and kind in b:
323
+ raise NotMergeable(f"cells that both hold {kind} cannot be merged")
324
+ if kind in a or kind in b:
325
+ out[kind] = copy.deepcopy(a.get(kind, b.get(kind)))
326
+ return out
327
+
328
+
329
+ def merge_cells(cells_a: list[dict], cells_b: list[dict], scalar_merge: dict | None = None) -> list[dict]:
330
+ """Two cell lists of one section (same dims): cells with an equal ``at`` are merged, the others kept."""
331
+ out: dict[tuple, dict] = {}
332
+ for c in list(cells_a) + list(cells_b):
333
+ key = tuple(c["at"])
334
+ out[key] = merge_cell(out[key], c, scalar_merge) if key in out else copy.deepcopy(c)
335
+ return list(out.values())
336
+
337
+
338
+ def rollup(cells: list[dict], dims: list[str], keep: list[str], scalar_merge: dict | None = None,
339
+ where: dict | None = None) -> list[dict]:
340
+ """Sum the cube over every dimension that is not in ``keep``.
341
+
342
+ ``where`` restricts first: ``{"filter": ["pass"]}``. ``rollup(cells, dims, [])`` is the run total;
343
+ ``rollup(cells, ["sample", "filter"], ["sample"])`` the per-sample view. Cells never overlap, so the sum is exact.
344
+ """
345
+ idx = [dims.index(d) for d in keep]
346
+ rolled = len(idx) != len(dims)
347
+ out: dict[tuple, dict] = {}
348
+ for c in cells:
349
+ if where and any(c["at"][dims.index(d)] not in allowed for d, allowed in where.items()):
350
+ continue
351
+ key = tuple(c["at"][i] for i in idx)
352
+ if rolled and ("curves" in c or "grids" in c):
353
+ # curves and grids are derived per cell and cannot be added: a view over fewer dimensions has none,
354
+ # however many of its cells held one
355
+ c = {k: v for k, v in c.items() if k not in ("curves", "grids")}
356
+ if key in out:
357
+ out[key] = merge_cell(out[key], c, scalar_merge, at=list(key))
358
+ else:
359
+ cc = copy.deepcopy(c)
360
+ cc["at"] = list(key)
361
+ out[key] = cc
362
+ return list(out.values())