flowcellar 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flowcellar/__init__.py +9 -0
- flowcellar/__main__.py +4 -0
- flowcellar/_version.py +1 -0
- flowcellar/aggregates/__init__.py +11 -0
- flowcellar/aggregates/build.py +151 -0
- flowcellar/aggregates/derive.py +109 -0
- flowcellar/aggregates/merge.py +362 -0
- flowcellar/catalogue/__init__.py +63 -0
- flowcellar/catalogue/metrics_nanopore.json +1 -0
- flowcellar/cli/__init__.py +97 -0
- flowcellar/cli/_options.py +25 -0
- flowcellar/cli/analysis.py +317 -0
- flowcellar/cli/discover.py +168 -0
- flowcellar/cli/doctor.py +31 -0
- flowcellar/cli/hub.py +47 -0
- flowcellar/cli/init.py +41 -0
- flowcellar/cli/ls.py +94 -0
- flowcellar/cli/recompute.py +52 -0
- flowcellar/cli/rm.py +21 -0
- flowcellar/cli/run.py +333 -0
- flowcellar/cli/scan.py +60 -0
- flowcellar/cli/schema.py +25 -0
- flowcellar/cli/serve.py +46 -0
- flowcellar/cli/show.py +109 -0
- flowcellar/cli/validate.py +44 -0
- flowcellar/config/__init__.py +4 -0
- flowcellar/config/defaults.toml +51 -0
- flowcellar/config/loader.py +369 -0
- flowcellar/contracts.py +159 -0
- flowcellar/discovery/__init__.py +141 -0
- flowcellar/discovery/guard.py +208 -0
- flowcellar/discovery/patterns.py +209 -0
- flowcellar/discovery/walk.py +348 -0
- flowcellar/engine/__init__.py +1 -0
- flowcellar/engine/api.py +119 -0
- flowcellar/engine/assemble.py +363 -0
- flowcellar/engine/baselines.py +128 -0
- flowcellar/engine/card.py +250 -0
- flowcellar/engine/copy_pending.py +110 -0
- flowcellar/engine/flowcell_use.py +85 -0
- flowcellar/engine/identity.py +351 -0
- flowcellar/engine/limits.py +135 -0
- flowcellar/engine/passes.py +213 -0
- flowcellar/engine/status.py +728 -0
- flowcellar/library/__init__.py +4 -0
- flowcellar/library/api.py +61 -0
- flowcellar/library/cards.py +69 -0
- flowcellar/library/draft.py +247 -0
- flowcellar/library/index.py +323 -0
- flowcellar/library/layout.py +69 -0
- flowcellar/library/lease.py +248 -0
- flowcellar/library/maintain.py +347 -0
- flowcellar/library/projects.py +84 -0
- flowcellar/library/query.py +382 -0
- flowcellar/library/scan.py +1770 -0
- flowcellar/library/store.py +225 -0
- flowcellar/library/trash.py +63 -0
- flowcellar/model/__init__.py +1 -0
- flowcellar/model/axes.py +200 -0
- flowcellar/model/bundle.py +143 -0
- flowcellar/model/partial.py +96 -0
- flowcellar/model/runfolder.py +83 -0
- flowcellar/model/runkey.py +98 -0
- flowcellar/model/schema.py +46 -0
- flowcellar/model/schemas/bundle-0.1.1.schema.json +3713 -0
- flowcellar/model/status.py +97 -0
- flowcellar/modules/__init__.py +0 -0
- flowcellar/modules/common.py +73 -0
- flowcellar/modules/dyn.py +124 -0
- flowcellar/modules/fc.py +390 -0
- flowcellar/modules/len_est.py +74 -0
- flowcellar/modules/run.py +88 -0
- flowcellar/modules/yield_.py +120 -0
- flowcellar/readdata/__init__.py +1 -0
- flowcellar/readdata/_pool.py +87 -0
- flowcellar/readdata/acc.py +161 -0
- flowcellar/readdata/alnacc.py +572 -0
- flowcellar/readdata/alnpass.py +85 -0
- flowcellar/readdata/api.py +189 -0
- flowcellar/readdata/bam.py +851 -0
- flowcellar/readdata/bamindex.py +149 -0
- flowcellar/readdata/cards.py +82 -0
- flowcellar/readdata/checks.py +312 -0
- flowcellar/readdata/copy_pending.py +73 -0
- flowcellar/readdata/fate.py +341 -0
- flowcellar/readdata/finder.py +147 -0
- flowcellar/readdata/libprep.py +97 -0
- flowcellar/readdata/link.py +486 -0
- flowcellar/readdata/passes.py +149 -0
- flowcellar/readdata/patterns.py +230 -0
- flowcellar/readdata/pychopper.py +431 -0
- flowcellar/readdata/readtable.py +416 -0
- flowcellar/readdata/record.py +159 -0
- flowcellar/readdata/reference.py +139 -0
- flowcellar/readdata/rnacomp.py +329 -0
- flowcellar/readdata/sample_sheet.py +153 -0
- flowcellar/readdata/saturation.py +110 -0
- flowcellar/readdata/sections.py +892 -0
- flowcellar/readdata/sections_align.py +644 -0
- flowcellar/readdata/tables.py +262 -0
- flowcellar/report/__init__.py +1 -0
- flowcellar/report/api.py +54 -0
- flowcellar/report/html.py +342 -0
- flowcellar/report/hub.py +159 -0
- flowcellar/report/index_one.py +144 -0
- flowcellar/report/served.py +67 -0
- flowcellar/sc/__init__.py +57 -0
- flowcellar/sc/axes.py +211 -0
- flowcellar/sc/catalogue.json +1 -0
- flowcellar/sc/catalogue.py +51 -0
- flowcellar/sc/check.py +57 -0
- flowcellar/sc/collect.py +299 -0
- flowcellar/sc/collect_sr.py +450 -0
- flowcellar/sc/collect_wf.py +479 -0
- flowcellar/sc/detect.py +303 -0
- flowcellar/sc/judge.py +269 -0
- flowcellar/sc/link.py +326 -0
- flowcellar/sc/listing.py +144 -0
- flowcellar/sc/maps.py +181 -0
- flowcellar/sc/readpass.py +416 -0
- flowcellar/sc/report.py +230 -0
- flowcellar/sc/scan.py +176 -0
- flowcellar/sc/spaceranger.py +483 -0
- flowcellar/sc/store.py +537 -0
- flowcellar/sc/wfsc.py +419 -0
- flowcellar/serve/__init__.py +4 -0
- flowcellar/serve/server.py +153 -0
- flowcellar/sources/__init__.py +0 -0
- flowcellar/sources/minknow.py +455 -0
- flowcellar/sources/porescan.py +218 -0
- flowcellar/sources/seqsummary.py +450 -0
- flowcellar/sources/workers.py +244 -0
- flowcellar/testing/__init__.py +1 -0
- flowcellar/testing/demo.py +1217 -0
- flowcellar/testing/demo_spatial.py +370 -0
- flowcellar/testing/fixtures.py +459 -0
- flowcellar/testing/libprep.py +252 -0
- flowcellar/testing/minknow_variants.py +384 -0
- flowcellar/testing/readdata.py +744 -0
- flowcellar/testing/sc.py +817 -0
- flowcellar/testing/trees.py +175 -0
- flowcellar/util/__init__.py +1 -0
- flowcellar/util/atomic.py +112 -0
- flowcellar/util/errors.py +72 -0
- flowcellar/util/fingerprint.py +49 -0
- flowcellar/util/log.py +50 -0
- flowcellar/util/text.py +43 -0
- flowcellar/util/timeutil.py +52 -0
- flowcellar/viewer_dist/favicon.svg +1 -0
- flowcellar/viewer_dist/manifest.json +37 -0
- flowcellar/viewer_dist/template.html +12 -0
- flowcellar/viewer_dist/viewer.core.js +3 -0
- flowcellar/viewer_dist/viewer.css +2151 -0
- flowcellar-0.1.0.dist-info/METADATA +506 -0
- flowcellar-0.1.0.dist-info/RECORD +159 -0
- flowcellar-0.1.0.dist-info/WHEEL +4 -0
- flowcellar-0.1.0.dist-info/entry_points.txt +2 -0
- flowcellar-0.1.0.dist-info/licenses/LICENSE +674 -0
- flowcellar-0.1.0.dist-info/licenses/THIRD_PARTY_NOTICES.md +85 -0
flowcellar/report/hub.py
ADDED
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
"""The static hub of a library (mode B).
|
|
2
|
+
|
|
3
|
+
hub/index.html the viewer and no data; rewritten only when the viewer changed
|
|
4
|
+
hub/index.js the index, as one call of __FC_INDEX__; rewritten every time
|
|
5
|
+
hub/data/<run_key>.js one run, as one call of __FCB_SIDECAR__; rewritten only when its bundle or its
|
|
6
|
+
status file is newer than it, or when the caller names the run
|
|
7
|
+
|
|
8
|
+
Every write is a whole-file replace, so a browser that reads while a scan writes never sees half a file.
|
|
9
|
+
"""
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import os
|
|
13
|
+
|
|
14
|
+
from flowcellar.model.bundle import bundle_filename, report_filename
|
|
15
|
+
from flowcellar.model.status import status_filename
|
|
16
|
+
from flowcellar.report import html as page
|
|
17
|
+
from flowcellar.report import index_one
|
|
18
|
+
from flowcellar.util import atomic
|
|
19
|
+
|
|
20
|
+
SIDECAR_SUFFIX = ".js"
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def sidecar_path(hub_dir: str, run_key: str) -> str:
|
|
24
|
+
return os.path.join(hub_dir, "data", run_key + SIDECAR_SUFFIX)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _mtime_ns(path: str) -> int | None:
|
|
28
|
+
try:
|
|
29
|
+
return os.stat(path).st_mtime_ns
|
|
30
|
+
except OSError:
|
|
31
|
+
return None
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def source_mtime_ns(library, run_key: str) -> int | None:
|
|
35
|
+
"""When the run's bundle or its status file last changed; None when neither can be found (then nothing can
|
|
36
|
+
be trusted to be up to date)."""
|
|
37
|
+
times = [_mtime_ns(os.path.join(library.path, "bundles", bundle_filename(run_key))),
|
|
38
|
+
_mtime_ns(os.path.join(library.path, "status", status_filename(run_key)))]
|
|
39
|
+
times = [t for t in times if t is not None]
|
|
40
|
+
return max(times) if times else None
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def is_fresh(library, hub_dir: str, run_key: str) -> bool:
|
|
44
|
+
"""True when the sidecar exists and is newer than the run's bundle and status file."""
|
|
45
|
+
mine, source = _mtime_ns(sidecar_path(hub_dir, run_key)), source_mtime_ns(library, run_key)
|
|
46
|
+
return mine is not None and source is not None and mine > source
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _read(path: str) -> bytes | None:
|
|
50
|
+
try:
|
|
51
|
+
with open(path, "rb") as fh:
|
|
52
|
+
return fh.read()
|
|
53
|
+
except OSError:
|
|
54
|
+
return None
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _head(path: str, n: int = 2048) -> bytes:
|
|
58
|
+
try:
|
|
59
|
+
with open(path, "rb") as fh:
|
|
60
|
+
return fh.read(n)
|
|
61
|
+
except OSError:
|
|
62
|
+
return b""
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _write_if_changed(path: str, data: bytes) -> bool:
|
|
66
|
+
if _read(path) == data:
|
|
67
|
+
return False
|
|
68
|
+
atomic.atomic_write_bytes(path, data)
|
|
69
|
+
return True
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _has_analyses(library_path: str, data_dir: str) -> bool:
|
|
73
|
+
for folder in (os.path.join(library_path, "bundles"), data_dir):
|
|
74
|
+
try:
|
|
75
|
+
if any(n.startswith("an_") for n in os.listdir(folder)):
|
|
76
|
+
return True
|
|
77
|
+
except OSError:
|
|
78
|
+
pass
|
|
79
|
+
return False
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def write_hub(library, *, sidecars: bool = True, reports: bool = False, only=None) -> dict:
|
|
83
|
+
hub_dir = library.hub_dir()
|
|
84
|
+
data_dir = os.path.join(hub_dir, "data")
|
|
85
|
+
os.makedirs(hub_dir, exist_ok=True)
|
|
86
|
+
keys = [k for k in library.run_keys() if page.RUN_KEY_RE.match(k)]
|
|
87
|
+
known = set(keys)
|
|
88
|
+
written: list[str] = []
|
|
89
|
+
removed: list[str] = []
|
|
90
|
+
|
|
91
|
+
# the page: only when the viewer (or this package's text around it) changed
|
|
92
|
+
index_html = os.path.join(hub_dir, "index.html")
|
|
93
|
+
if _write_if_changed(index_html, page.hub_page().encode("utf-8")):
|
|
94
|
+
written.append(index_html)
|
|
95
|
+
|
|
96
|
+
# the index: always
|
|
97
|
+
index_js = os.path.join(hub_dir, "index.js")
|
|
98
|
+
atomic.atomic_write_bytes(index_js, page.index_sidecar(library.index_gzip()))
|
|
99
|
+
written.append(index_js)
|
|
100
|
+
|
|
101
|
+
# data files of runs that are no longer in the library
|
|
102
|
+
if os.path.isdir(data_dir):
|
|
103
|
+
for name in sorted(os.listdir(data_dir)):
|
|
104
|
+
if not name.endswith(SIDECAR_SUFFIX) or name.startswith("."):
|
|
105
|
+
continue
|
|
106
|
+
key = name[: -len(SIDECAR_SUFFIX)]
|
|
107
|
+
if key.startswith("an_"):
|
|
108
|
+
continue # the data file of an analysis: looked after below
|
|
109
|
+
stale = key not in known or (not sidecars and not is_fresh(library, hub_dir, key))
|
|
110
|
+
if stale:
|
|
111
|
+
os.unlink(os.path.join(data_dir, name))
|
|
112
|
+
removed.append(os.path.join(data_dir, name))
|
|
113
|
+
|
|
114
|
+
todo: list[str] = []
|
|
115
|
+
if sidecars:
|
|
116
|
+
wanted = set(only) & known if only is not None else None
|
|
117
|
+
for key in keys:
|
|
118
|
+
if (wanted is not None and key in wanted) or not is_fresh(library, hub_dir, key):
|
|
119
|
+
todo.append(key)
|
|
120
|
+
for key in todo:
|
|
121
|
+
path = sidecar_path(hub_dir, key)
|
|
122
|
+
data = page.run_sidecar(key, library.composed_gzip(key))
|
|
123
|
+
if _read(path) == data and (wanted is None or key not in wanted):
|
|
124
|
+
os.utime(path) # same content: only say that it is up to date
|
|
125
|
+
continue
|
|
126
|
+
atomic.atomic_write_bytes(path, data)
|
|
127
|
+
written.append(path)
|
|
128
|
+
|
|
129
|
+
# single-cell and spatial analyses keep their data files beside those of the runs; the add-on writes them and
|
|
130
|
+
# is loaded only when the library holds an analysis or the folder still has such a file
|
|
131
|
+
if _has_analyses(library.path, data_dir):
|
|
132
|
+
from flowcellar.sc import report as sc_report
|
|
133
|
+
out = sc_report.write_sidecars(library, hub_dir, sidecars=sidecars, only=only)
|
|
134
|
+
written.extend(out["written"])
|
|
135
|
+
removed.extend(out["removed"])
|
|
136
|
+
|
|
137
|
+
if reports:
|
|
138
|
+
index = library.index()
|
|
139
|
+
rep_dir = os.path.join(library.path, "reports")
|
|
140
|
+
named = set(only or ())
|
|
141
|
+
script = page.load_viewer().js_csp.encode("ascii").replace(b"'", b"'") # as the policy element spells it
|
|
142
|
+
for key in keys:
|
|
143
|
+
path = os.path.join(rep_dir, report_filename(key))
|
|
144
|
+
source, mine = source_mtime_ns(library, key), _mtime_ns(path)
|
|
145
|
+
# a report is kept when it is newer than the run's data, was made with this viewer and was not asked for
|
|
146
|
+
if mine is not None and source is not None and mine > source and key not in named and key not in todo and script in _head(path):
|
|
147
|
+
continue
|
|
148
|
+
bundle = library.composed(key)
|
|
149
|
+
one = index_one.slice_index(index, key) or index_one.plain_index(bundle)
|
|
150
|
+
text, _ = page.report_page(bundle, one)
|
|
151
|
+
atomic.atomic_write_bytes(path, text.encode("utf-8"))
|
|
152
|
+
written.append(path)
|
|
153
|
+
if os.path.isdir(rep_dir):
|
|
154
|
+
for name in sorted(os.listdir(rep_dir)):
|
|
155
|
+
if name.endswith(report_filename("")) and name[: -len(report_filename(""))] not in known:
|
|
156
|
+
os.unlink(os.path.join(rep_dir, name))
|
|
157
|
+
removed.append(os.path.join(rep_dir, name))
|
|
158
|
+
|
|
159
|
+
return {"index_html": index_html, "written": written, "removed": removed}
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
"""The one-run index a report embeds (mode A).
|
|
2
|
+
|
|
3
|
+
Two sources:
|
|
4
|
+
|
|
5
|
+
* ``slice_index``: one row cut out of a library's index (``flowcellar hub --reports``);
|
|
6
|
+
* ``index_for_bundle``: built from a composed bundle alone. It first asks LIBRARY
|
|
7
|
+
(``library.api.build_index([card_for(bundle)])``), which owns the index; until that is built, or when it
|
|
8
|
+
fails, ``plain_index`` writes the columns that can be read straight from the bundle. The viewer draws a run
|
|
9
|
+
from its bundle and needs the row only for the page title and for the line it shows while the bundle decodes.
|
|
10
|
+
"""
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
from flowcellar import SCHEMA_VERSION, __version__
|
|
14
|
+
from flowcellar.model import status as status_model
|
|
15
|
+
from flowcellar.util.errors import FlowCellarError
|
|
16
|
+
from flowcellar.util.timeutil import now_z
|
|
17
|
+
|
|
18
|
+
FORMAT = "flowcellar.index"
|
|
19
|
+
_VERDICT_KEYS = ("pass", "warn", "fail", "incomplete", "running", "unknown")
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _summaries(bundle: dict) -> dict:
|
|
23
|
+
out: dict = {}
|
|
24
|
+
for m in (bundle.get("modules") or {}).values():
|
|
25
|
+
out.update(m.get("summary") or {})
|
|
26
|
+
return out
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def plain_columns(bundle: dict) -> dict:
|
|
30
|
+
"""The index row of one run, as far as the bundle alone tells it."""
|
|
31
|
+
ident, st, tiers = bundle.get("identity") or {}, bundle.get("status") or {}, bundle.get("tiers") or {}
|
|
32
|
+
fc, pr, dev = ident.get("flow_cell") or {}, ident.get("protocol_run") or {}, ident.get("device") or {}
|
|
33
|
+
kit, assay, rel = ident.get("kit") or {}, ident.get("assay") or {}, bundle.get("relations") or {}
|
|
34
|
+
use, by, s = rel.get("flow_cell_use") or {}, st.get("by_tier") or {}, _summaries(bundle)
|
|
35
|
+
rh, rl = tiers.get("run_health") or {}, tiers.get("read_level") or {}
|
|
36
|
+
rs = status_model.reasons(st)
|
|
37
|
+
projects = rel.get("projects") or []
|
|
38
|
+
gb = s.get("yield_estimated_bases")
|
|
39
|
+
return {
|
|
40
|
+
"id": bundle.get("bundle_id") or ident.get("run_key"), "fc": fc.get("id"),
|
|
41
|
+
"use": use.get("index") or 1, "use_of": use.get("of") or 1, "use_source": use.get("source") or "unknown",
|
|
42
|
+
"restart": bool(use.get("restart_of")), "restart_of": use.get("restart_of"),
|
|
43
|
+
"started": pr.get("started"), "hours": s.get("run_duration_h"),
|
|
44
|
+
"planned_h": (pr.get("planned_duration_s") or 0) / 3600 or None,
|
|
45
|
+
"run_state": pr.get("run_state") or "finished",
|
|
46
|
+
"device_type": dev.get("type"), "position": dev.get("position"), "product": fc.get("product_code"),
|
|
47
|
+
"kit": kit.get("sequencing_kit"), "assay": assay.get("type"), "assay_group": assay.get("group") or "unknown",
|
|
48
|
+
"chemistry": assay.get("chemistry"), "minknow": (ident.get("software") or {}).get("minknow"),
|
|
49
|
+
"experiment": pr.get("experiment_id"), "run_name": pr.get("sample_id"), "channels": fc.get("channel_count"),
|
|
50
|
+
"group": " | ".join(str(x) for x in (dev.get("type"), fc.get("product_code"), kit.get("sequencing_kit"))),
|
|
51
|
+
"folder": None, "roots": sorted({r for p in projects for r in (p.get("roots") or [])}),
|
|
52
|
+
"project": projects[0]["name"] if projects else None, "project_ids": [p["id"] for p in projects],
|
|
53
|
+
"source_state": "ok",
|
|
54
|
+
"verdict": st.get("verdict") or "unknown", "health": by.get("run_health") or "not_available",
|
|
55
|
+
"data": by.get("read_level") or "not_available", "provisional": bool(st.get("provisional")),
|
|
56
|
+
"pending": len(st.get("pending") or []), "checks_done": len(st.get("rules") or []), "cause": st.get("cause"),
|
|
57
|
+
"modifiers": [m.get("code") for m in st.get("modifiers") or []],
|
|
58
|
+
"headline": st.get("headline") or "", "detail": st.get("detail") or "",
|
|
59
|
+
"reasons": [r["message"] for r in rs], "reason_levels": [r["severity"] for r in rs],
|
|
60
|
+
"reason_checks": [r["check"] for r in rs], "reason_tiers": [r["tier"] for r in rs],
|
|
61
|
+
"reads": s.get("yield_reads"), "gb": gb / 1e9 if gb is not None else None,
|
|
62
|
+
"pores": s.get("fc_pores_first_scan"), "pores_last": s.get("fc_pores_last_scan"),
|
|
63
|
+
"occ_first": s.get("fc_occupancy_first_hour"),
|
|
64
|
+
"detail_t1": rh.get("detail") if rh.get("state") not in (None, "absent") else None,
|
|
65
|
+
"read_data": rl.get("state") or "absent", "t2_population": (rl.get("population") or {}).get("stage"),
|
|
66
|
+
"t2_samples": rl.get("samples_expected"), "t2_samples_with_data": rl.get("samples_with_data"),
|
|
67
|
+
"t2_reads": rl.get("reads_covered"),
|
|
68
|
+
"t2_passed_gb": s["yield_basecalled_bases"] / 1e9 if s.get("yield_basecalled_bases") is not None else None,
|
|
69
|
+
"t2_n50": s.get("len_n50"), "t2_q_med": s.get("qual_median"), "t2_usable": s.get("funnel_usable_frac"),
|
|
70
|
+
"t2_files_waiting": 0, "spark_output": "", "spark_output_max_gb_per_3h": 0,
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def plain_index(bundle: dict) -> dict:
|
|
75
|
+
"""A one-run index in the shape of the library index, from the bundle alone."""
|
|
76
|
+
cols = plain_columns(bundle)
|
|
77
|
+
prov_lib = (bundle.get("provenance") or {}).get("library") or {}
|
|
78
|
+
samples = [{"sample_uid": s.get("sample_uid"), "name": s.get("name") or s.get("alias") or s.get("key"), "alias": s.get("alias"),
|
|
79
|
+
"barcode": s.get("barcode"), "project": s.get("project"), "run": cols["id"], "key": s.get("key"),
|
|
80
|
+
"reads": None, "bases": None, "n50": None, "q_med": None}
|
|
81
|
+
for s in bundle.get("samples") or [] if s.get("key") != "unclassified"]
|
|
82
|
+
verdicts = {k: 0 for k in _VERDICT_KEYS}
|
|
83
|
+
verdicts[cols["verdict"] if cols["verdict"] in verdicts else "unknown"] = 1
|
|
84
|
+
started = cols["started"]
|
|
85
|
+
return {
|
|
86
|
+
"format": FORMAT, "schema": SCHEMA_VERSION, "written": now_z(),
|
|
87
|
+
"meta": {"library": prov_lib.get("name"), "runs": 1, "generator": {"name": "flowcellar", "version": __version__}},
|
|
88
|
+
"library": {"id": prov_lib.get("id"), "name": prov_lib.get("name"), "path": None, "created": None, "scanned": None,
|
|
89
|
+
"counts": {"runs": 1, "flow_cells": 1, "projects": len(cols["project_ids"]), "samples": len(samples),
|
|
90
|
+
"by_verdict": verdicts, "by_detail": {}, "with_read_data": int(cols["read_data"] != "absent")},
|
|
91
|
+
"estimated_bases": None, "reads": cols["reads"], "first_run": started, "last_run": started},
|
|
92
|
+
"runs": {"n": 1, "columns": {k: [v] for k, v in cols.items()}},
|
|
93
|
+
"samples": samples, "projects": [], "roots": [], "activity": {"months": [], "runs": [], "estimated_bases": [], "reads": []},
|
|
94
|
+
"baselines": {"groups": []}, "facets": {}, "notices": [],
|
|
95
|
+
"progress": {"state": "idle"},
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def index_for_bundle(bundle: dict) -> dict:
|
|
100
|
+
"""The index of a report made from one composed bundle: LIBRARY's when it can make one, else the plain one."""
|
|
101
|
+
try:
|
|
102
|
+
from flowcellar.library import api as library_api
|
|
103
|
+
|
|
104
|
+
status_file = None
|
|
105
|
+
if bundle.get("status") is not None:
|
|
106
|
+
status_file = status_model.new_status_file(bundle.get("bundle_id"), bundle.get("content_sha256"),
|
|
107
|
+
bundle["status"], bundle.get("relations") or {})
|
|
108
|
+
index = library_api.build_index([library_api.card_for(bundle, status_file)])
|
|
109
|
+
if isinstance(index, dict) and (index.get("runs") or {}).get("n") == 1:
|
|
110
|
+
return index
|
|
111
|
+
except FlowCellarError:
|
|
112
|
+
pass # not built yet, or it cannot make a card of this bundle
|
|
113
|
+
return plain_index(bundle)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def _value(col, i):
|
|
117
|
+
if isinstance(col, dict) and "dict" in col:
|
|
118
|
+
return col["dict"][col["codes"][i]]
|
|
119
|
+
return col[i]
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def slice_index(index: dict, run_key: str) -> dict | None:
|
|
123
|
+
"""One row of a library index as an index of its own: what a report of that run embeds. Names of things that
|
|
124
|
+
are not in the file (other runs, other projects, the folders that were scanned) are left out."""
|
|
125
|
+
cols = (index.get("runs") or {}).get("columns") or {}
|
|
126
|
+
n = (index.get("runs") or {}).get("n") or 0
|
|
127
|
+
ids = cols.get("id")
|
|
128
|
+
i = next((k for k in range(n) if _value(ids, k) == run_key), None) if ids is not None else None
|
|
129
|
+
if i is None:
|
|
130
|
+
return None
|
|
131
|
+
row = {k: _value(c, i) for k, c in cols.items()}
|
|
132
|
+
out = {k: v for k, v in index.items() if k not in ("runs", "samples", "projects", "roots", "activity", "baselines", "notices", "progress", "facets", "library")}
|
|
133
|
+
lib = index.get("library") or {}
|
|
134
|
+
out["library"] = {"id": lib.get("id"), "name": lib.get("name"), "path": None, "created": lib.get("created"), "scanned": lib.get("scanned")}
|
|
135
|
+
out["runs"] = {"n": 1, "columns": {k: [v] for k, v in row.items()}}
|
|
136
|
+
out["samples"] = [s for s in index.get("samples") or [] if s.get("run") == run_key]
|
|
137
|
+
out["projects"] = [p for p in index.get("projects") or [] if p.get("id") in (row.get("project_ids") or [])]
|
|
138
|
+
out["roots"] = []
|
|
139
|
+
out["activity"] = {"months": [], "runs": [], "estimated_bases": [], "reads": []}
|
|
140
|
+
out["baselines"] = {"groups": [g for g in (index.get("baselines") or {}).get("groups") or [] if g.get("group") == row.get("group")]}
|
|
141
|
+
out["facets"] = {}
|
|
142
|
+
out["notices"] = []
|
|
143
|
+
out["progress"] = {"state": "idle"}
|
|
144
|
+
return out
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
"""What ``flowcellar serve`` answers (mode C).
|
|
2
|
+
|
|
3
|
+
Four addresses and nothing else. No path that came from a request is ever joined to a folder: the page is the
|
|
4
|
+
one fixed file ``hub/index.html``, the index is made from the library, and a run's data file is found from a run
|
|
5
|
+
key that matched ``RUN_KEY_RE`` and is one of the library's own.
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import html as html_lib
|
|
10
|
+
import os
|
|
11
|
+
import re
|
|
12
|
+
|
|
13
|
+
from flowcellar.contracts import Served
|
|
14
|
+
from flowcellar.report import html as page
|
|
15
|
+
from flowcellar.report import hub as hub_files
|
|
16
|
+
from flowcellar.util import atomic
|
|
17
|
+
from flowcellar.util.errors import FlowCellarError
|
|
18
|
+
|
|
19
|
+
HTML = "text/html; charset=utf-8"
|
|
20
|
+
JS = "text/javascript; charset=utf-8"
|
|
21
|
+
_META_CSP = re.compile(rb'<meta http-equiv="Content-Security-Policy" content="([^"]*)">')
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def headers(policy: str | None = None) -> dict[str, str]:
|
|
25
|
+
"""On every answer. ``policy`` is the one of the page being served; by default that of the installed viewer."""
|
|
26
|
+
return {"Content-Security-Policy": policy or page.hub_policy(), "X-Content-Type-Options": "nosniff",
|
|
27
|
+
"Referrer-Policy": "no-referrer", "Cross-Origin-Opener-Policy": "same-origin", "Cache-Control": "no-store"}
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def page_policy(body: bytes) -> str | None:
|
|
31
|
+
"""The policy a hub page states in its own meta element. The header must say the same: a browser applies
|
|
32
|
+
both, and a page written by an older viewer carries the hash of that viewer's script."""
|
|
33
|
+
m = _META_CSP.search(body[:4096])
|
|
34
|
+
return html_lib.unescape(m.group(1).decode("utf-8")) if m else None
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def serve_file(library, url_path: str) -> Served | None:
|
|
38
|
+
path = (url_path or "").split("#", 1)[0].split("?", 1)[0] # the viewer asks for index.js?t=<counter>
|
|
39
|
+
hub_dir = library.hub_dir()
|
|
40
|
+
if path in ("/", "/index.html"):
|
|
41
|
+
try:
|
|
42
|
+
body = atomic.read_bytes(os.path.join(hub_dir, "index.html"), retries=1)
|
|
43
|
+
except OSError:
|
|
44
|
+
return None # no hub was written: serve says how to write one
|
|
45
|
+
return Served(body, HTML, headers(page_policy(body)))
|
|
46
|
+
if path == "/index.js":
|
|
47
|
+
try:
|
|
48
|
+
return Served(page.index_sidecar(library.index_gzip()), JS, headers())
|
|
49
|
+
except (FlowCellarError, OSError):
|
|
50
|
+
return None
|
|
51
|
+
if path.startswith("/data/") and path.endswith(".js"):
|
|
52
|
+
run_key = path[len("/data/"): -len(".js")]
|
|
53
|
+
if run_key.startswith("an_"): # an analysis of the single-cell and spatial add-on
|
|
54
|
+
from flowcellar.sc import report as sc_report
|
|
55
|
+
return sc_report.serve_sidecar(library, hub_dir, run_key, headers())
|
|
56
|
+
if not page.RUN_KEY_RE.match(run_key) or run_key not in set(library.run_keys()):
|
|
57
|
+
return None
|
|
58
|
+
if hub_files.is_fresh(library, hub_dir, run_key):
|
|
59
|
+
try:
|
|
60
|
+
return Served(atomic.read_bytes(hub_files.sidecar_path(hub_dir, run_key), retries=1), JS, headers())
|
|
61
|
+
except OSError:
|
|
62
|
+
pass
|
|
63
|
+
try:
|
|
64
|
+
return Served(page.run_sidecar(run_key, library.composed_gzip(run_key)), JS, headers())
|
|
65
|
+
except (FlowCellarError, OSError, KeyError):
|
|
66
|
+
return None
|
|
67
|
+
return None
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"""The single-cell and spatial add-on: quality control of wf-single-cell and Space Ranger outputs.
|
|
2
|
+
|
|
3
|
+
An *analysis* is one output folder of one pipeline for one sample: a wf-single-cell sample folder, or a Space
|
|
4
|
+
Ranger ``outs/``. It gets a bundle of its own (``kind: "analysis"``), because one analysis pools the reads of
|
|
5
|
+
several flow cells; the bundle names the runs it came from with the evidence for each link. An analysis has its
|
|
6
|
+
own verdict and never changes the verdict of a flow cell run.
|
|
7
|
+
|
|
8
|
+
Nothing of this package is imported until an analysis is looked for or shown: ``import flowcellar``, the run
|
|
9
|
+
commands and a run report do not load it. Importing this package itself loads nothing heavy either: ``h5py``
|
|
10
|
+
(the ``sc`` extra) is imported inside the functions that open an HDF5 file, ``pysam`` (the ``bam`` extra) inside
|
|
11
|
+
the function that tallies the read tags of a BAM. Without ``h5py`` the files that are tables or JSON are still
|
|
12
|
+
read and the bundle says what is missing.
|
|
13
|
+
|
|
14
|
+
flowcellar.sc.detect which folders are analyses (by their files, never by their names)
|
|
15
|
+
flowcellar.sc.collect one analysis folder -> the measured bundle
|
|
16
|
+
flowcellar.sc.link the evidence ladder from an analysis to the flow cell runs it came from
|
|
17
|
+
flowcellar.sc.judge the limits and the verdict of an analysis
|
|
18
|
+
flowcellar.sc.store analyses inside a library, the index table, what a scan does
|
|
19
|
+
flowcellar.sc.report the one-file report and the hub's data file of an analysis
|
|
20
|
+
"""
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
import re
|
|
24
|
+
|
|
25
|
+
MODULE_ID = "sc"
|
|
26
|
+
#: version of the section this package writes; a change makes every cached collection stale
|
|
27
|
+
MODULE_VERSION = "0.1.0"
|
|
28
|
+
SECTION_SCHEMA_VERSION = 1
|
|
29
|
+
|
|
30
|
+
PIPELINE_SPACERANGER = "spaceranger"
|
|
31
|
+
PIPELINE_WF = "wf-single-cell"
|
|
32
|
+
#: how a pipeline is spelled inside a bundle id (letters only: the id is a file name and an address)
|
|
33
|
+
PIPELINE_CODE = {PIPELINE_SPACERANGER: "spaceranger", PIPELINE_WF: "wfsinglecell"}
|
|
34
|
+
PIPELINE_WORD = {PIPELINE_SPACERANGER: "Space Ranger", PIPELINE_WF: "wf-single-cell"}
|
|
35
|
+
|
|
36
|
+
#: the id of an analysis bundle as it may appear in a file name, an element id and an address
|
|
37
|
+
ANALYSIS_ID_RE = re.compile(r"^an_[a-z]{2,16}_[0-9a-f]{12}$")
|
|
38
|
+
|
|
39
|
+
#: the roles under which the discovery walk reports the anchor file of an analysis
|
|
40
|
+
ROLE_SPACERANGER = "analysis.spaceranger"
|
|
41
|
+
ROLE_WF = "analysis.wf_single_cell"
|
|
42
|
+
ROLE_PREFIX = "analysis."
|
|
43
|
+
|
|
44
|
+
ASSAY_WORD = {"visium": "Visium", "visium_hd": "Visium HD", "singlecell": "Single cell", None: "Assay not known"}
|
|
45
|
+
|
|
46
|
+
#: size budget of one analysis section as JSON text (bytes): the default aim and the cap a bundle may not pass
|
|
47
|
+
SECTION_BUDGET_BYTES = 150_000
|
|
48
|
+
SECTION_CAP_BYTES = 400_000
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def bundle_id(pipeline: str, analysis_hash: str) -> str:
|
|
52
|
+
"""``an_<pipeline>_<first 12 hex of the content hash>``."""
|
|
53
|
+
return f"an_{PIPELINE_CODE[pipeline]}_{analysis_hash[:12]}"
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def is_analysis_id(text) -> bool:
|
|
57
|
+
return isinstance(text, str) and bool(ANALYSIS_ID_RE.match(text))
|
flowcellar/sc/axes.py
ADDED
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
"""The three fixed axes of the add-on, and its aggregates in their stored form.
|
|
2
|
+
|
|
3
|
+
The edges are the same for every analysis, so histograms of two analyses can be overlaid and added. The
|
|
4
|
+
definitions are copied into ``bundle.axes`` of every analysis bundle (a bundle describes itself); they are kept
|
|
5
|
+
here and not in the core's axis registry so that the core does not grow.
|
|
6
|
+
|
|
7
|
+
log_counts_8_per_decade 8 bins per decade from 1 to 1,000,000; a count of 0 is the underflow cell
|
|
8
|
+
fraction_50 50 bins of 0.02 from 0 to 1 (1.0 falls in the last bin)
|
|
9
|
+
reads_per_molecule 1 to 64 one by one, then one bin per power of two up to 2^20
|
|
10
|
+
"""
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import base64
|
|
14
|
+
|
|
15
|
+
import numpy as np
|
|
16
|
+
|
|
17
|
+
LOG_COUNTS = "log_counts_8_per_decade"
|
|
18
|
+
FRACTION_50 = "fraction_50"
|
|
19
|
+
READS_PER_MOLECULE = "reads_per_molecule"
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _log_edges() -> np.ndarray:
|
|
23
|
+
i = np.arange(0, 6 * 8 + 1)
|
|
24
|
+
e = np.power(10.0, i / 8.0)
|
|
25
|
+
whole = (i % 8) == 0
|
|
26
|
+
e[whole] = np.power(10.0, (i[whole] // 8).astype(np.float64)) # whole decades are exact
|
|
27
|
+
return e
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
EDGES: dict[str, np.ndarray] = {
|
|
31
|
+
LOG_COUNTS: _log_edges(),
|
|
32
|
+
FRACTION_50: np.arange(0, 51) / 50.0,
|
|
33
|
+
READS_PER_MOLECULE: np.concatenate([np.arange(1, 66), 2 ** np.arange(7, 21)]).astype(np.float64),
|
|
34
|
+
}
|
|
35
|
+
for _e in EDGES.values():
|
|
36
|
+
_e.setflags(write=False)
|
|
37
|
+
|
|
38
|
+
AXES: dict[str, dict] = {
|
|
39
|
+
LOG_COUNTS: {"type": "log10", "min": 0, "bins_per_decade": 8, "n": 48, "unit": "count", "family": "log_counts",
|
|
40
|
+
"note": "a unit with a count of 0 is in the underflow cell"},
|
|
41
|
+
FRACTION_50: {"type": "linear", "start": 0, "step": 0.02, "n": 50, "unit": "fraction", "family": "fraction",
|
|
42
|
+
"closed_last": True},
|
|
43
|
+
READS_PER_MOLECULE: {"type": "edges", "edges": [float(x) for x in EDGES[READS_PER_MOLECULE]], "unit": "reads",
|
|
44
|
+
"family": "reads_per_molecule",
|
|
45
|
+
"note": "1 to 64 reads one by one, then one bin per power of two; w holds the reads of each bin"},
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def axis_defs(names) -> dict:
|
|
50
|
+
return {n: dict(AXES[n]) for n in sorted(set(names))}
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def bin_counts(values, axis: str, weights=None) -> np.ndarray:
|
|
54
|
+
"""Internal array (underflow, the bins, overflow) of ``values`` on ``axis``; with ``weights`` their sums."""
|
|
55
|
+
e = EDGES[axis]
|
|
56
|
+
v = np.asarray(values)
|
|
57
|
+
idx = np.searchsorted(e, v, side="right")
|
|
58
|
+
if AXES[axis].get("closed_last"):
|
|
59
|
+
idx = np.where(v == e[-1], len(e) - 1, idx)
|
|
60
|
+
if weights is None:
|
|
61
|
+
return np.bincount(idx, minlength=len(e) + 1).astype(np.int64)
|
|
62
|
+
return np.rint(np.bincount(idx, weights=np.asarray(weights, dtype=np.float64), minlength=len(e) + 1)).astype(np.int64)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def hist_from_full(axis: str, full, w_full=None, *, w_name: str | None = None, missing: int | None = None,
|
|
66
|
+
vmin=None, vmax=None) -> dict:
|
|
67
|
+
"""Internal array -> stored histogram with empty ends trimmed (the layout of the bundle schema)."""
|
|
68
|
+
full = np.asarray(full)
|
|
69
|
+
inner = full[1:-1]
|
|
70
|
+
nz = np.nonzero(inner)[0]
|
|
71
|
+
a, b = (int(nz[0]), int(nz[-1]) + 1) if len(nz) else (0, 0)
|
|
72
|
+
h: dict = {"axis": axis, "start": a, "n": [int(x) for x in inner[a:b]]}
|
|
73
|
+
if w_full is not None:
|
|
74
|
+
w_full = np.asarray(w_full)
|
|
75
|
+
h["w"] = [int(x) for x in w_full[1:-1][a:b]]
|
|
76
|
+
if w_name:
|
|
77
|
+
h["w_name"] = w_name
|
|
78
|
+
h["w_under"], h["w_over"] = int(w_full[0]), int(w_full[-1])
|
|
79
|
+
h["under"], h["over"] = int(full[0]), int(full[-1])
|
|
80
|
+
if missing is not None:
|
|
81
|
+
h["missing"] = int(missing)
|
|
82
|
+
if vmin is not None:
|
|
83
|
+
h["min"] = vmin
|
|
84
|
+
if vmax is not None:
|
|
85
|
+
h["max"] = vmax
|
|
86
|
+
return h
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def hist(axis: str, values, *, missing: int | None = None) -> dict:
|
|
90
|
+
"""Stored histogram of ``values`` (one per unit)."""
|
|
91
|
+
v = np.asarray(values)
|
|
92
|
+
vmin = float(v.min()) if len(v) else None
|
|
93
|
+
vmax = float(v.max()) if len(v) else None
|
|
94
|
+
return hist_from_full(axis, bin_counts(v, axis), missing=missing, vmin=_num(vmin), vmax=_num(vmax))
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _num(x):
|
|
98
|
+
if x is None:
|
|
99
|
+
return None
|
|
100
|
+
return int(x) if float(x).is_integer() else float(x)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def reads_per_molecule_hist(counts_by_c) -> dict:
|
|
104
|
+
"""``counts_by_c[c]`` = molecules seen ``c`` times -> stored histogram with ``w`` = the reads of each bin.
|
|
105
|
+
|
|
106
|
+
With ``w`` the mean number of reads of the molecules in a wide bin is known, and the total number of reads is
|
|
107
|
+
exact, so the saturation curve drawn from the histogram ends exactly at the saturation of the data.
|
|
108
|
+
"""
|
|
109
|
+
h = np.asarray(counts_by_c, dtype=np.int64)
|
|
110
|
+
c = np.arange(len(h), dtype=np.int64)
|
|
111
|
+
present = np.nonzero(h)[0]
|
|
112
|
+
if len(present) == 0:
|
|
113
|
+
return hist_from_full(READS_PER_MOLECULE, np.zeros(len(EDGES[READS_PER_MOLECULE]) + 1, np.int64),
|
|
114
|
+
np.zeros(len(EDGES[READS_PER_MOLECULE]) + 1, np.int64), w_name="reads")
|
|
115
|
+
idx = np.searchsorted(EDGES[READS_PER_MOLECULE], c[present], side="right")
|
|
116
|
+
n_cells = len(EDGES[READS_PER_MOLECULE]) + 1
|
|
117
|
+
full = np.bincount(idx, weights=h[present], minlength=n_cells).astype(np.int64)
|
|
118
|
+
w = np.bincount(idx, weights=(h[present] * c[present]).astype(np.float64), minlength=n_cells).astype(np.int64)
|
|
119
|
+
return hist_from_full(READS_PER_MOLECULE, full, w, w_name="reads", vmin=int(c[present][0]), vmax=int(c[present][-1]))
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def saturation_from_hist(h: dict, fraction: float = 1.0) -> float | None:
|
|
123
|
+
"""Saturation at a share ``fraction`` of the reads: ``1 - sum h[c] (1 - (1 - p)^c) / (p sum c h[c])``, the
|
|
124
|
+
exact expectation under random subsampling. A wide bin stands for its mean number of reads per molecule."""
|
|
125
|
+
n = np.asarray(h.get("n") or [], dtype=np.float64)
|
|
126
|
+
w = np.asarray(h.get("w") or [], dtype=np.float64)
|
|
127
|
+
if len(n) == 0 or w.sum() <= 0:
|
|
128
|
+
return None
|
|
129
|
+
ok = n > 0
|
|
130
|
+
c = w[ok] / n[ok]
|
|
131
|
+
p = float(fraction)
|
|
132
|
+
unique = float((n[ok] * (1.0 - np.power(1.0 - p, c))).sum()) if p < 1 else float(n[ok].sum())
|
|
133
|
+
return 1.0 - unique / (p * float(w.sum()))
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def rank_curve(counts, *, points: int = 200) -> tuple[list[int], list[float]]:
|
|
137
|
+
"""Counts sorted from the largest down, kept at up to ``points`` log-spaced ranks (rank 1 and the last rank
|
|
138
|
+
always kept): the barcode rank plot."""
|
|
139
|
+
v = np.sort(np.asarray(counts))[::-1]
|
|
140
|
+
v = v[v > 0]
|
|
141
|
+
n = len(v)
|
|
142
|
+
if n == 0:
|
|
143
|
+
return [], []
|
|
144
|
+
ranks = np.unique(np.concatenate([[1], np.round(np.logspace(0, np.log10(n), points)).astype(np.int64), [n]]))
|
|
145
|
+
ranks = ranks[(ranks >= 1) & (ranks <= n)]
|
|
146
|
+
return [int(r) for r in ranks], [_num(float(x)) for x in v[ranks - 1]]
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def summary_stats(values) -> dict:
|
|
150
|
+
"""Median, mean and the 10th and 90th percentile of one number per unit (None when there is no unit)."""
|
|
151
|
+
v = np.asarray(values, dtype=np.float64)
|
|
152
|
+
if len(v) == 0:
|
|
153
|
+
return {"median": None, "mean": None, "p10": None, "p90": None, "n": 0}
|
|
154
|
+
p10, med, p90 = np.percentile(v, [10, 50, 90])
|
|
155
|
+
return {"median": float(med), "mean": float(v.mean()), "p10": float(p10), "p90": float(p90), "n": int(len(v))}
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
# ---------------------------------------------------------------------------------------------------------
|
|
159
|
+
# grid channels (the spatial map): one byte per cell, row-major, base64 of the raw bytes ("u8-b64")
|
|
160
|
+
# ---------------------------------------------------------------------------------------------------------
|
|
161
|
+
LEVELS = 255 # levels 0..254 carry values; 255 means "no data" on a channel that says nodata
|
|
162
|
+
NODATA = 255
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def encode_u8(q) -> str:
|
|
166
|
+
return base64.b64encode(np.ascontiguousarray(q, dtype=np.uint8).tobytes()).decode("ascii")
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def decode_u8(text: str) -> np.ndarray:
|
|
170
|
+
return np.frombuffer(base64.b64decode(text), dtype=np.uint8)
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def channel_log1p(a, *, unit: str | None = None, valid=None) -> dict:
|
|
174
|
+
"""value = expm1(level * scale). ``valid`` marks the cells that have a value (the others say "no data")."""
|
|
175
|
+
a = np.asarray(a, dtype=np.float64)
|
|
176
|
+
vmax = float(a.max()) if a.size else 0.0
|
|
177
|
+
scale = float(np.log1p(vmax) / (LEVELS - 1)) if vmax > 0 else 1.0
|
|
178
|
+
q = np.rint(np.log1p(np.maximum(a, 0)) / scale).astype(np.int64) if vmax > 0 else np.zeros(a.shape, np.int64)
|
|
179
|
+
q = np.clip(q, 0, LEVELS - 1).astype(np.uint8)
|
|
180
|
+
ch = {"enc": "u8-b64", "transform": "log1p", "scale": scale, "offset": 0, "levels": LEVELS}
|
|
181
|
+
if valid is not None:
|
|
182
|
+
q = np.where(np.asarray(valid, dtype=bool), q, NODATA).astype(np.uint8)
|
|
183
|
+
ch["nodata"] = True
|
|
184
|
+
if unit:
|
|
185
|
+
ch["unit"] = unit
|
|
186
|
+
ch["data"] = encode_u8(q)
|
|
187
|
+
return ch
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def channel_linear(a, *, lo: float = 0.0, hi: float = 1.0, unit: str | None = None, valid=None) -> dict:
|
|
191
|
+
"""value = offset + level * scale, between ``lo`` and ``hi``."""
|
|
192
|
+
a = np.asarray(a, dtype=np.float64)
|
|
193
|
+
scale = (hi - lo) / (LEVELS - 1) if hi > lo else 1.0
|
|
194
|
+
q = np.clip(np.rint((np.clip(a, lo, hi) - lo) / scale), 0, LEVELS - 1).astype(np.uint8)
|
|
195
|
+
ch = {"enc": "u8-b64", "transform": "linear", "scale": float(scale), "offset": float(lo), "levels": LEVELS}
|
|
196
|
+
if valid is not None:
|
|
197
|
+
q = np.where(np.asarray(valid, dtype=bool), q, NODATA).astype(np.uint8)
|
|
198
|
+
ch["nodata"] = True
|
|
199
|
+
if unit:
|
|
200
|
+
ch["unit"] = unit
|
|
201
|
+
ch["data"] = encode_u8(q)
|
|
202
|
+
return ch
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def channel_values(ch: dict) -> np.ndarray:
|
|
206
|
+
"""The values a channel stands for (NaN where it says "no data"): what the viewer shows, for the tests."""
|
|
207
|
+
q = decode_u8(ch["data"]).astype(np.float64)
|
|
208
|
+
v = np.expm1(q * ch["scale"]) if ch.get("transform") == "log1p" else ch.get("offset", 0.0) + q * ch["scale"]
|
|
209
|
+
if ch.get("nodata"):
|
|
210
|
+
v = np.where(q == NODATA, np.nan, v)
|
|
211
|
+
return v
|