flowcellar 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (159) hide show
  1. flowcellar/__init__.py +9 -0
  2. flowcellar/__main__.py +4 -0
  3. flowcellar/_version.py +1 -0
  4. flowcellar/aggregates/__init__.py +11 -0
  5. flowcellar/aggregates/build.py +151 -0
  6. flowcellar/aggregates/derive.py +109 -0
  7. flowcellar/aggregates/merge.py +362 -0
  8. flowcellar/catalogue/__init__.py +63 -0
  9. flowcellar/catalogue/metrics_nanopore.json +1 -0
  10. flowcellar/cli/__init__.py +97 -0
  11. flowcellar/cli/_options.py +25 -0
  12. flowcellar/cli/analysis.py +317 -0
  13. flowcellar/cli/discover.py +168 -0
  14. flowcellar/cli/doctor.py +31 -0
  15. flowcellar/cli/hub.py +47 -0
  16. flowcellar/cli/init.py +41 -0
  17. flowcellar/cli/ls.py +94 -0
  18. flowcellar/cli/recompute.py +52 -0
  19. flowcellar/cli/rm.py +21 -0
  20. flowcellar/cli/run.py +333 -0
  21. flowcellar/cli/scan.py +60 -0
  22. flowcellar/cli/schema.py +25 -0
  23. flowcellar/cli/serve.py +46 -0
  24. flowcellar/cli/show.py +109 -0
  25. flowcellar/cli/validate.py +44 -0
  26. flowcellar/config/__init__.py +4 -0
  27. flowcellar/config/defaults.toml +51 -0
  28. flowcellar/config/loader.py +369 -0
  29. flowcellar/contracts.py +159 -0
  30. flowcellar/discovery/__init__.py +141 -0
  31. flowcellar/discovery/guard.py +208 -0
  32. flowcellar/discovery/patterns.py +209 -0
  33. flowcellar/discovery/walk.py +348 -0
  34. flowcellar/engine/__init__.py +1 -0
  35. flowcellar/engine/api.py +119 -0
  36. flowcellar/engine/assemble.py +363 -0
  37. flowcellar/engine/baselines.py +128 -0
  38. flowcellar/engine/card.py +250 -0
  39. flowcellar/engine/copy_pending.py +110 -0
  40. flowcellar/engine/flowcell_use.py +85 -0
  41. flowcellar/engine/identity.py +351 -0
  42. flowcellar/engine/limits.py +135 -0
  43. flowcellar/engine/passes.py +213 -0
  44. flowcellar/engine/status.py +728 -0
  45. flowcellar/library/__init__.py +4 -0
  46. flowcellar/library/api.py +61 -0
  47. flowcellar/library/cards.py +69 -0
  48. flowcellar/library/draft.py +247 -0
  49. flowcellar/library/index.py +323 -0
  50. flowcellar/library/layout.py +69 -0
  51. flowcellar/library/lease.py +248 -0
  52. flowcellar/library/maintain.py +347 -0
  53. flowcellar/library/projects.py +84 -0
  54. flowcellar/library/query.py +382 -0
  55. flowcellar/library/scan.py +1770 -0
  56. flowcellar/library/store.py +225 -0
  57. flowcellar/library/trash.py +63 -0
  58. flowcellar/model/__init__.py +1 -0
  59. flowcellar/model/axes.py +200 -0
  60. flowcellar/model/bundle.py +143 -0
  61. flowcellar/model/partial.py +96 -0
  62. flowcellar/model/runfolder.py +83 -0
  63. flowcellar/model/runkey.py +98 -0
  64. flowcellar/model/schema.py +46 -0
  65. flowcellar/model/schemas/bundle-0.1.1.schema.json +3713 -0
  66. flowcellar/model/status.py +97 -0
  67. flowcellar/modules/__init__.py +0 -0
  68. flowcellar/modules/common.py +73 -0
  69. flowcellar/modules/dyn.py +124 -0
  70. flowcellar/modules/fc.py +390 -0
  71. flowcellar/modules/len_est.py +74 -0
  72. flowcellar/modules/run.py +88 -0
  73. flowcellar/modules/yield_.py +120 -0
  74. flowcellar/readdata/__init__.py +1 -0
  75. flowcellar/readdata/_pool.py +87 -0
  76. flowcellar/readdata/acc.py +161 -0
  77. flowcellar/readdata/alnacc.py +572 -0
  78. flowcellar/readdata/alnpass.py +85 -0
  79. flowcellar/readdata/api.py +189 -0
  80. flowcellar/readdata/bam.py +851 -0
  81. flowcellar/readdata/bamindex.py +149 -0
  82. flowcellar/readdata/cards.py +82 -0
  83. flowcellar/readdata/checks.py +312 -0
  84. flowcellar/readdata/copy_pending.py +73 -0
  85. flowcellar/readdata/fate.py +341 -0
  86. flowcellar/readdata/finder.py +147 -0
  87. flowcellar/readdata/libprep.py +97 -0
  88. flowcellar/readdata/link.py +486 -0
  89. flowcellar/readdata/passes.py +149 -0
  90. flowcellar/readdata/patterns.py +230 -0
  91. flowcellar/readdata/pychopper.py +431 -0
  92. flowcellar/readdata/readtable.py +416 -0
  93. flowcellar/readdata/record.py +159 -0
  94. flowcellar/readdata/reference.py +139 -0
  95. flowcellar/readdata/rnacomp.py +329 -0
  96. flowcellar/readdata/sample_sheet.py +153 -0
  97. flowcellar/readdata/saturation.py +110 -0
  98. flowcellar/readdata/sections.py +892 -0
  99. flowcellar/readdata/sections_align.py +644 -0
  100. flowcellar/readdata/tables.py +262 -0
  101. flowcellar/report/__init__.py +1 -0
  102. flowcellar/report/api.py +54 -0
  103. flowcellar/report/html.py +342 -0
  104. flowcellar/report/hub.py +159 -0
  105. flowcellar/report/index_one.py +144 -0
  106. flowcellar/report/served.py +67 -0
  107. flowcellar/sc/__init__.py +57 -0
  108. flowcellar/sc/axes.py +211 -0
  109. flowcellar/sc/catalogue.json +1 -0
  110. flowcellar/sc/catalogue.py +51 -0
  111. flowcellar/sc/check.py +57 -0
  112. flowcellar/sc/collect.py +299 -0
  113. flowcellar/sc/collect_sr.py +450 -0
  114. flowcellar/sc/collect_wf.py +479 -0
  115. flowcellar/sc/detect.py +303 -0
  116. flowcellar/sc/judge.py +269 -0
  117. flowcellar/sc/link.py +326 -0
  118. flowcellar/sc/listing.py +144 -0
  119. flowcellar/sc/maps.py +181 -0
  120. flowcellar/sc/readpass.py +416 -0
  121. flowcellar/sc/report.py +230 -0
  122. flowcellar/sc/scan.py +176 -0
  123. flowcellar/sc/spaceranger.py +483 -0
  124. flowcellar/sc/store.py +537 -0
  125. flowcellar/sc/wfsc.py +419 -0
  126. flowcellar/serve/__init__.py +4 -0
  127. flowcellar/serve/server.py +153 -0
  128. flowcellar/sources/__init__.py +0 -0
  129. flowcellar/sources/minknow.py +455 -0
  130. flowcellar/sources/porescan.py +218 -0
  131. flowcellar/sources/seqsummary.py +450 -0
  132. flowcellar/sources/workers.py +244 -0
  133. flowcellar/testing/__init__.py +1 -0
  134. flowcellar/testing/demo.py +1217 -0
  135. flowcellar/testing/demo_spatial.py +370 -0
  136. flowcellar/testing/fixtures.py +459 -0
  137. flowcellar/testing/libprep.py +252 -0
  138. flowcellar/testing/minknow_variants.py +384 -0
  139. flowcellar/testing/readdata.py +744 -0
  140. flowcellar/testing/sc.py +817 -0
  141. flowcellar/testing/trees.py +175 -0
  142. flowcellar/util/__init__.py +1 -0
  143. flowcellar/util/atomic.py +112 -0
  144. flowcellar/util/errors.py +72 -0
  145. flowcellar/util/fingerprint.py +49 -0
  146. flowcellar/util/log.py +50 -0
  147. flowcellar/util/text.py +43 -0
  148. flowcellar/util/timeutil.py +52 -0
  149. flowcellar/viewer_dist/favicon.svg +1 -0
  150. flowcellar/viewer_dist/manifest.json +37 -0
  151. flowcellar/viewer_dist/template.html +12 -0
  152. flowcellar/viewer_dist/viewer.core.js +3 -0
  153. flowcellar/viewer_dist/viewer.css +2151 -0
  154. flowcellar-0.1.0.dist-info/METADATA +506 -0
  155. flowcellar-0.1.0.dist-info/RECORD +159 -0
  156. flowcellar-0.1.0.dist-info/WHEEL +4 -0
  157. flowcellar-0.1.0.dist-info/entry_points.txt +2 -0
  158. flowcellar-0.1.0.dist-info/licenses/LICENSE +674 -0
  159. flowcellar-0.1.0.dist-info/licenses/THIRD_PARTY_NOTICES.md +85 -0
@@ -0,0 +1,159 @@
1
+ """The static hub of a library (mode B).
2
+
3
+ hub/index.html the viewer and no data; rewritten only when the viewer changed
4
+ hub/index.js the index, as one call of __FC_INDEX__; rewritten every time
5
+ hub/data/<run_key>.js one run, as one call of __FCB_SIDECAR__; rewritten only when its bundle or its
6
+ status file is newer than it, or when the caller names the run
7
+
8
+ Every write is a whole-file replace, so a browser that reads while a scan writes never sees half a file.
9
+ """
10
+ from __future__ import annotations
11
+
12
+ import os
13
+
14
+ from flowcellar.model.bundle import bundle_filename, report_filename
15
+ from flowcellar.model.status import status_filename
16
+ from flowcellar.report import html as page
17
+ from flowcellar.report import index_one
18
+ from flowcellar.util import atomic
19
+
20
+ SIDECAR_SUFFIX = ".js"
21
+
22
+
23
+ def sidecar_path(hub_dir: str, run_key: str) -> str:
24
+ return os.path.join(hub_dir, "data", run_key + SIDECAR_SUFFIX)
25
+
26
+
27
+ def _mtime_ns(path: str) -> int | None:
28
+ try:
29
+ return os.stat(path).st_mtime_ns
30
+ except OSError:
31
+ return None
32
+
33
+
34
+ def source_mtime_ns(library, run_key: str) -> int | None:
35
+ """When the run's bundle or its status file last changed; None when neither can be found (then nothing can
36
+ be trusted to be up to date)."""
37
+ times = [_mtime_ns(os.path.join(library.path, "bundles", bundle_filename(run_key))),
38
+ _mtime_ns(os.path.join(library.path, "status", status_filename(run_key)))]
39
+ times = [t for t in times if t is not None]
40
+ return max(times) if times else None
41
+
42
+
43
+ def is_fresh(library, hub_dir: str, run_key: str) -> bool:
44
+ """True when the sidecar exists and is newer than the run's bundle and status file."""
45
+ mine, source = _mtime_ns(sidecar_path(hub_dir, run_key)), source_mtime_ns(library, run_key)
46
+ return mine is not None and source is not None and mine > source
47
+
48
+
49
+ def _read(path: str) -> bytes | None:
50
+ try:
51
+ with open(path, "rb") as fh:
52
+ return fh.read()
53
+ except OSError:
54
+ return None
55
+
56
+
57
+ def _head(path: str, n: int = 2048) -> bytes:
58
+ try:
59
+ with open(path, "rb") as fh:
60
+ return fh.read(n)
61
+ except OSError:
62
+ return b""
63
+
64
+
65
+ def _write_if_changed(path: str, data: bytes) -> bool:
66
+ if _read(path) == data:
67
+ return False
68
+ atomic.atomic_write_bytes(path, data)
69
+ return True
70
+
71
+
72
+ def _has_analyses(library_path: str, data_dir: str) -> bool:
73
+ for folder in (os.path.join(library_path, "bundles"), data_dir):
74
+ try:
75
+ if any(n.startswith("an_") for n in os.listdir(folder)):
76
+ return True
77
+ except OSError:
78
+ pass
79
+ return False
80
+
81
+
82
+ def write_hub(library, *, sidecars: bool = True, reports: bool = False, only=None) -> dict:
83
+ hub_dir = library.hub_dir()
84
+ data_dir = os.path.join(hub_dir, "data")
85
+ os.makedirs(hub_dir, exist_ok=True)
86
+ keys = [k for k in library.run_keys() if page.RUN_KEY_RE.match(k)]
87
+ known = set(keys)
88
+ written: list[str] = []
89
+ removed: list[str] = []
90
+
91
+ # the page: only when the viewer (or this package's text around it) changed
92
+ index_html = os.path.join(hub_dir, "index.html")
93
+ if _write_if_changed(index_html, page.hub_page().encode("utf-8")):
94
+ written.append(index_html)
95
+
96
+ # the index: always
97
+ index_js = os.path.join(hub_dir, "index.js")
98
+ atomic.atomic_write_bytes(index_js, page.index_sidecar(library.index_gzip()))
99
+ written.append(index_js)
100
+
101
+ # data files of runs that are no longer in the library
102
+ if os.path.isdir(data_dir):
103
+ for name in sorted(os.listdir(data_dir)):
104
+ if not name.endswith(SIDECAR_SUFFIX) or name.startswith("."):
105
+ continue
106
+ key = name[: -len(SIDECAR_SUFFIX)]
107
+ if key.startswith("an_"):
108
+ continue # the data file of an analysis: looked after below
109
+ stale = key not in known or (not sidecars and not is_fresh(library, hub_dir, key))
110
+ if stale:
111
+ os.unlink(os.path.join(data_dir, name))
112
+ removed.append(os.path.join(data_dir, name))
113
+
114
+ todo: list[str] = []
115
+ if sidecars:
116
+ wanted = set(only) & known if only is not None else None
117
+ for key in keys:
118
+ if (wanted is not None and key in wanted) or not is_fresh(library, hub_dir, key):
119
+ todo.append(key)
120
+ for key in todo:
121
+ path = sidecar_path(hub_dir, key)
122
+ data = page.run_sidecar(key, library.composed_gzip(key))
123
+ if _read(path) == data and (wanted is None or key not in wanted):
124
+ os.utime(path) # same content: only say that it is up to date
125
+ continue
126
+ atomic.atomic_write_bytes(path, data)
127
+ written.append(path)
128
+
129
+ # single-cell and spatial analyses keep their data files beside those of the runs; the add-on writes them and
130
+ # is loaded only when the library holds an analysis or the folder still has such a file
131
+ if _has_analyses(library.path, data_dir):
132
+ from flowcellar.sc import report as sc_report
133
+ out = sc_report.write_sidecars(library, hub_dir, sidecars=sidecars, only=only)
134
+ written.extend(out["written"])
135
+ removed.extend(out["removed"])
136
+
137
+ if reports:
138
+ index = library.index()
139
+ rep_dir = os.path.join(library.path, "reports")
140
+ named = set(only or ())
141
+ script = page.load_viewer().js_csp.encode("ascii").replace(b"'", b"&#x27;") # as the policy element spells it
142
+ for key in keys:
143
+ path = os.path.join(rep_dir, report_filename(key))
144
+ source, mine = source_mtime_ns(library, key), _mtime_ns(path)
145
+ # a report is kept when it is newer than the run's data, was made with this viewer and was not asked for
146
+ if mine is not None and source is not None and mine > source and key not in named and key not in todo and script in _head(path):
147
+ continue
148
+ bundle = library.composed(key)
149
+ one = index_one.slice_index(index, key) or index_one.plain_index(bundle)
150
+ text, _ = page.report_page(bundle, one)
151
+ atomic.atomic_write_bytes(path, text.encode("utf-8"))
152
+ written.append(path)
153
+ if os.path.isdir(rep_dir):
154
+ for name in sorted(os.listdir(rep_dir)):
155
+ if name.endswith(report_filename("")) and name[: -len(report_filename(""))] not in known:
156
+ os.unlink(os.path.join(rep_dir, name))
157
+ removed.append(os.path.join(rep_dir, name))
158
+
159
+ return {"index_html": index_html, "written": written, "removed": removed}
@@ -0,0 +1,144 @@
1
+ """The one-run index a report embeds (mode A).
2
+
3
+ Two sources:
4
+
5
+ * ``slice_index``: one row cut out of a library's index (``flowcellar hub --reports``);
6
+ * ``index_for_bundle``: built from a composed bundle alone. It first asks LIBRARY
7
+ (``library.api.build_index([card_for(bundle)])``), which owns the index; until that is built, or when it
8
+ fails, ``plain_index`` writes the columns that can be read straight from the bundle. The viewer draws a run
9
+ from its bundle and needs the row only for the page title and for the line it shows while the bundle decodes.
10
+ """
11
+ from __future__ import annotations
12
+
13
+ from flowcellar import SCHEMA_VERSION, __version__
14
+ from flowcellar.model import status as status_model
15
+ from flowcellar.util.errors import FlowCellarError
16
+ from flowcellar.util.timeutil import now_z
17
+
18
+ FORMAT = "flowcellar.index"
19
+ _VERDICT_KEYS = ("pass", "warn", "fail", "incomplete", "running", "unknown")
20
+
21
+
22
+ def _summaries(bundle: dict) -> dict:
23
+ out: dict = {}
24
+ for m in (bundle.get("modules") or {}).values():
25
+ out.update(m.get("summary") or {})
26
+ return out
27
+
28
+
29
+ def plain_columns(bundle: dict) -> dict:
30
+ """The index row of one run, as far as the bundle alone tells it."""
31
+ ident, st, tiers = bundle.get("identity") or {}, bundle.get("status") or {}, bundle.get("tiers") or {}
32
+ fc, pr, dev = ident.get("flow_cell") or {}, ident.get("protocol_run") or {}, ident.get("device") or {}
33
+ kit, assay, rel = ident.get("kit") or {}, ident.get("assay") or {}, bundle.get("relations") or {}
34
+ use, by, s = rel.get("flow_cell_use") or {}, st.get("by_tier") or {}, _summaries(bundle)
35
+ rh, rl = tiers.get("run_health") or {}, tiers.get("read_level") or {}
36
+ rs = status_model.reasons(st)
37
+ projects = rel.get("projects") or []
38
+ gb = s.get("yield_estimated_bases")
39
+ return {
40
+ "id": bundle.get("bundle_id") or ident.get("run_key"), "fc": fc.get("id"),
41
+ "use": use.get("index") or 1, "use_of": use.get("of") or 1, "use_source": use.get("source") or "unknown",
42
+ "restart": bool(use.get("restart_of")), "restart_of": use.get("restart_of"),
43
+ "started": pr.get("started"), "hours": s.get("run_duration_h"),
44
+ "planned_h": (pr.get("planned_duration_s") or 0) / 3600 or None,
45
+ "run_state": pr.get("run_state") or "finished",
46
+ "device_type": dev.get("type"), "position": dev.get("position"), "product": fc.get("product_code"),
47
+ "kit": kit.get("sequencing_kit"), "assay": assay.get("type"), "assay_group": assay.get("group") or "unknown",
48
+ "chemistry": assay.get("chemistry"), "minknow": (ident.get("software") or {}).get("minknow"),
49
+ "experiment": pr.get("experiment_id"), "run_name": pr.get("sample_id"), "channels": fc.get("channel_count"),
50
+ "group": " | ".join(str(x) for x in (dev.get("type"), fc.get("product_code"), kit.get("sequencing_kit"))),
51
+ "folder": None, "roots": sorted({r for p in projects for r in (p.get("roots") or [])}),
52
+ "project": projects[0]["name"] if projects else None, "project_ids": [p["id"] for p in projects],
53
+ "source_state": "ok",
54
+ "verdict": st.get("verdict") or "unknown", "health": by.get("run_health") or "not_available",
55
+ "data": by.get("read_level") or "not_available", "provisional": bool(st.get("provisional")),
56
+ "pending": len(st.get("pending") or []), "checks_done": len(st.get("rules") or []), "cause": st.get("cause"),
57
+ "modifiers": [m.get("code") for m in st.get("modifiers") or []],
58
+ "headline": st.get("headline") or "", "detail": st.get("detail") or "",
59
+ "reasons": [r["message"] for r in rs], "reason_levels": [r["severity"] for r in rs],
60
+ "reason_checks": [r["check"] for r in rs], "reason_tiers": [r["tier"] for r in rs],
61
+ "reads": s.get("yield_reads"), "gb": gb / 1e9 if gb is not None else None,
62
+ "pores": s.get("fc_pores_first_scan"), "pores_last": s.get("fc_pores_last_scan"),
63
+ "occ_first": s.get("fc_occupancy_first_hour"),
64
+ "detail_t1": rh.get("detail") if rh.get("state") not in (None, "absent") else None,
65
+ "read_data": rl.get("state") or "absent", "t2_population": (rl.get("population") or {}).get("stage"),
66
+ "t2_samples": rl.get("samples_expected"), "t2_samples_with_data": rl.get("samples_with_data"),
67
+ "t2_reads": rl.get("reads_covered"),
68
+ "t2_passed_gb": s["yield_basecalled_bases"] / 1e9 if s.get("yield_basecalled_bases") is not None else None,
69
+ "t2_n50": s.get("len_n50"), "t2_q_med": s.get("qual_median"), "t2_usable": s.get("funnel_usable_frac"),
70
+ "t2_files_waiting": 0, "spark_output": "", "spark_output_max_gb_per_3h": 0,
71
+ }
72
+
73
+
74
+ def plain_index(bundle: dict) -> dict:
75
+ """A one-run index in the shape of the library index, from the bundle alone."""
76
+ cols = plain_columns(bundle)
77
+ prov_lib = (bundle.get("provenance") or {}).get("library") or {}
78
+ samples = [{"sample_uid": s.get("sample_uid"), "name": s.get("name") or s.get("alias") or s.get("key"), "alias": s.get("alias"),
79
+ "barcode": s.get("barcode"), "project": s.get("project"), "run": cols["id"], "key": s.get("key"),
80
+ "reads": None, "bases": None, "n50": None, "q_med": None}
81
+ for s in bundle.get("samples") or [] if s.get("key") != "unclassified"]
82
+ verdicts = {k: 0 for k in _VERDICT_KEYS}
83
+ verdicts[cols["verdict"] if cols["verdict"] in verdicts else "unknown"] = 1
84
+ started = cols["started"]
85
+ return {
86
+ "format": FORMAT, "schema": SCHEMA_VERSION, "written": now_z(),
87
+ "meta": {"library": prov_lib.get("name"), "runs": 1, "generator": {"name": "flowcellar", "version": __version__}},
88
+ "library": {"id": prov_lib.get("id"), "name": prov_lib.get("name"), "path": None, "created": None, "scanned": None,
89
+ "counts": {"runs": 1, "flow_cells": 1, "projects": len(cols["project_ids"]), "samples": len(samples),
90
+ "by_verdict": verdicts, "by_detail": {}, "with_read_data": int(cols["read_data"] != "absent")},
91
+ "estimated_bases": None, "reads": cols["reads"], "first_run": started, "last_run": started},
92
+ "runs": {"n": 1, "columns": {k: [v] for k, v in cols.items()}},
93
+ "samples": samples, "projects": [], "roots": [], "activity": {"months": [], "runs": [], "estimated_bases": [], "reads": []},
94
+ "baselines": {"groups": []}, "facets": {}, "notices": [],
95
+ "progress": {"state": "idle"},
96
+ }
97
+
98
+
99
+ def index_for_bundle(bundle: dict) -> dict:
100
+ """The index of a report made from one composed bundle: LIBRARY's when it can make one, else the plain one."""
101
+ try:
102
+ from flowcellar.library import api as library_api
103
+
104
+ status_file = None
105
+ if bundle.get("status") is not None:
106
+ status_file = status_model.new_status_file(bundle.get("bundle_id"), bundle.get("content_sha256"),
107
+ bundle["status"], bundle.get("relations") or {})
108
+ index = library_api.build_index([library_api.card_for(bundle, status_file)])
109
+ if isinstance(index, dict) and (index.get("runs") or {}).get("n") == 1:
110
+ return index
111
+ except FlowCellarError:
112
+ pass # not built yet, or it cannot make a card of this bundle
113
+ return plain_index(bundle)
114
+
115
+
116
+ def _value(col, i):
117
+ if isinstance(col, dict) and "dict" in col:
118
+ return col["dict"][col["codes"][i]]
119
+ return col[i]
120
+
121
+
122
+ def slice_index(index: dict, run_key: str) -> dict | None:
123
+ """One row of a library index as an index of its own: what a report of that run embeds. Names of things that
124
+ are not in the file (other runs, other projects, the folders that were scanned) are left out."""
125
+ cols = (index.get("runs") or {}).get("columns") or {}
126
+ n = (index.get("runs") or {}).get("n") or 0
127
+ ids = cols.get("id")
128
+ i = next((k for k in range(n) if _value(ids, k) == run_key), None) if ids is not None else None
129
+ if i is None:
130
+ return None
131
+ row = {k: _value(c, i) for k, c in cols.items()}
132
+ out = {k: v for k, v in index.items() if k not in ("runs", "samples", "projects", "roots", "activity", "baselines", "notices", "progress", "facets", "library")}
133
+ lib = index.get("library") or {}
134
+ out["library"] = {"id": lib.get("id"), "name": lib.get("name"), "path": None, "created": lib.get("created"), "scanned": lib.get("scanned")}
135
+ out["runs"] = {"n": 1, "columns": {k: [v] for k, v in row.items()}}
136
+ out["samples"] = [s for s in index.get("samples") or [] if s.get("run") == run_key]
137
+ out["projects"] = [p for p in index.get("projects") or [] if p.get("id") in (row.get("project_ids") or [])]
138
+ out["roots"] = []
139
+ out["activity"] = {"months": [], "runs": [], "estimated_bases": [], "reads": []}
140
+ out["baselines"] = {"groups": [g for g in (index.get("baselines") or {}).get("groups") or [] if g.get("group") == row.get("group")]}
141
+ out["facets"] = {}
142
+ out["notices"] = []
143
+ out["progress"] = {"state": "idle"}
144
+ return out
@@ -0,0 +1,67 @@
1
+ """What ``flowcellar serve`` answers (mode C).
2
+
3
+ Four addresses and nothing else. No path that came from a request is ever joined to a folder: the page is the
4
+ one fixed file ``hub/index.html``, the index is made from the library, and a run's data file is found from a run
5
+ key that matched ``RUN_KEY_RE`` and is one of the library's own.
6
+ """
7
+ from __future__ import annotations
8
+
9
+ import html as html_lib
10
+ import os
11
+ import re
12
+
13
+ from flowcellar.contracts import Served
14
+ from flowcellar.report import html as page
15
+ from flowcellar.report import hub as hub_files
16
+ from flowcellar.util import atomic
17
+ from flowcellar.util.errors import FlowCellarError
18
+
19
+ HTML = "text/html; charset=utf-8"
20
+ JS = "text/javascript; charset=utf-8"
21
+ _META_CSP = re.compile(rb'<meta http-equiv="Content-Security-Policy" content="([^"]*)">')
22
+
23
+
24
+ def headers(policy: str | None = None) -> dict[str, str]:
25
+ """On every answer. ``policy`` is the one of the page being served; by default that of the installed viewer."""
26
+ return {"Content-Security-Policy": policy or page.hub_policy(), "X-Content-Type-Options": "nosniff",
27
+ "Referrer-Policy": "no-referrer", "Cross-Origin-Opener-Policy": "same-origin", "Cache-Control": "no-store"}
28
+
29
+
30
+ def page_policy(body: bytes) -> str | None:
31
+ """The policy a hub page states in its own meta element. The header must say the same: a browser applies
32
+ both, and a page written by an older viewer carries the hash of that viewer's script."""
33
+ m = _META_CSP.search(body[:4096])
34
+ return html_lib.unescape(m.group(1).decode("utf-8")) if m else None
35
+
36
+
37
+ def serve_file(library, url_path: str) -> Served | None:
38
+ path = (url_path or "").split("#", 1)[0].split("?", 1)[0] # the viewer asks for index.js?t=<counter>
39
+ hub_dir = library.hub_dir()
40
+ if path in ("/", "/index.html"):
41
+ try:
42
+ body = atomic.read_bytes(os.path.join(hub_dir, "index.html"), retries=1)
43
+ except OSError:
44
+ return None # no hub was written: serve says how to write one
45
+ return Served(body, HTML, headers(page_policy(body)))
46
+ if path == "/index.js":
47
+ try:
48
+ return Served(page.index_sidecar(library.index_gzip()), JS, headers())
49
+ except (FlowCellarError, OSError):
50
+ return None
51
+ if path.startswith("/data/") and path.endswith(".js"):
52
+ run_key = path[len("/data/"): -len(".js")]
53
+ if run_key.startswith("an_"): # an analysis of the single-cell and spatial add-on
54
+ from flowcellar.sc import report as sc_report
55
+ return sc_report.serve_sidecar(library, hub_dir, run_key, headers())
56
+ if not page.RUN_KEY_RE.match(run_key) or run_key not in set(library.run_keys()):
57
+ return None
58
+ if hub_files.is_fresh(library, hub_dir, run_key):
59
+ try:
60
+ return Served(atomic.read_bytes(hub_files.sidecar_path(hub_dir, run_key), retries=1), JS, headers())
61
+ except OSError:
62
+ pass
63
+ try:
64
+ return Served(page.run_sidecar(run_key, library.composed_gzip(run_key)), JS, headers())
65
+ except (FlowCellarError, OSError, KeyError):
66
+ return None
67
+ return None
@@ -0,0 +1,57 @@
1
+ """The single-cell and spatial add-on: quality control of wf-single-cell and Space Ranger outputs.
2
+
3
+ An *analysis* is one output folder of one pipeline for one sample: a wf-single-cell sample folder, or a Space
4
+ Ranger ``outs/``. It gets a bundle of its own (``kind: "analysis"``), because one analysis pools the reads of
5
+ several flow cells; the bundle names the runs it came from with the evidence for each link. An analysis has its
6
+ own verdict and never changes the verdict of a flow cell run.
7
+
8
+ Nothing of this package is imported until an analysis is looked for or shown: ``import flowcellar``, the run
9
+ commands and a run report do not load it. Importing this package itself loads nothing heavy either: ``h5py``
10
+ (the ``sc`` extra) is imported inside the functions that open an HDF5 file, ``pysam`` (the ``bam`` extra) inside
11
+ the function that tallies the read tags of a BAM. Without ``h5py`` the files that are tables or JSON are still
12
+ read and the bundle says what is missing.
13
+
14
+ flowcellar.sc.detect which folders are analyses (by their files, never by their names)
15
+ flowcellar.sc.collect one analysis folder -> the measured bundle
16
+ flowcellar.sc.link the evidence ladder from an analysis to the flow cell runs it came from
17
+ flowcellar.sc.judge the limits and the verdict of an analysis
18
+ flowcellar.sc.store analyses inside a library, the index table, what a scan does
19
+ flowcellar.sc.report the one-file report and the hub's data file of an analysis
20
+ """
21
+ from __future__ import annotations
22
+
23
+ import re
24
+
25
+ MODULE_ID = "sc"
26
+ #: version of the section this package writes; a change makes every cached collection stale
27
+ MODULE_VERSION = "0.1.0"
28
+ SECTION_SCHEMA_VERSION = 1
29
+
30
+ PIPELINE_SPACERANGER = "spaceranger"
31
+ PIPELINE_WF = "wf-single-cell"
32
+ #: how a pipeline is spelled inside a bundle id (letters only: the id is a file name and an address)
33
+ PIPELINE_CODE = {PIPELINE_SPACERANGER: "spaceranger", PIPELINE_WF: "wfsinglecell"}
34
+ PIPELINE_WORD = {PIPELINE_SPACERANGER: "Space Ranger", PIPELINE_WF: "wf-single-cell"}
35
+
36
+ #: the id of an analysis bundle as it may appear in a file name, an element id and an address
37
+ ANALYSIS_ID_RE = re.compile(r"^an_[a-z]{2,16}_[0-9a-f]{12}$")
38
+
39
+ #: the roles under which the discovery walk reports the anchor file of an analysis
40
+ ROLE_SPACERANGER = "analysis.spaceranger"
41
+ ROLE_WF = "analysis.wf_single_cell"
42
+ ROLE_PREFIX = "analysis."
43
+
44
+ ASSAY_WORD = {"visium": "Visium", "visium_hd": "Visium HD", "singlecell": "Single cell", None: "Assay not known"}
45
+
46
+ #: size budget of one analysis section as JSON text (bytes): the default aim and the cap a bundle may not pass
47
+ SECTION_BUDGET_BYTES = 150_000
48
+ SECTION_CAP_BYTES = 400_000
49
+
50
+
51
+ def bundle_id(pipeline: str, analysis_hash: str) -> str:
52
+ """``an_<pipeline>_<first 12 hex of the content hash>``."""
53
+ return f"an_{PIPELINE_CODE[pipeline]}_{analysis_hash[:12]}"
54
+
55
+
56
+ def is_analysis_id(text) -> bool:
57
+ return isinstance(text, str) and bool(ANALYSIS_ID_RE.match(text))
flowcellar/sc/axes.py ADDED
@@ -0,0 +1,211 @@
1
+ """The three fixed axes of the add-on, and its aggregates in their stored form.
2
+
3
+ The edges are the same for every analysis, so histograms of two analyses can be overlaid and added. The
4
+ definitions are copied into ``bundle.axes`` of every analysis bundle (a bundle describes itself); they are kept
5
+ here and not in the core's axis registry so that the core does not grow.
6
+
7
+ log_counts_8_per_decade 8 bins per decade from 1 to 1,000,000; a count of 0 is the underflow cell
8
+ fraction_50 50 bins of 0.02 from 0 to 1 (1.0 falls in the last bin)
9
+ reads_per_molecule 1 to 64 one by one, then one bin per power of two up to 2^20
10
+ """
11
+ from __future__ import annotations
12
+
13
+ import base64
14
+
15
+ import numpy as np
16
+
17
+ LOG_COUNTS = "log_counts_8_per_decade"
18
+ FRACTION_50 = "fraction_50"
19
+ READS_PER_MOLECULE = "reads_per_molecule"
20
+
21
+
22
+ def _log_edges() -> np.ndarray:
23
+ i = np.arange(0, 6 * 8 + 1)
24
+ e = np.power(10.0, i / 8.0)
25
+ whole = (i % 8) == 0
26
+ e[whole] = np.power(10.0, (i[whole] // 8).astype(np.float64)) # whole decades are exact
27
+ return e
28
+
29
+
30
+ EDGES: dict[str, np.ndarray] = {
31
+ LOG_COUNTS: _log_edges(),
32
+ FRACTION_50: np.arange(0, 51) / 50.0,
33
+ READS_PER_MOLECULE: np.concatenate([np.arange(1, 66), 2 ** np.arange(7, 21)]).astype(np.float64),
34
+ }
35
+ for _e in EDGES.values():
36
+ _e.setflags(write=False)
37
+
38
+ AXES: dict[str, dict] = {
39
+ LOG_COUNTS: {"type": "log10", "min": 0, "bins_per_decade": 8, "n": 48, "unit": "count", "family": "log_counts",
40
+ "note": "a unit with a count of 0 is in the underflow cell"},
41
+ FRACTION_50: {"type": "linear", "start": 0, "step": 0.02, "n": 50, "unit": "fraction", "family": "fraction",
42
+ "closed_last": True},
43
+ READS_PER_MOLECULE: {"type": "edges", "edges": [float(x) for x in EDGES[READS_PER_MOLECULE]], "unit": "reads",
44
+ "family": "reads_per_molecule",
45
+ "note": "1 to 64 reads one by one, then one bin per power of two; w holds the reads of each bin"},
46
+ }
47
+
48
+
49
+ def axis_defs(names) -> dict:
50
+ return {n: dict(AXES[n]) for n in sorted(set(names))}
51
+
52
+
53
+ def bin_counts(values, axis: str, weights=None) -> np.ndarray:
54
+ """Internal array (underflow, the bins, overflow) of ``values`` on ``axis``; with ``weights`` their sums."""
55
+ e = EDGES[axis]
56
+ v = np.asarray(values)
57
+ idx = np.searchsorted(e, v, side="right")
58
+ if AXES[axis].get("closed_last"):
59
+ idx = np.where(v == e[-1], len(e) - 1, idx)
60
+ if weights is None:
61
+ return np.bincount(idx, minlength=len(e) + 1).astype(np.int64)
62
+ return np.rint(np.bincount(idx, weights=np.asarray(weights, dtype=np.float64), minlength=len(e) + 1)).astype(np.int64)
63
+
64
+
65
+ def hist_from_full(axis: str, full, w_full=None, *, w_name: str | None = None, missing: int | None = None,
66
+ vmin=None, vmax=None) -> dict:
67
+ """Internal array -> stored histogram with empty ends trimmed (the layout of the bundle schema)."""
68
+ full = np.asarray(full)
69
+ inner = full[1:-1]
70
+ nz = np.nonzero(inner)[0]
71
+ a, b = (int(nz[0]), int(nz[-1]) + 1) if len(nz) else (0, 0)
72
+ h: dict = {"axis": axis, "start": a, "n": [int(x) for x in inner[a:b]]}
73
+ if w_full is not None:
74
+ w_full = np.asarray(w_full)
75
+ h["w"] = [int(x) for x in w_full[1:-1][a:b]]
76
+ if w_name:
77
+ h["w_name"] = w_name
78
+ h["w_under"], h["w_over"] = int(w_full[0]), int(w_full[-1])
79
+ h["under"], h["over"] = int(full[0]), int(full[-1])
80
+ if missing is not None:
81
+ h["missing"] = int(missing)
82
+ if vmin is not None:
83
+ h["min"] = vmin
84
+ if vmax is not None:
85
+ h["max"] = vmax
86
+ return h
87
+
88
+
89
+ def hist(axis: str, values, *, missing: int | None = None) -> dict:
90
+ """Stored histogram of ``values`` (one per unit)."""
91
+ v = np.asarray(values)
92
+ vmin = float(v.min()) if len(v) else None
93
+ vmax = float(v.max()) if len(v) else None
94
+ return hist_from_full(axis, bin_counts(v, axis), missing=missing, vmin=_num(vmin), vmax=_num(vmax))
95
+
96
+
97
+ def _num(x):
98
+ if x is None:
99
+ return None
100
+ return int(x) if float(x).is_integer() else float(x)
101
+
102
+
103
+ def reads_per_molecule_hist(counts_by_c) -> dict:
104
+ """``counts_by_c[c]`` = molecules seen ``c`` times -> stored histogram with ``w`` = the reads of each bin.
105
+
106
+ With ``w`` the mean number of reads of the molecules in a wide bin is known, and the total number of reads is
107
+ exact, so the saturation curve drawn from the histogram ends exactly at the saturation of the data.
108
+ """
109
+ h = np.asarray(counts_by_c, dtype=np.int64)
110
+ c = np.arange(len(h), dtype=np.int64)
111
+ present = np.nonzero(h)[0]
112
+ if len(present) == 0:
113
+ return hist_from_full(READS_PER_MOLECULE, np.zeros(len(EDGES[READS_PER_MOLECULE]) + 1, np.int64),
114
+ np.zeros(len(EDGES[READS_PER_MOLECULE]) + 1, np.int64), w_name="reads")
115
+ idx = np.searchsorted(EDGES[READS_PER_MOLECULE], c[present], side="right")
116
+ n_cells = len(EDGES[READS_PER_MOLECULE]) + 1
117
+ full = np.bincount(idx, weights=h[present], minlength=n_cells).astype(np.int64)
118
+ w = np.bincount(idx, weights=(h[present] * c[present]).astype(np.float64), minlength=n_cells).astype(np.int64)
119
+ return hist_from_full(READS_PER_MOLECULE, full, w, w_name="reads", vmin=int(c[present][0]), vmax=int(c[present][-1]))
120
+
121
+
122
+ def saturation_from_hist(h: dict, fraction: float = 1.0) -> float | None:
123
+ """Saturation at a share ``fraction`` of the reads: ``1 - sum h[c] (1 - (1 - p)^c) / (p sum c h[c])``, the
124
+ exact expectation under random subsampling. A wide bin stands for its mean number of reads per molecule."""
125
+ n = np.asarray(h.get("n") or [], dtype=np.float64)
126
+ w = np.asarray(h.get("w") or [], dtype=np.float64)
127
+ if len(n) == 0 or w.sum() <= 0:
128
+ return None
129
+ ok = n > 0
130
+ c = w[ok] / n[ok]
131
+ p = float(fraction)
132
+ unique = float((n[ok] * (1.0 - np.power(1.0 - p, c))).sum()) if p < 1 else float(n[ok].sum())
133
+ return 1.0 - unique / (p * float(w.sum()))
134
+
135
+
136
+ def rank_curve(counts, *, points: int = 200) -> tuple[list[int], list[float]]:
137
+ """Counts sorted from the largest down, kept at up to ``points`` log-spaced ranks (rank 1 and the last rank
138
+ always kept): the barcode rank plot."""
139
+ v = np.sort(np.asarray(counts))[::-1]
140
+ v = v[v > 0]
141
+ n = len(v)
142
+ if n == 0:
143
+ return [], []
144
+ ranks = np.unique(np.concatenate([[1], np.round(np.logspace(0, np.log10(n), points)).astype(np.int64), [n]]))
145
+ ranks = ranks[(ranks >= 1) & (ranks <= n)]
146
+ return [int(r) for r in ranks], [_num(float(x)) for x in v[ranks - 1]]
147
+
148
+
149
+ def summary_stats(values) -> dict:
150
+ """Median, mean and the 10th and 90th percentile of one number per unit (None when there is no unit)."""
151
+ v = np.asarray(values, dtype=np.float64)
152
+ if len(v) == 0:
153
+ return {"median": None, "mean": None, "p10": None, "p90": None, "n": 0}
154
+ p10, med, p90 = np.percentile(v, [10, 50, 90])
155
+ return {"median": float(med), "mean": float(v.mean()), "p10": float(p10), "p90": float(p90), "n": int(len(v))}
156
+
157
+
158
+ # ---------------------------------------------------------------------------------------------------------
159
+ # grid channels (the spatial map): one byte per cell, row-major, base64 of the raw bytes ("u8-b64")
160
+ # ---------------------------------------------------------------------------------------------------------
161
+ LEVELS = 255 # levels 0..254 carry values; 255 means "no data" on a channel that says nodata
162
+ NODATA = 255
163
+
164
+
165
+ def encode_u8(q) -> str:
166
+ return base64.b64encode(np.ascontiguousarray(q, dtype=np.uint8).tobytes()).decode("ascii")
167
+
168
+
169
+ def decode_u8(text: str) -> np.ndarray:
170
+ return np.frombuffer(base64.b64decode(text), dtype=np.uint8)
171
+
172
+
173
+ def channel_log1p(a, *, unit: str | None = None, valid=None) -> dict:
174
+ """value = expm1(level * scale). ``valid`` marks the cells that have a value (the others say "no data")."""
175
+ a = np.asarray(a, dtype=np.float64)
176
+ vmax = float(a.max()) if a.size else 0.0
177
+ scale = float(np.log1p(vmax) / (LEVELS - 1)) if vmax > 0 else 1.0
178
+ q = np.rint(np.log1p(np.maximum(a, 0)) / scale).astype(np.int64) if vmax > 0 else np.zeros(a.shape, np.int64)
179
+ q = np.clip(q, 0, LEVELS - 1).astype(np.uint8)
180
+ ch = {"enc": "u8-b64", "transform": "log1p", "scale": scale, "offset": 0, "levels": LEVELS}
181
+ if valid is not None:
182
+ q = np.where(np.asarray(valid, dtype=bool), q, NODATA).astype(np.uint8)
183
+ ch["nodata"] = True
184
+ if unit:
185
+ ch["unit"] = unit
186
+ ch["data"] = encode_u8(q)
187
+ return ch
188
+
189
+
190
+ def channel_linear(a, *, lo: float = 0.0, hi: float = 1.0, unit: str | None = None, valid=None) -> dict:
191
+ """value = offset + level * scale, between ``lo`` and ``hi``."""
192
+ a = np.asarray(a, dtype=np.float64)
193
+ scale = (hi - lo) / (LEVELS - 1) if hi > lo else 1.0
194
+ q = np.clip(np.rint((np.clip(a, lo, hi) - lo) / scale), 0, LEVELS - 1).astype(np.uint8)
195
+ ch = {"enc": "u8-b64", "transform": "linear", "scale": float(scale), "offset": float(lo), "levels": LEVELS}
196
+ if valid is not None:
197
+ q = np.where(np.asarray(valid, dtype=bool), q, NODATA).astype(np.uint8)
198
+ ch["nodata"] = True
199
+ if unit:
200
+ ch["unit"] = unit
201
+ ch["data"] = encode_u8(q)
202
+ return ch
203
+
204
+
205
+ def channel_values(ch: dict) -> np.ndarray:
206
+ """The values a channel stands for (NaN where it says "no data"): what the viewer shows, for the tests."""
207
+ q = decode_u8(ch["data"]).astype(np.float64)
208
+ v = np.expm1(q * ch["scale"]) if ch.get("transform") == "log1p" else ch.get("offset", 0.0) + q * ch["scale"]
209
+ if ch.get("nodata"):
210
+ v = np.where(q == NODATA, np.nan, v)
211
+ return v