opengwasdb 0.11.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (182) hide show
  1. opengwasdb/__init__.py +8 -0
  2. opengwasdb/admission/__init__.py +47 -0
  3. opengwasdb/admission/cells.py +83 -0
  4. opengwasdb/admission/declared.py +86 -0
  5. opengwasdb/admission/duplicates.py +36 -0
  6. opengwasdb/admission/gate.py +175 -0
  7. opengwasdb/admission/record.py +229 -0
  8. opengwasdb/admission/rules.py +409 -0
  9. opengwasdb/ancestry/__init__.py +52 -0
  10. opengwasdb/ancestry/calibrate.py +234 -0
  11. opengwasdb/ancestry/catalogue.py +171 -0
  12. opengwasdb/ancestry/mixture.py +299 -0
  13. opengwasdb/ancestry/pipeline.py +232 -0
  14. opengwasdb/ancestry/reference.py +133 -0
  15. opengwasdb/ancestry/routing.py +184 -0
  16. opengwasdb/ancestry/subset.py +219 -0
  17. opengwasdb/associations/__init__.py +47 -0
  18. opengwasdb/associations/cells.py +185 -0
  19. opengwasdb/associations/csr.py +383 -0
  20. opengwasdb/associations/dense.py +317 -0
  21. opengwasdb/associations/record.py +585 -0
  22. opengwasdb/associations/subset.py +64 -0
  23. opengwasdb/build/__init__.py +7 -0
  24. opengwasdb/build/complete/__init__.py +2 -0
  25. opengwasdb/build/eaf_orientation.py +1067 -0
  26. opengwasdb/build/effect_consistency.py +187 -0
  27. opengwasdb/build/hash_sample.py +62 -0
  28. opengwasdb/build/info_score_filter.py +146 -0
  29. opengwasdb/build/liftover.py +466 -0
  30. opengwasdb/build/observed/__init__.py +96 -0
  31. opengwasdb/build/ordered_pool.py +228 -0
  32. opengwasdb/build/phenotype_sd.py +196 -0
  33. opengwasdb/build/phenotype_sd_pipeline.py +367 -0
  34. opengwasdb/build/resolve.py +1182 -0
  35. opengwasdb/build/resolve_besd.py +204 -0
  36. opengwasdb/build/resolve_manifest.py +1696 -0
  37. opengwasdb/cli/__init__.py +2 -0
  38. opengwasdb/cli/main.py +2303 -0
  39. opengwasdb/completion/__init__.py +1 -0
  40. opengwasdb/completion/ancestry_filter.py +133 -0
  41. opengwasdb/completion/block.py +356 -0
  42. opengwasdb/completion/checkpoint.py +224 -0
  43. opengwasdb/completion/fill_shards.py +422 -0
  44. opengwasdb/completion/impute.py +215 -0
  45. opengwasdb/completion/ld_panel.py +902 -0
  46. opengwasdb/completion/manifest.py +107 -0
  47. opengwasdb/completion/parallel.py +58 -0
  48. opengwasdb/completion/reference_eaf.py +78 -0
  49. opengwasdb/completion/regions.py +241 -0
  50. opengwasdb/completion/schema.py +76 -0
  51. opengwasdb/encoding/__init__.py +192 -0
  52. opengwasdb/encoding/association_codec.py +533 -0
  53. opengwasdb/encoding/banding.py +73 -0
  54. opengwasdb/encoding/codec.py +1608 -0
  55. opengwasdb/encoding/measure.py +446 -0
  56. opengwasdb/encoding/memory.py +187 -0
  57. opengwasdb/encoding/plan.py +617 -0
  58. opengwasdb/encoding/planes.py +1135 -0
  59. opengwasdb/encoding/se.py +1701 -0
  60. opengwasdb/encoding/timing.py +172 -0
  61. opengwasdb/index/__init__.py +17 -0
  62. opengwasdb/index/analyses.py +38 -0
  63. opengwasdb/index/sqlite.py +48 -0
  64. opengwasdb/layouts/__init__.py +2 -0
  65. opengwasdb/layouts/build_manifest.py +359 -0
  66. opengwasdb/layouts/dense/__init__.py +15 -0
  67. opengwasdb/layouts/dense/build.py +595 -0
  68. opengwasdb/layouts/dense/build_vcf.py +1870 -0
  69. opengwasdb/layouts/dense/complete.py +1686 -0
  70. opengwasdb/layouts/dense/constants.py +84 -0
  71. opengwasdb/layouts/dense/indexed_subsets.py +4513 -0
  72. opengwasdb/layouts/dense/rho.py +427 -0
  73. opengwasdb/layouts/dense/top_hits.py +491 -0
  74. opengwasdb/layouts/hybrid/__init__.py +28 -0
  75. opengwasdb/layouts/hybrid/build.py +3645 -0
  76. opengwasdb/layouts/hybrid/checkpoint.py +434 -0
  77. opengwasdb/layouts/hybrid/complete.py +924 -0
  78. opengwasdb/layouts/hybrid/key_runs.py +307 -0
  79. opengwasdb/layouts/hybrid/key_table.py +515 -0
  80. opengwasdb/layouts/hybrid/layout.py +41 -0
  81. opengwasdb/layouts/hybrid/unknown_keys.py +491 -0
  82. opengwasdb/layouts/ragged/__init__.py +30 -0
  83. opengwasdb/layouts/ragged/analyses.py +48 -0
  84. opengwasdb/layouts/ragged/besd_eaf.py +153 -0
  85. opengwasdb/layouts/ragged/besd_reader.py +317 -0
  86. opengwasdb/layouts/ragged/build_besd.py +1522 -0
  87. opengwasdb/layouts/ragged/build_ssf.py +1053 -0
  88. opengwasdb/layouts/ragged/by_variant.py +1117 -0
  89. opengwasdb/layouts/ragged/complete.py +1713 -0
  90. opengwasdb/layouts/ragged/provenance.py +38 -0
  91. opengwasdb/layouts/ragged/sample_size_table.py +213 -0
  92. opengwasdb/layouts/ragged/ssf_lift.py +267 -0
  93. opengwasdb/layouts/ragged/top_hits.py +191 -0
  94. opengwasdb/layouts/ragged/trait_positions.py +119 -0
  95. opengwasdb/layouts/ragged/zarr_csr.py +1676 -0
  96. opengwasdb/layouts/release_manifest.py +60 -0
  97. opengwasdb/model/__init__.py +78 -0
  98. opengwasdb/model/analyses.py +755 -0
  99. opengwasdb/model/enums.py +174 -0
  100. opengwasdb/model/info_score_policy.py +169 -0
  101. opengwasdb/model/maf_policy.py +96 -0
  102. opengwasdb/model/manifest.py +171 -0
  103. opengwasdb/model/manifest_columns.py +128 -0
  104. opengwasdb/model/regions.py +201 -0
  105. opengwasdb/overview.py +1477 -0
  106. opengwasdb/query/__init__.py +7 -0
  107. opengwasdb/query/facade.py +946 -0
  108. opengwasdb/query/resolve.py +142 -0
  109. opengwasdb/readers/__init__.py +84 -0
  110. opengwasdb/readers/assembly.py +100 -0
  111. opengwasdb/readers/association_tsv.py +289 -0
  112. opengwasdb/readers/besd.py +444 -0
  113. opengwasdb/readers/column_map.py +524 -0
  114. opengwasdb/readers/column_map_reader.py +522 -0
  115. opengwasdb/readers/column_maps/finngen-r13.toml +39 -0
  116. opengwasdb/readers/column_maps/gwas-ssf.toml +44 -0
  117. opengwasdb/readers/columnar.py +586 -0
  118. opengwasdb/readers/effect_source.py +470 -0
  119. opengwasdb/readers/fake.py +105 -0
  120. opengwasdb/readers/finngen.py +537 -0
  121. opengwasdb/readers/gwas_ssf.py +1211 -0
  122. opengwasdb/readers/gwas_vcf.py +680 -0
  123. opengwasdb/readers/interface.py +472 -0
  124. opengwasdb/readers/orientation.py +66 -0
  125. opengwasdb/readers/registry.py +292 -0
  126. opengwasdb/readers/rows.py +493 -0
  127. opengwasdb/readers/tabular.py +1454 -0
  128. opengwasdb/reencode.py +466 -0
  129. opengwasdb/repair.py +113 -0
  130. opengwasdb/stats.py +128 -0
  131. opengwasdb/store/__init__.py +65 -0
  132. opengwasdb/store/arrays.py +2284 -0
  133. opengwasdb/store/convert.py +1270 -0
  134. opengwasdb/store/open.py +884 -0
  135. opengwasdb/store/swap.py +77 -0
  136. opengwasdb/top_hits.py +489 -0
  137. opengwasdb/validation/__init__.py +6 -0
  138. opengwasdb/validation/eaf_audit.py +168 -0
  139. opengwasdb/validation/observed_view.py +765 -0
  140. opengwasdb/validation/observed_view_hybrid.py +697 -0
  141. opengwasdb/validation/observed_view_ragged.py +669 -0
  142. opengwasdb/validation/rule.py +558 -0
  143. opengwasdb/validation/rules/__init__.py +5 -0
  144. opengwasdb/validation/rules/analyses.py +393 -0
  145. opengwasdb/validation/rules/associations.py +336 -0
  146. opengwasdb/validation/rules/by_variant.py +654 -0
  147. opengwasdb/validation/rules/completion.py +380 -0
  148. opengwasdb/validation/rules/csr.py +103 -0
  149. opengwasdb/validation/rules/encoding.py +540 -0
  150. opengwasdb/validation/rules/hybrid.py +118 -0
  151. opengwasdb/validation/rules/layout.py +468 -0
  152. opengwasdb/validation/rules/release.py +404 -0
  153. opengwasdb/validation/rules/rho.py +118 -0
  154. opengwasdb/validation/rules/sample_size.py +128 -0
  155. opengwasdb/validation/rules/sequence.py +215 -0
  156. opengwasdb/validation/rules/side_tables.py +385 -0
  157. opengwasdb/validation/rules/subsets.py +147 -0
  158. opengwasdb/validation/rules/top_hits.py +488 -0
  159. opengwasdb/validation/rules/variants.py +507 -0
  160. opengwasdb/validation/run.py +977 -0
  161. opengwasdb/validation/scans.py +1080 -0
  162. opengwasdb/validation/validate.py +724 -0
  163. opengwasdb/variants/__init__.py +73 -0
  164. opengwasdb/variants/axis.py +725 -0
  165. opengwasdb/variants/normalise.py +503 -0
  166. opengwasdb/variants/reference.py +863 -0
  167. opengwasdb/variants/routing/__init__.py +65 -0
  168. opengwasdb/variants/routing/extract.py +739 -0
  169. opengwasdb/variants/routing/keys.py +210 -0
  170. opengwasdb/variants/routing/lift.py +297 -0
  171. opengwasdb/variants/routing/names.py +419 -0
  172. opengwasdb/variants/routing/route.py +345 -0
  173. opengwasdb/variants/routing/table.py +730 -0
  174. opengwasdb/variants/routing/union.py +747 -0
  175. opengwasdb/variants/rsid.py +149 -0
  176. opengwasdb/variants/windows.py +46 -0
  177. opengwasdb/version.py +41 -0
  178. opengwasdb-0.11.1.dist-info/METADATA +105 -0
  179. opengwasdb-0.11.1.dist-info/RECORD +182 -0
  180. opengwasdb-0.11.1.dist-info/WHEEL +4 -0
  181. opengwasdb-0.11.1.dist-info/entry_points.txt +2 -0
  182. opengwasdb-0.11.1.dist-info/licenses/LICENSE +81 -0
opengwasdb/__init__.py ADDED
@@ -0,0 +1,8 @@
1
+ """OpenGWASDB storage and query engine."""
2
+
3
+ from opengwasdb.model.manifest import StoreManifest
4
+ from opengwasdb.store.open import OpenGWASDBStore, open_store
5
+ from opengwasdb.version import __version__
6
+
7
+ __all__ = ["OpenGWASDBStore", "StoreManifest", "__version__", "open_store"]
8
+
@@ -0,0 +1,47 @@
1
+ """Association Admission: which of an Analysis's source rows a Store admits.
2
+
3
+ One rule set for every builder and the resolver. A caller opens a `Gate` per
4
+ Analysis and hands it each batch of rows before they are matched to the
5
+ Variant Index; the gate returns the rows to keep and counts the rest by
6
+ `Reason`. A build's gates come from its `AdmissionRecord`, which writes what
7
+ each Analysis examined, admitted and dropped beside the Store Release; the
8
+ resolver's come from `counting_gate`, which only counts.
9
+ `opengwasdb.admission.rules` states the rules and their precedence, and
10
+ `require_admitted` checks Admission's guarantee on cells a writer is handed
11
+ from anywhere else.
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ from opengwasdb.admission.cells import (
17
+ NotAdmitted,
18
+ admissible,
19
+ require_admitted,
20
+ )
21
+ from opengwasdb.admission.gate import Gate, RoutedBatch, counting_gate
22
+ from opengwasdb.admission.record import (
23
+ ADMISSION_RULES,
24
+ AdmissionRecord,
25
+ AdmissionSummary,
26
+ RecordIncomplete,
27
+ admission_record_for,
28
+ )
29
+ from opengwasdb.admission.rules import AdmissionCounts, AnalysisRules, Reason, SourceRows
30
+
31
+ __all__ = [
32
+ "ADMISSION_RULES",
33
+ "AdmissionCounts",
34
+ "AdmissionRecord",
35
+ "AdmissionSummary",
36
+ "AnalysisRules",
37
+ "Gate",
38
+ "NotAdmitted",
39
+ "Reason",
40
+ "RecordIncomplete",
41
+ "RoutedBatch",
42
+ "SourceRows",
43
+ "admissible",
44
+ "admission_record_for",
45
+ "counting_gate",
46
+ "require_admitted",
47
+ ]
@@ -0,0 +1,83 @@
1
+ """What Admission guarantees of every cell a Store holds, and its checks.
2
+
3
+ A present cell has a finite z and a finite, positive SE; z is missing exactly
4
+ where SE is (spec §15, `opengwasdb.associations.record.states`); a frequency is
5
+ NaN or inside (0, 1); and a cell is flagged imputed only when it is present.
6
+ A builder's rows reach a Store through a `Gate`, which admits only such
7
+ cells; completion's candidates are checked by `admissible`, and a writer
8
+ handed cells from anywhere else checks them with `require_admitted`. A
9
+ completion reads its source Store's cells through the Association Reader, whose
10
+ presence rule refuses an invalid one with `InvalidAssociations` (ADR 0070).
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ import numpy as np
16
+
17
+ from opengwasdb.associations.record import State, states
18
+
19
+ __all__ = ["NotAdmitted", "admissible", "require_admitted"]
20
+
21
+
22
+ class NotAdmitted(ValueError):
23
+ """A cell Admission would not admit reached a Store's writer."""
24
+
25
+
26
+ def admissible(z: np.ndarray, se: np.ndarray, *, narrowed_se: bool = False) -> np.ndarray:
27
+ """Which candidate cells may be stored as present: z finite, SE finite and positive.
28
+
29
+ ``narrowed_se`` admits an SE of ``+inf`` too: what a ``float16`` SE plane
30
+ holds for an admitted SE of 65,520 or more, read back or re-encoded, and
31
+ still an association (spec §15, issue #337).
32
+ """
33
+ with np.errstate(invalid="ignore"):
34
+ se_value = np.isfinite(se) | (narrowed_se & np.isposinf(se))
35
+ keep: np.ndarray = np.isfinite(z) & se_value & (se > 0)
36
+ return keep
37
+
38
+
39
+ def require_admitted(
40
+ z: np.ndarray,
41
+ se: np.ndarray,
42
+ eaf: np.ndarray | None = None,
43
+ imputed: np.ndarray | None = None,
44
+ *,
45
+ where: str,
46
+ narrowed_se: bool = False,
47
+ ) -> None:
48
+ """Refuse cells Admission would not have admitted, naming ``where`` and the first.
49
+
50
+ ``z`` and ``se`` are parallel arrays of any shape, NaN where missing;
51
+ ``eaf`` (NaN where absent) and ``imputed`` (the 0/1 mask) are optional and
52
+ parallel too. ``narrowed_se`` is `admissible`'s. Raises `NotAdmitted`
53
+ listing each broken rule's cell count.
54
+ """
55
+ z, se = np.asarray(z, dtype=np.float64), np.asarray(se, dtype=np.float64)
56
+ z_missing, se_missing = np.isnan(z), np.isnan(se)
57
+ state = states(z_missing, se_missing, None if imputed is None else np.asarray(imputed))
58
+ present = ~z_missing & ~se_missing
59
+ broken = {
60
+ "one of z and SE missing": state == State.PARTIAL,
61
+ "imputed flag on a missing cell": state == State.IMPUTED_MISSING,
62
+ "imputed flag neither 0 nor 1": state == State.FLAG_INVALID,
63
+ "present with an infinite z or an SE not finite and positive": present
64
+ & ~admissible(z, se, narrowed_se=narrowed_se),
65
+ }
66
+ if eaf is not None:
67
+ eaf = np.asarray(eaf, dtype=np.float64)
68
+ with np.errstate(invalid="ignore"):
69
+ broken["frequency outside (0, 1)"] = ~np.isnan(eaf) & ~((eaf > 0) & (eaf < 1))
70
+ _raise_if_broken(broken, where)
71
+
72
+
73
+ def _raise_if_broken(broken: dict[str, np.ndarray], where: str) -> None:
74
+ found = {rule: mask for rule, mask in broken.items() if mask.any()}
75
+ if not found:
76
+ return
77
+ first_rule, first_mask = next(iter(found.items()))
78
+ first = tuple(int(axis[0]) for axis in np.nonzero(first_mask))
79
+ counts = "; ".join(f"{rule}: {int(mask.sum()):,}" for rule, mask in found.items())
80
+ raise NotAdmitted(
81
+ f"{where}: cells Association Admission does not admit ({counts}); first "
82
+ f"{first_rule} at {first}"
83
+ )
@@ -0,0 +1,86 @@
1
+ """The filters each Analysis declared, read from its build manifest (stores #175, #176).
2
+
3
+ Every builder judges a row against the same declared INFO and MAF policies, so
4
+ they are parsed from the manifest one way: `declared_rules` reads each row's
5
+ policy columns, keyed by the row's Analysis id, and returns one
6
+ `AnalysisRules` per Analysis a builder parsed. A partial or contradictory
7
+ declaration is a manifest error naming its Analysis, never a filter silently
8
+ not applied.
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ import csv
14
+ from collections.abc import Mapping
15
+ from pathlib import Path
16
+
17
+ from opengwasdb.admission.rules import AnalysisRules
18
+ from opengwasdb.model.info_score_policy import parse_info_score_policy
19
+ from opengwasdb.model.maf_policy import parse_maf_policy
20
+ from opengwasdb.model.manifest_columns import manifest_column
21
+
22
+ __all__ = ["declared_rules"]
23
+
24
+
25
+ def declared_rules(
26
+ manifest_path: str | Path, capabilities: Mapping[str, str | None]
27
+ ) -> dict[str, AnalysisRules]:
28
+ """Each Analysis's declared rules, by Analysis id.
29
+
30
+ ``capabilities`` names the Analyses the builder parsed from the same
31
+ manifest, each with its resolved reader capability (the row's, or the
32
+ build's default), which decides whether a declared score can be read: a
33
+ numeric INFO threshold needs a GWAS-SSF source. Rows are matched by their
34
+ `analysis_id` (or legacy `trait_id`) column, so a builder's own row order
35
+ does not matter, and a manifest whose Analyses differ from the builder's
36
+ is refused.
37
+ """
38
+ with open(manifest_path, newline="", encoding="utf-8") as fh:
39
+ reader = csv.DictReader(fh, delimiter="\t")
40
+ id_column = manifest_column(reader.fieldnames or [], "analysis_id")
41
+ raw_rows = list(reader)
42
+ _refuse_repeated(manifest_path, id_column, raw_rows)
43
+ found = {raw.get(id_column or "", "") for raw in raw_rows}
44
+ if id_column is None or found != set(capabilities) or len(raw_rows) != len(capabilities):
45
+ raise ValueError(
46
+ f"manifest {manifest_path} changed while it was read: its Analyses are not "
47
+ f"the {len(capabilities)} the build parsed"
48
+ )
49
+ return {
50
+ raw[id_column]: _rules(manifest_path, raw[id_column], raw, capabilities[raw[id_column]])
51
+ for raw in raw_rows
52
+ }
53
+
54
+
55
+ def _refuse_repeated(
56
+ manifest_path: str | Path, id_column: str | None, raw_rows: list[dict[str, str]]
57
+ ) -> None:
58
+ """Refuse an Analysis a manifest names in more than one row, naming it and the rows."""
59
+ rows: dict[str, list[int]] = {}
60
+ for number, raw in enumerate(raw_rows, start=2): # the header is line 1
61
+ rows.setdefault(raw.get(id_column or "", ""), []).append(number)
62
+ for analysis_id, numbers in rows.items():
63
+ if len(numbers) > 1:
64
+ named = ", ".join(map(str, numbers[:-1])) + f" and {numbers[-1]}"
65
+ raise ValueError(
66
+ f"manifest {manifest_path}: analysis {analysis_id!r} is named by rows {named}; "
67
+ "one row each"
68
+ )
69
+
70
+
71
+ def _rules(
72
+ manifest_path: str | Path, analysis_id: str, raw: Mapping[str, str], capability: str | None
73
+ ) -> AnalysisRules:
74
+ try:
75
+ info = parse_info_score_policy(raw, reader_capability=capability)
76
+ except ValueError as exc:
77
+ raise ValueError(
78
+ f"manifest {manifest_path}: analysis {analysis_id!r} has invalid INFO policy: {exc}"
79
+ ) from exc
80
+ try:
81
+ maf = parse_maf_policy(raw)
82
+ except ValueError as exc:
83
+ raise ValueError(
84
+ f"manifest {manifest_path}: analysis {analysis_id!r} has invalid MAF policy: {exc}"
85
+ ) from exc
86
+ return AnalysisRules(analysis_id, info=info, maf=maf)
@@ -0,0 +1,36 @@
1
+ """How a gate records what Variant Routing resolved among an axis's duplicates.
2
+
3
+ Routing places each (variant, Analysis) once (`collapse_duplicates`): rows that
4
+ agree collapse to the first, and rows that disagree are all dropped. Both leave
5
+ the gate's admitted rows; the dropped ones are logged by the ALID of the
6
+ variant they reached.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ from collections.abc import Sequence
12
+ from typing import Protocol
13
+
14
+ import numpy as np
15
+
16
+ from opengwasdb.admission.gate import Gate
17
+
18
+ __all__ = ["record_duplicates"]
19
+
20
+
21
+ class PlacedDuplicates(Protocol):
22
+ """What routing reports of one axis's duplicates: `Placed`'s fields."""
23
+
24
+ @property
25
+ def dropped(self) -> np.ndarray: ...
26
+
27
+ @property
28
+ def collapsed(self) -> int: ...
29
+
30
+
31
+ def record_duplicates(gate: Gate, placed: PlacedDuplicates, alids: Sequence[str]) -> None:
32
+ """Count one axis's collapsed rows and count and log its dropped ones.
33
+
34
+ ``alids`` is that axis, so ``placed.dropped``'s indices name variants.
35
+ """
36
+ gate.duplicates([alids[i] for i in placed.dropped.tolist()], placed.collapsed)
@@ -0,0 +1,175 @@
1
+ """One Analysis's admission gate: judge each batch, keep the counts and the log.
2
+
3
+ A `Gate` is opened once per Analysis and fed its rows batch by batch. It
4
+ returns each batch's keep mask, adds what it judged to the Analysis's
5
+ `AdmissionCounts`, and keeps the source key and reason of every row it drops
6
+ for a logged reason, so the rows a caller keeps and the counts and log it
7
+ reports come from one judgement and cannot disagree (stores #176).
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ from collections import Counter
13
+ from collections.abc import Collection, Sequence
14
+ from typing import Protocol
15
+
16
+ import numpy as np
17
+
18
+ from opengwasdb.admission.rules import (
19
+ LOGGED,
20
+ AdmissionCounts,
21
+ AnalysisRules,
22
+ Reason,
23
+ SourceRows,
24
+ judge,
25
+ )
26
+ from opengwasdb.readers.interface import StreamTally
27
+
28
+ __all__ = ["Gate", "RoutedBatch", "counting_gate"]
29
+
30
+
31
+ class RoutedBatch(Protocol):
32
+ """What Variant Routing's ``admit=`` hands a gate: one batch, before matching."""
33
+
34
+ @property
35
+ def chromosome(self) -> Sequence[str]: ...
36
+
37
+ @property
38
+ def position(self) -> Sequence[int]: ...
39
+
40
+ @property
41
+ def ref(self) -> Sequence[str]: ...
42
+
43
+ @property
44
+ def alt(self) -> Sequence[str]: ...
45
+
46
+ @property
47
+ def z(self) -> np.ndarray: ...
48
+
49
+ @property
50
+ def se(self) -> np.ndarray: ...
51
+
52
+ @property
53
+ def eaf(self) -> np.ndarray: ...
54
+
55
+ @property
56
+ def imputation_score(self) -> np.ndarray: ...
57
+
58
+ @property
59
+ def imputation_status(self) -> np.ndarray: ...
60
+
61
+
62
+ class Gate:
63
+ """One Analysis's rows judged batch by batch, and what it dropped counted and logged."""
64
+
65
+ def __init__(self, rules: AnalysisRules) -> None:
66
+ self.rules = rules
67
+ self._counts = AdmissionCounts()
68
+ self._dropped: list[tuple[str, Reason]] = []
69
+
70
+ @property
71
+ def counts(self) -> AdmissionCounts:
72
+ """Everything this gate has judged so far."""
73
+ return self._counts
74
+
75
+ @property
76
+ def reads_imputation_scores(self) -> bool:
77
+ """Whether the Analysis declared a score, so its rows' scores are judged."""
78
+ return self.rules.info.imputation_score_declaration is not None
79
+
80
+ @property
81
+ def dropped(self) -> list[tuple[str, Reason]]:
82
+ """The logged drops so far, ``(source key, reason)``, in the order judged."""
83
+ return list(self._dropped)
84
+
85
+ def source_rows(self, rows: SourceRows) -> np.ndarray:
86
+ """The rows of one batch Admission keeps; the rest are counted by reason."""
87
+ verdict = judge(rows, self.rules)
88
+ positions, reasons = verdict.logged()
89
+ if len(positions):
90
+ if rows.keys_at is None:
91
+ raise ValueError(
92
+ f"Analysis {self.rules.analysis_id}: {len(positions)} row(s) were dropped "
93
+ "for a logged reason from a batch that names no source keys"
94
+ )
95
+ self._dropped.extend(zip(rows.keys_at(positions), reasons, strict=True))
96
+ self._counts += verdict.counts
97
+ return verdict.keep
98
+
99
+ def unplaced(self, reason: Reason, keys: Sequence[str]) -> None:
100
+ """Rows this gate admitted that were not placed, named by ``keys``.
101
+
102
+ They leave ``admitted`` for ``reason``, and are logged when it is a
103
+ logged reason: duplicates that disagree, and (D6) rows no axis holds.
104
+ """
105
+ if not keys:
106
+ return
107
+ self._counts += AdmissionCounts(admitted=-len(keys), dropped={reason: len(keys)})
108
+ if reason in LOGGED:
109
+ self._dropped.extend((key, reason) for key in keys)
110
+
111
+ def unplaced_off_axis(self, keys: Sequence[str], lifted_off_primary: Collection[str]) -> None:
112
+ """Rows no axis holds, named by ``keys``: `off_axis`, except those whose key
113
+ is in ``lifted_off_primary``, which a lift placed off the primary assembly
114
+ and are counted as `lift_off_primary` (#376)."""
115
+ if not lifted_off_primary:
116
+ self.unplaced(Reason.OFF_AXIS, keys)
117
+ return
118
+ self.unplaced(Reason.OFF_AXIS, [key for key in keys if key not in lifted_off_primary])
119
+ self.unplaced(Reason.LIFT_OFF_PRIMARY, [key for key in keys if key in lifted_off_primary])
120
+
121
+ def streamed(self, tally: StreamTally) -> None:
122
+ """What the Source Reader did to this Analysis's rows as it read them.
123
+
124
+ Each row it dropped because no ALID can spell its identity (#365) is
125
+ examined, dropped and logged under its reason; each reported frequency
126
+ it made absent is counted (D7).
127
+ """
128
+ rows = tally.faults.rows
129
+ if rows:
130
+ reasons = [Reason(reason) for _, reason in rows]
131
+ self._counts += AdmissionCounts(examined=len(rows), dropped=Counter(reasons))
132
+ self._dropped.extend(zip((key for key, _ in rows), reasons, strict=True))
133
+ made_absent = tally.frequencies.made_absent
134
+ if made_absent:
135
+ self._counts += AdmissionCounts(eaf_made_absent=made_absent)
136
+ if kept := tally.indel_rows_kept_by_hm_code:
137
+ self._counts += AdmissionCounts(indel_rows_kept_by_hm_code=kept)
138
+
139
+ def collapsed(self, n: int) -> None:
140
+ """``n`` admitted rows agreed with an earlier row for their place."""
141
+ if n:
142
+ self._counts += AdmissionCounts(admitted=-n, dropped={Reason.DUPLICATE_COLLAPSED: n})
143
+
144
+ def duplicates(self, conflicting: Sequence[str], collapsed: int) -> None:
145
+ """One batch of duplicates resolved: the keys of the rows dropped because
146
+ their (variant, Analysis) conflicts, and how many rows collapsed."""
147
+ self.unplaced(Reason.DUPLICATE_CONFLICTING, conflicting)
148
+ self.collapsed(collapsed)
149
+
150
+ def admit(self, batch: RoutedBatch) -> np.ndarray:
151
+ """`source_rows` for one of Variant Routing's batches: its ``admit=``.
152
+
153
+ An Analysis that declared no score was read without one, so its rows'
154
+ scores are not handed on. A logged row is named by its source
155
+ coordinate, ``chromosome:position:ref:alt`` as the reader spelt it.
156
+ """
157
+ declared = self.reads_imputation_scores
158
+ return self.source_rows(
159
+ SourceRows(
160
+ eaf=batch.eaf,
161
+ imputation_score=batch.imputation_score if declared else None,
162
+ imputation_status=batch.imputation_status if declared else None,
163
+ se=batch.se,
164
+ z=batch.z,
165
+ keys_at=lambda positions: [
166
+ f"{batch.chromosome[i]}:{batch.position[i]}:{batch.ref[i]}:{batch.alt[i]}"
167
+ for i in positions.tolist()
168
+ ],
169
+ )
170
+ )
171
+
172
+
173
+ def counting_gate(rules: AnalysisRules) -> Gate:
174
+ """A gate that writes no record and keeps only its own counts: the resolver's."""
175
+ return Gate(rules)
@@ -0,0 +1,229 @@
1
+ """A build's Admission record: what each Analysis examined, admitted and dropped.
2
+
3
+ Written beside the Store Release, never inside it, so a Store's bytes and its
4
+ `format_version` do not depend on it:
5
+
6
+ <directory>/admission.json the summary, written last
7
+ <directory>/parts/<index>.json one Analysis's counts, while building
8
+ <directory>/dropped/<index>.tsv.gz one Analysis's logged drops, if any:
9
+ `source_key`, `reason`, in source order
10
+ <directory>/parts/<index>.<stage>.json
11
+ <directory>/dropped/<index>.<stage>.tsv.gz
12
+ what a later stage dropped of the rows
13
+ the gate admitted (the Hybrid's fold)
14
+
15
+ A part is written by whichever process judged the Analysis -- a forked Pass 2
16
+ worker, typically -- temp-then-rename, so a part is either whole or absent.
17
+ `AdmissionRecord.finish` reads one part per Analysis, refuses a record that
18
+ lacks one, adds each stage's part to it, and writes `admission.json`. A
19
+ resumed build keeps the parts its first run wrote; any other build discards
20
+ whatever record it finds, because
21
+ the Store it described is the one this build replaces (the Store's own
22
+ destination check decides whether it may).
23
+ """
24
+
25
+ from __future__ import annotations
26
+
27
+ import gzip
28
+ import io
29
+ import json
30
+ import os
31
+ import shutil
32
+ from collections.abc import Iterator, Mapping, Sequence
33
+ from contextlib import contextmanager
34
+ from dataclasses import dataclass
35
+ from pathlib import Path
36
+ from typing import Any
37
+
38
+ from opengwasdb.admission.gate import Gate
39
+ from opengwasdb.admission.rules import AdmissionCounts, AnalysisRules, Reason
40
+
41
+ __all__ = [
42
+ "ADMISSION_RULES",
43
+ "AdmissionRecord",
44
+ "AdmissionSummary",
45
+ "RecordIncomplete",
46
+ "admission_record_for",
47
+ ]
48
+
49
+ #: The rule set's version, recorded in every summary; each change to what a rule
50
+ #: admits moves it.
51
+ ADMISSION_RULES = "admission/14"
52
+
53
+ SUMMARY = "admission.json"
54
+ #: The Analyses whose Trait position a lifting build could not lift (#433); written
55
+ #: by the build beside the summary, and cleared with the rest of a record.
56
+ UNLIFTED_TRAIT_POSITIONS = "trait_positions_unlifted.tsv"
57
+ _PARTS = "parts"
58
+ _DROPPED = "dropped"
59
+
60
+
61
+ class RecordIncomplete(RuntimeError):
62
+ """A record was finished without a part for every Analysis it was opened for."""
63
+
64
+
65
+ def admission_record_for(destination: str | Path, named: str | Path | None = None) -> Path:
66
+ """Where a build for ``destination`` writes its record: ``named``, or beside the Store."""
67
+ if named is not None:
68
+ return Path(named)
69
+ path = Path(destination)
70
+ return path.with_name(f"{path.name}.admission")
71
+
72
+
73
+ @dataclass(frozen=True)
74
+ class AdmissionSummary:
75
+ """A finished record's counts, by Analysis, in the order it was opened with."""
76
+
77
+ counts: Mapping[str, AdmissionCounts]
78
+
79
+
80
+ class AdmissionRecord:
81
+ """One build's record, opened for its Analyses in Manifest Row order."""
82
+
83
+ def __init__(self, directory: Path, analyses: Sequence[str], command: str) -> None:
84
+ if len(set(analyses)) != len(analyses):
85
+ raise ValueError("an Admission record's Analyses must be distinct")
86
+ self.directory = directory
87
+ self.analyses = tuple(analyses)
88
+ self.command = command
89
+
90
+ @classmethod
91
+ def open(
92
+ cls,
93
+ directory: str | Path,
94
+ analyses: Sequence[str],
95
+ *,
96
+ command: str,
97
+ resume: bool = False,
98
+ ) -> AdmissionRecord:
99
+ """Open a record for a build; ``resume`` keeps the parts already written,
100
+ and anything else starts the record afresh."""
101
+ record = cls(Path(directory), analyses, command)
102
+ if not resume:
103
+ record._clear()
104
+ (record.directory / _PARTS).mkdir(parents=True, exist_ok=True)
105
+ return record
106
+
107
+ @contextmanager
108
+ def gate(self, rules: AnalysisRules) -> Iterator[Gate]:
109
+ """One Analysis's gate; its log and part are written when the block exits
110
+ cleanly, the part last, so a part's presence means its log is whole."""
111
+ index = self.analyses.index(rules.analysis_id)
112
+ gate = Gate(rules)
113
+ yield gate
114
+ self._write_log(index, gate.dropped)
115
+ self._write_part(index, gate.counts)
116
+
117
+ @contextmanager
118
+ def stage(self, analysis_id: str, name: str) -> Iterator[Gate]:
119
+ """A later stage's gate for one Analysis: the rows the Analysis's gate
120
+ admitted that ``name`` then dropped, through `Gate.unplaced` and
121
+ `Gate.duplicates`.
122
+
123
+ Written as that stage's own part and log when the block exits cleanly,
124
+ replacing whatever an interrupted run of the stage wrote, so a stage
125
+ run again for an Analysis counts its rows once.
126
+ """
127
+ if not name or not name.isidentifier():
128
+ raise ValueError(f"an Admission stage is named by an identifier, not {name!r}")
129
+ index = self.analyses.index(analysis_id)
130
+ gate = Gate(AnalysisRules(analysis_id))
131
+ yield gate
132
+ self._write_log(index, gate.dropped, name)
133
+ self._write_part(index, gate.counts, name)
134
+
135
+ def finish(self) -> AdmissionSummary:
136
+ """Read every part, write the summary, and return the counts.
137
+
138
+ Raises `RecordIncomplete` naming the Analyses that have no part, before
139
+ anything is written.
140
+ """
141
+ parts = {i: self._read_part(i) for i in range(len(self.analyses))}
142
+ missing = [self.analyses[i] for i, part in parts.items() if part is None]
143
+ if missing:
144
+ raise RecordIncomplete(
145
+ f"the Admission record at {self.directory} has no part for "
146
+ f"{len(missing)} Analysis/Analyses (first: {missing[:5]}); the build "
147
+ "that should have written them did not finish them"
148
+ )
149
+ counts = {
150
+ self.analyses[i]: sum(self._stages(i), part)
151
+ for i, part in parts.items()
152
+ if part is not None
153
+ }
154
+ _write_json(self.directory / SUMMARY, self._summary(counts))
155
+ return AdmissionSummary(counts=counts)
156
+
157
+ def _summary(self, counts: Mapping[str, AdmissionCounts]) -> dict[str, Any]:
158
+ return {
159
+ "rules": ADMISSION_RULES,
160
+ "command": self.command,
161
+ "analyses": [
162
+ {"analysis_id": analysis_id, **counts[analysis_id].as_dict()}
163
+ for analysis_id in self.analyses
164
+ ],
165
+ }
166
+
167
+ def _clear(self) -> None:
168
+ (self.directory / SUMMARY).unlink(missing_ok=True)
169
+ (self.directory / UNLIFTED_TRAIT_POSITIONS).unlink(missing_ok=True)
170
+ for name in (_PARTS, _DROPPED):
171
+ shutil.rmtree(self.directory / name, ignore_errors=True)
172
+
173
+ def _part_path(self, index: int, stage: str | None = None) -> Path:
174
+ name = f"{index}.json" if stage is None else f"{index}.{stage}.json"
175
+ return self.directory / _PARTS / name
176
+
177
+ def _stages(self, index: int) -> list[AdmissionCounts]:
178
+ """The parts later stages wrote for one Analysis, in name order."""
179
+ paths = sorted((self.directory / _PARTS).glob(f"{index}.*.json"))
180
+ return [self._read_counts(index, path) for path in paths]
181
+
182
+ def _write_log(
183
+ self, index: int, dropped: list[tuple[str, Reason]], stage: str | None = None
184
+ ) -> None:
185
+ name = f"{index}.tsv.gz" if stage is None else f"{index}.{stage}.tsv.gz"
186
+ path = self.directory / _DROPPED / name
187
+ if not dropped:
188
+ path.unlink(missing_ok=True)
189
+ return
190
+ path.parent.mkdir(exist_ok=True)
191
+ text = "source_key\treason\n" + "".join(f"{key}\t{reason}\n" for key, reason in dropped)
192
+ buffer = io.BytesIO()
193
+ # No name and a zero mtime in the header: the same drops give the same bytes.
194
+ with gzip.GzipFile(filename="", mode="wb", fileobj=buffer, mtime=0) as fh:
195
+ fh.write(text.encode("utf-8"))
196
+ _write_bytes(path, buffer.getvalue())
197
+
198
+ def _write_part(self, index: int, counts: AdmissionCounts, stage: str | None = None) -> None:
199
+ _write_json(
200
+ self._part_path(index, stage),
201
+ {"analysis_id": self.analyses[index], **counts.as_dict()},
202
+ )
203
+
204
+ def _read_part(self, index: int) -> AdmissionCounts | None:
205
+ path = self._part_path(index)
206
+ if not path.exists():
207
+ return None
208
+ return self._read_counts(index, path)
209
+
210
+ def _read_counts(self, index: int, path: Path) -> AdmissionCounts:
211
+ data = json.loads(path.read_text(encoding="utf-8"))
212
+ if data.get("analysis_id") != self.analyses[index]:
213
+ raise RecordIncomplete(
214
+ f"{path} records Analysis {data.get('analysis_id')!r}, not "
215
+ f"{self.analyses[index]!r}: the record was written for another manifest"
216
+ )
217
+ return AdmissionCounts.from_dict(data)
218
+
219
+
220
+ def _write_json(path: Path, payload: Mapping[str, Any]) -> None:
221
+ """Write ``payload`` whole or not at all."""
222
+ _write_bytes(path, (json.dumps(payload, indent=1) + "\n").encode("utf-8"))
223
+
224
+
225
+ def _write_bytes(path: Path, data: bytes) -> None:
226
+ """Write ``data`` whole or not at all: a temp file renamed into place."""
227
+ tmp = path.with_name(f".{path.name}.{os.getpid()}.tmp")
228
+ tmp.write_bytes(data)
229
+ tmp.replace(path)