opengwasdb 0.11.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- opengwasdb/__init__.py +8 -0
- opengwasdb/admission/__init__.py +47 -0
- opengwasdb/admission/cells.py +83 -0
- opengwasdb/admission/declared.py +86 -0
- opengwasdb/admission/duplicates.py +36 -0
- opengwasdb/admission/gate.py +175 -0
- opengwasdb/admission/record.py +229 -0
- opengwasdb/admission/rules.py +409 -0
- opengwasdb/ancestry/__init__.py +52 -0
- opengwasdb/ancestry/calibrate.py +234 -0
- opengwasdb/ancestry/catalogue.py +171 -0
- opengwasdb/ancestry/mixture.py +299 -0
- opengwasdb/ancestry/pipeline.py +232 -0
- opengwasdb/ancestry/reference.py +133 -0
- opengwasdb/ancestry/routing.py +184 -0
- opengwasdb/ancestry/subset.py +219 -0
- opengwasdb/associations/__init__.py +47 -0
- opengwasdb/associations/cells.py +185 -0
- opengwasdb/associations/csr.py +383 -0
- opengwasdb/associations/dense.py +317 -0
- opengwasdb/associations/record.py +585 -0
- opengwasdb/associations/subset.py +64 -0
- opengwasdb/build/__init__.py +7 -0
- opengwasdb/build/complete/__init__.py +2 -0
- opengwasdb/build/eaf_orientation.py +1067 -0
- opengwasdb/build/effect_consistency.py +187 -0
- opengwasdb/build/hash_sample.py +62 -0
- opengwasdb/build/info_score_filter.py +146 -0
- opengwasdb/build/liftover.py +466 -0
- opengwasdb/build/observed/__init__.py +96 -0
- opengwasdb/build/ordered_pool.py +228 -0
- opengwasdb/build/phenotype_sd.py +196 -0
- opengwasdb/build/phenotype_sd_pipeline.py +367 -0
- opengwasdb/build/resolve.py +1182 -0
- opengwasdb/build/resolve_besd.py +204 -0
- opengwasdb/build/resolve_manifest.py +1696 -0
- opengwasdb/cli/__init__.py +2 -0
- opengwasdb/cli/main.py +2303 -0
- opengwasdb/completion/__init__.py +1 -0
- opengwasdb/completion/ancestry_filter.py +133 -0
- opengwasdb/completion/block.py +356 -0
- opengwasdb/completion/checkpoint.py +224 -0
- opengwasdb/completion/fill_shards.py +422 -0
- opengwasdb/completion/impute.py +215 -0
- opengwasdb/completion/ld_panel.py +902 -0
- opengwasdb/completion/manifest.py +107 -0
- opengwasdb/completion/parallel.py +58 -0
- opengwasdb/completion/reference_eaf.py +78 -0
- opengwasdb/completion/regions.py +241 -0
- opengwasdb/completion/schema.py +76 -0
- opengwasdb/encoding/__init__.py +192 -0
- opengwasdb/encoding/association_codec.py +533 -0
- opengwasdb/encoding/banding.py +73 -0
- opengwasdb/encoding/codec.py +1608 -0
- opengwasdb/encoding/measure.py +446 -0
- opengwasdb/encoding/memory.py +187 -0
- opengwasdb/encoding/plan.py +617 -0
- opengwasdb/encoding/planes.py +1135 -0
- opengwasdb/encoding/se.py +1701 -0
- opengwasdb/encoding/timing.py +172 -0
- opengwasdb/index/__init__.py +17 -0
- opengwasdb/index/analyses.py +38 -0
- opengwasdb/index/sqlite.py +48 -0
- opengwasdb/layouts/__init__.py +2 -0
- opengwasdb/layouts/build_manifest.py +359 -0
- opengwasdb/layouts/dense/__init__.py +15 -0
- opengwasdb/layouts/dense/build.py +595 -0
- opengwasdb/layouts/dense/build_vcf.py +1870 -0
- opengwasdb/layouts/dense/complete.py +1686 -0
- opengwasdb/layouts/dense/constants.py +84 -0
- opengwasdb/layouts/dense/indexed_subsets.py +4513 -0
- opengwasdb/layouts/dense/rho.py +427 -0
- opengwasdb/layouts/dense/top_hits.py +491 -0
- opengwasdb/layouts/hybrid/__init__.py +28 -0
- opengwasdb/layouts/hybrid/build.py +3645 -0
- opengwasdb/layouts/hybrid/checkpoint.py +434 -0
- opengwasdb/layouts/hybrid/complete.py +924 -0
- opengwasdb/layouts/hybrid/key_runs.py +307 -0
- opengwasdb/layouts/hybrid/key_table.py +515 -0
- opengwasdb/layouts/hybrid/layout.py +41 -0
- opengwasdb/layouts/hybrid/unknown_keys.py +491 -0
- opengwasdb/layouts/ragged/__init__.py +30 -0
- opengwasdb/layouts/ragged/analyses.py +48 -0
- opengwasdb/layouts/ragged/besd_eaf.py +153 -0
- opengwasdb/layouts/ragged/besd_reader.py +317 -0
- opengwasdb/layouts/ragged/build_besd.py +1522 -0
- opengwasdb/layouts/ragged/build_ssf.py +1053 -0
- opengwasdb/layouts/ragged/by_variant.py +1117 -0
- opengwasdb/layouts/ragged/complete.py +1713 -0
- opengwasdb/layouts/ragged/provenance.py +38 -0
- opengwasdb/layouts/ragged/sample_size_table.py +213 -0
- opengwasdb/layouts/ragged/ssf_lift.py +267 -0
- opengwasdb/layouts/ragged/top_hits.py +191 -0
- opengwasdb/layouts/ragged/trait_positions.py +119 -0
- opengwasdb/layouts/ragged/zarr_csr.py +1676 -0
- opengwasdb/layouts/release_manifest.py +60 -0
- opengwasdb/model/__init__.py +78 -0
- opengwasdb/model/analyses.py +755 -0
- opengwasdb/model/enums.py +174 -0
- opengwasdb/model/info_score_policy.py +169 -0
- opengwasdb/model/maf_policy.py +96 -0
- opengwasdb/model/manifest.py +171 -0
- opengwasdb/model/manifest_columns.py +128 -0
- opengwasdb/model/regions.py +201 -0
- opengwasdb/overview.py +1477 -0
- opengwasdb/query/__init__.py +7 -0
- opengwasdb/query/facade.py +946 -0
- opengwasdb/query/resolve.py +142 -0
- opengwasdb/readers/__init__.py +84 -0
- opengwasdb/readers/assembly.py +100 -0
- opengwasdb/readers/association_tsv.py +289 -0
- opengwasdb/readers/besd.py +444 -0
- opengwasdb/readers/column_map.py +524 -0
- opengwasdb/readers/column_map_reader.py +522 -0
- opengwasdb/readers/column_maps/finngen-r13.toml +39 -0
- opengwasdb/readers/column_maps/gwas-ssf.toml +44 -0
- opengwasdb/readers/columnar.py +586 -0
- opengwasdb/readers/effect_source.py +470 -0
- opengwasdb/readers/fake.py +105 -0
- opengwasdb/readers/finngen.py +537 -0
- opengwasdb/readers/gwas_ssf.py +1211 -0
- opengwasdb/readers/gwas_vcf.py +680 -0
- opengwasdb/readers/interface.py +472 -0
- opengwasdb/readers/orientation.py +66 -0
- opengwasdb/readers/registry.py +292 -0
- opengwasdb/readers/rows.py +493 -0
- opengwasdb/readers/tabular.py +1454 -0
- opengwasdb/reencode.py +466 -0
- opengwasdb/repair.py +113 -0
- opengwasdb/stats.py +128 -0
- opengwasdb/store/__init__.py +65 -0
- opengwasdb/store/arrays.py +2284 -0
- opengwasdb/store/convert.py +1270 -0
- opengwasdb/store/open.py +884 -0
- opengwasdb/store/swap.py +77 -0
- opengwasdb/top_hits.py +489 -0
- opengwasdb/validation/__init__.py +6 -0
- opengwasdb/validation/eaf_audit.py +168 -0
- opengwasdb/validation/observed_view.py +765 -0
- opengwasdb/validation/observed_view_hybrid.py +697 -0
- opengwasdb/validation/observed_view_ragged.py +669 -0
- opengwasdb/validation/rule.py +558 -0
- opengwasdb/validation/rules/__init__.py +5 -0
- opengwasdb/validation/rules/analyses.py +393 -0
- opengwasdb/validation/rules/associations.py +336 -0
- opengwasdb/validation/rules/by_variant.py +654 -0
- opengwasdb/validation/rules/completion.py +380 -0
- opengwasdb/validation/rules/csr.py +103 -0
- opengwasdb/validation/rules/encoding.py +540 -0
- opengwasdb/validation/rules/hybrid.py +118 -0
- opengwasdb/validation/rules/layout.py +468 -0
- opengwasdb/validation/rules/release.py +404 -0
- opengwasdb/validation/rules/rho.py +118 -0
- opengwasdb/validation/rules/sample_size.py +128 -0
- opengwasdb/validation/rules/sequence.py +215 -0
- opengwasdb/validation/rules/side_tables.py +385 -0
- opengwasdb/validation/rules/subsets.py +147 -0
- opengwasdb/validation/rules/top_hits.py +488 -0
- opengwasdb/validation/rules/variants.py +507 -0
- opengwasdb/validation/run.py +977 -0
- opengwasdb/validation/scans.py +1080 -0
- opengwasdb/validation/validate.py +724 -0
- opengwasdb/variants/__init__.py +73 -0
- opengwasdb/variants/axis.py +725 -0
- opengwasdb/variants/normalise.py +503 -0
- opengwasdb/variants/reference.py +863 -0
- opengwasdb/variants/routing/__init__.py +65 -0
- opengwasdb/variants/routing/extract.py +739 -0
- opengwasdb/variants/routing/keys.py +210 -0
- opengwasdb/variants/routing/lift.py +297 -0
- opengwasdb/variants/routing/names.py +419 -0
- opengwasdb/variants/routing/route.py +345 -0
- opengwasdb/variants/routing/table.py +730 -0
- opengwasdb/variants/routing/union.py +747 -0
- opengwasdb/variants/rsid.py +149 -0
- opengwasdb/variants/windows.py +46 -0
- opengwasdb/version.py +41 -0
- opengwasdb-0.11.1.dist-info/METADATA +105 -0
- opengwasdb-0.11.1.dist-info/RECORD +182 -0
- opengwasdb-0.11.1.dist-info/WHEEL +4 -0
- opengwasdb-0.11.1.dist-info/entry_points.txt +2 -0
- opengwasdb-0.11.1.dist-info/licenses/LICENSE +81 -0
opengwasdb/__init__.py
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
"""OpenGWASDB storage and query engine."""
|
|
2
|
+
|
|
3
|
+
from opengwasdb.model.manifest import StoreManifest
|
|
4
|
+
from opengwasdb.store.open import OpenGWASDBStore, open_store
|
|
5
|
+
from opengwasdb.version import __version__
|
|
6
|
+
|
|
7
|
+
__all__ = ["OpenGWASDBStore", "StoreManifest", "__version__", "open_store"]
|
|
8
|
+
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
"""Association Admission: which of an Analysis's source rows a Store admits.
|
|
2
|
+
|
|
3
|
+
One rule set for every builder and the resolver. A caller opens a `Gate` per
|
|
4
|
+
Analysis and hands it each batch of rows before they are matched to the
|
|
5
|
+
Variant Index; the gate returns the rows to keep and counts the rest by
|
|
6
|
+
`Reason`. A build's gates come from its `AdmissionRecord`, which writes what
|
|
7
|
+
each Analysis examined, admitted and dropped beside the Store Release; the
|
|
8
|
+
resolver's come from `counting_gate`, which only counts.
|
|
9
|
+
`opengwasdb.admission.rules` states the rules and their precedence, and
|
|
10
|
+
`require_admitted` checks Admission's guarantee on cells a writer is handed
|
|
11
|
+
from anywhere else.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
from opengwasdb.admission.cells import (
|
|
17
|
+
NotAdmitted,
|
|
18
|
+
admissible,
|
|
19
|
+
require_admitted,
|
|
20
|
+
)
|
|
21
|
+
from opengwasdb.admission.gate import Gate, RoutedBatch, counting_gate
|
|
22
|
+
from opengwasdb.admission.record import (
|
|
23
|
+
ADMISSION_RULES,
|
|
24
|
+
AdmissionRecord,
|
|
25
|
+
AdmissionSummary,
|
|
26
|
+
RecordIncomplete,
|
|
27
|
+
admission_record_for,
|
|
28
|
+
)
|
|
29
|
+
from opengwasdb.admission.rules import AdmissionCounts, AnalysisRules, Reason, SourceRows
|
|
30
|
+
|
|
31
|
+
__all__ = [
|
|
32
|
+
"ADMISSION_RULES",
|
|
33
|
+
"AdmissionCounts",
|
|
34
|
+
"AdmissionRecord",
|
|
35
|
+
"AdmissionSummary",
|
|
36
|
+
"AnalysisRules",
|
|
37
|
+
"Gate",
|
|
38
|
+
"NotAdmitted",
|
|
39
|
+
"Reason",
|
|
40
|
+
"RecordIncomplete",
|
|
41
|
+
"RoutedBatch",
|
|
42
|
+
"SourceRows",
|
|
43
|
+
"admissible",
|
|
44
|
+
"admission_record_for",
|
|
45
|
+
"counting_gate",
|
|
46
|
+
"require_admitted",
|
|
47
|
+
]
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
"""What Admission guarantees of every cell a Store holds, and its checks.
|
|
2
|
+
|
|
3
|
+
A present cell has a finite z and a finite, positive SE; z is missing exactly
|
|
4
|
+
where SE is (spec §15, `opengwasdb.associations.record.states`); a frequency is
|
|
5
|
+
NaN or inside (0, 1); and a cell is flagged imputed only when it is present.
|
|
6
|
+
A builder's rows reach a Store through a `Gate`, which admits only such
|
|
7
|
+
cells; completion's candidates are checked by `admissible`, and a writer
|
|
8
|
+
handed cells from anywhere else checks them with `require_admitted`. A
|
|
9
|
+
completion reads its source Store's cells through the Association Reader, whose
|
|
10
|
+
presence rule refuses an invalid one with `InvalidAssociations` (ADR 0070).
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import numpy as np
|
|
16
|
+
|
|
17
|
+
from opengwasdb.associations.record import State, states
|
|
18
|
+
|
|
19
|
+
__all__ = ["NotAdmitted", "admissible", "require_admitted"]
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class NotAdmitted(ValueError):
|
|
23
|
+
"""A cell Admission would not admit reached a Store's writer."""
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def admissible(z: np.ndarray, se: np.ndarray, *, narrowed_se: bool = False) -> np.ndarray:
|
|
27
|
+
"""Which candidate cells may be stored as present: z finite, SE finite and positive.
|
|
28
|
+
|
|
29
|
+
``narrowed_se`` admits an SE of ``+inf`` too: what a ``float16`` SE plane
|
|
30
|
+
holds for an admitted SE of 65,520 or more, read back or re-encoded, and
|
|
31
|
+
still an association (spec §15, issue #337).
|
|
32
|
+
"""
|
|
33
|
+
with np.errstate(invalid="ignore"):
|
|
34
|
+
se_value = np.isfinite(se) | (narrowed_se & np.isposinf(se))
|
|
35
|
+
keep: np.ndarray = np.isfinite(z) & se_value & (se > 0)
|
|
36
|
+
return keep
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def require_admitted(
|
|
40
|
+
z: np.ndarray,
|
|
41
|
+
se: np.ndarray,
|
|
42
|
+
eaf: np.ndarray | None = None,
|
|
43
|
+
imputed: np.ndarray | None = None,
|
|
44
|
+
*,
|
|
45
|
+
where: str,
|
|
46
|
+
narrowed_se: bool = False,
|
|
47
|
+
) -> None:
|
|
48
|
+
"""Refuse cells Admission would not have admitted, naming ``where`` and the first.
|
|
49
|
+
|
|
50
|
+
``z`` and ``se`` are parallel arrays of any shape, NaN where missing;
|
|
51
|
+
``eaf`` (NaN where absent) and ``imputed`` (the 0/1 mask) are optional and
|
|
52
|
+
parallel too. ``narrowed_se`` is `admissible`'s. Raises `NotAdmitted`
|
|
53
|
+
listing each broken rule's cell count.
|
|
54
|
+
"""
|
|
55
|
+
z, se = np.asarray(z, dtype=np.float64), np.asarray(se, dtype=np.float64)
|
|
56
|
+
z_missing, se_missing = np.isnan(z), np.isnan(se)
|
|
57
|
+
state = states(z_missing, se_missing, None if imputed is None else np.asarray(imputed))
|
|
58
|
+
present = ~z_missing & ~se_missing
|
|
59
|
+
broken = {
|
|
60
|
+
"one of z and SE missing": state == State.PARTIAL,
|
|
61
|
+
"imputed flag on a missing cell": state == State.IMPUTED_MISSING,
|
|
62
|
+
"imputed flag neither 0 nor 1": state == State.FLAG_INVALID,
|
|
63
|
+
"present with an infinite z or an SE not finite and positive": present
|
|
64
|
+
& ~admissible(z, se, narrowed_se=narrowed_se),
|
|
65
|
+
}
|
|
66
|
+
if eaf is not None:
|
|
67
|
+
eaf = np.asarray(eaf, dtype=np.float64)
|
|
68
|
+
with np.errstate(invalid="ignore"):
|
|
69
|
+
broken["frequency outside (0, 1)"] = ~np.isnan(eaf) & ~((eaf > 0) & (eaf < 1))
|
|
70
|
+
_raise_if_broken(broken, where)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _raise_if_broken(broken: dict[str, np.ndarray], where: str) -> None:
|
|
74
|
+
found = {rule: mask for rule, mask in broken.items() if mask.any()}
|
|
75
|
+
if not found:
|
|
76
|
+
return
|
|
77
|
+
first_rule, first_mask = next(iter(found.items()))
|
|
78
|
+
first = tuple(int(axis[0]) for axis in np.nonzero(first_mask))
|
|
79
|
+
counts = "; ".join(f"{rule}: {int(mask.sum()):,}" for rule, mask in found.items())
|
|
80
|
+
raise NotAdmitted(
|
|
81
|
+
f"{where}: cells Association Admission does not admit ({counts}); first "
|
|
82
|
+
f"{first_rule} at {first}"
|
|
83
|
+
)
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
"""The filters each Analysis declared, read from its build manifest (stores #175, #176).
|
|
2
|
+
|
|
3
|
+
Every builder judges a row against the same declared INFO and MAF policies, so
|
|
4
|
+
they are parsed from the manifest one way: `declared_rules` reads each row's
|
|
5
|
+
policy columns, keyed by the row's Analysis id, and returns one
|
|
6
|
+
`AnalysisRules` per Analysis a builder parsed. A partial or contradictory
|
|
7
|
+
declaration is a manifest error naming its Analysis, never a filter silently
|
|
8
|
+
not applied.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import csv
|
|
14
|
+
from collections.abc import Mapping
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
|
|
17
|
+
from opengwasdb.admission.rules import AnalysisRules
|
|
18
|
+
from opengwasdb.model.info_score_policy import parse_info_score_policy
|
|
19
|
+
from opengwasdb.model.maf_policy import parse_maf_policy
|
|
20
|
+
from opengwasdb.model.manifest_columns import manifest_column
|
|
21
|
+
|
|
22
|
+
__all__ = ["declared_rules"]
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def declared_rules(
|
|
26
|
+
manifest_path: str | Path, capabilities: Mapping[str, str | None]
|
|
27
|
+
) -> dict[str, AnalysisRules]:
|
|
28
|
+
"""Each Analysis's declared rules, by Analysis id.
|
|
29
|
+
|
|
30
|
+
``capabilities`` names the Analyses the builder parsed from the same
|
|
31
|
+
manifest, each with its resolved reader capability (the row's, or the
|
|
32
|
+
build's default), which decides whether a declared score can be read: a
|
|
33
|
+
numeric INFO threshold needs a GWAS-SSF source. Rows are matched by their
|
|
34
|
+
`analysis_id` (or legacy `trait_id`) column, so a builder's own row order
|
|
35
|
+
does not matter, and a manifest whose Analyses differ from the builder's
|
|
36
|
+
is refused.
|
|
37
|
+
"""
|
|
38
|
+
with open(manifest_path, newline="", encoding="utf-8") as fh:
|
|
39
|
+
reader = csv.DictReader(fh, delimiter="\t")
|
|
40
|
+
id_column = manifest_column(reader.fieldnames or [], "analysis_id")
|
|
41
|
+
raw_rows = list(reader)
|
|
42
|
+
_refuse_repeated(manifest_path, id_column, raw_rows)
|
|
43
|
+
found = {raw.get(id_column or "", "") for raw in raw_rows}
|
|
44
|
+
if id_column is None or found != set(capabilities) or len(raw_rows) != len(capabilities):
|
|
45
|
+
raise ValueError(
|
|
46
|
+
f"manifest {manifest_path} changed while it was read: its Analyses are not "
|
|
47
|
+
f"the {len(capabilities)} the build parsed"
|
|
48
|
+
)
|
|
49
|
+
return {
|
|
50
|
+
raw[id_column]: _rules(manifest_path, raw[id_column], raw, capabilities[raw[id_column]])
|
|
51
|
+
for raw in raw_rows
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _refuse_repeated(
|
|
56
|
+
manifest_path: str | Path, id_column: str | None, raw_rows: list[dict[str, str]]
|
|
57
|
+
) -> None:
|
|
58
|
+
"""Refuse an Analysis a manifest names in more than one row, naming it and the rows."""
|
|
59
|
+
rows: dict[str, list[int]] = {}
|
|
60
|
+
for number, raw in enumerate(raw_rows, start=2): # the header is line 1
|
|
61
|
+
rows.setdefault(raw.get(id_column or "", ""), []).append(number)
|
|
62
|
+
for analysis_id, numbers in rows.items():
|
|
63
|
+
if len(numbers) > 1:
|
|
64
|
+
named = ", ".join(map(str, numbers[:-1])) + f" and {numbers[-1]}"
|
|
65
|
+
raise ValueError(
|
|
66
|
+
f"manifest {manifest_path}: analysis {analysis_id!r} is named by rows {named}; "
|
|
67
|
+
"one row each"
|
|
68
|
+
)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _rules(
|
|
72
|
+
manifest_path: str | Path, analysis_id: str, raw: Mapping[str, str], capability: str | None
|
|
73
|
+
) -> AnalysisRules:
|
|
74
|
+
try:
|
|
75
|
+
info = parse_info_score_policy(raw, reader_capability=capability)
|
|
76
|
+
except ValueError as exc:
|
|
77
|
+
raise ValueError(
|
|
78
|
+
f"manifest {manifest_path}: analysis {analysis_id!r} has invalid INFO policy: {exc}"
|
|
79
|
+
) from exc
|
|
80
|
+
try:
|
|
81
|
+
maf = parse_maf_policy(raw)
|
|
82
|
+
except ValueError as exc:
|
|
83
|
+
raise ValueError(
|
|
84
|
+
f"manifest {manifest_path}: analysis {analysis_id!r} has invalid MAF policy: {exc}"
|
|
85
|
+
) from exc
|
|
86
|
+
return AnalysisRules(analysis_id, info=info, maf=maf)
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
"""How a gate records what Variant Routing resolved among an axis's duplicates.
|
|
2
|
+
|
|
3
|
+
Routing places each (variant, Analysis) once (`collapse_duplicates`): rows that
|
|
4
|
+
agree collapse to the first, and rows that disagree are all dropped. Both leave
|
|
5
|
+
the gate's admitted rows; the dropped ones are logged by the ALID of the
|
|
6
|
+
variant they reached.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from collections.abc import Sequence
|
|
12
|
+
from typing import Protocol
|
|
13
|
+
|
|
14
|
+
import numpy as np
|
|
15
|
+
|
|
16
|
+
from opengwasdb.admission.gate import Gate
|
|
17
|
+
|
|
18
|
+
__all__ = ["record_duplicates"]
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class PlacedDuplicates(Protocol):
|
|
22
|
+
"""What routing reports of one axis's duplicates: `Placed`'s fields."""
|
|
23
|
+
|
|
24
|
+
@property
|
|
25
|
+
def dropped(self) -> np.ndarray: ...
|
|
26
|
+
|
|
27
|
+
@property
|
|
28
|
+
def collapsed(self) -> int: ...
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def record_duplicates(gate: Gate, placed: PlacedDuplicates, alids: Sequence[str]) -> None:
|
|
32
|
+
"""Count one axis's collapsed rows and count and log its dropped ones.
|
|
33
|
+
|
|
34
|
+
``alids`` is that axis, so ``placed.dropped``'s indices name variants.
|
|
35
|
+
"""
|
|
36
|
+
gate.duplicates([alids[i] for i in placed.dropped.tolist()], placed.collapsed)
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
"""One Analysis's admission gate: judge each batch, keep the counts and the log.
|
|
2
|
+
|
|
3
|
+
A `Gate` is opened once per Analysis and fed its rows batch by batch. It
|
|
4
|
+
returns each batch's keep mask, adds what it judged to the Analysis's
|
|
5
|
+
`AdmissionCounts`, and keeps the source key and reason of every row it drops
|
|
6
|
+
for a logged reason, so the rows a caller keeps and the counts and log it
|
|
7
|
+
reports come from one judgement and cannot disagree (stores #176).
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
from collections import Counter
|
|
13
|
+
from collections.abc import Collection, Sequence
|
|
14
|
+
from typing import Protocol
|
|
15
|
+
|
|
16
|
+
import numpy as np
|
|
17
|
+
|
|
18
|
+
from opengwasdb.admission.rules import (
|
|
19
|
+
LOGGED,
|
|
20
|
+
AdmissionCounts,
|
|
21
|
+
AnalysisRules,
|
|
22
|
+
Reason,
|
|
23
|
+
SourceRows,
|
|
24
|
+
judge,
|
|
25
|
+
)
|
|
26
|
+
from opengwasdb.readers.interface import StreamTally
|
|
27
|
+
|
|
28
|
+
__all__ = ["Gate", "RoutedBatch", "counting_gate"]
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class RoutedBatch(Protocol):
|
|
32
|
+
"""What Variant Routing's ``admit=`` hands a gate: one batch, before matching."""
|
|
33
|
+
|
|
34
|
+
@property
|
|
35
|
+
def chromosome(self) -> Sequence[str]: ...
|
|
36
|
+
|
|
37
|
+
@property
|
|
38
|
+
def position(self) -> Sequence[int]: ...
|
|
39
|
+
|
|
40
|
+
@property
|
|
41
|
+
def ref(self) -> Sequence[str]: ...
|
|
42
|
+
|
|
43
|
+
@property
|
|
44
|
+
def alt(self) -> Sequence[str]: ...
|
|
45
|
+
|
|
46
|
+
@property
|
|
47
|
+
def z(self) -> np.ndarray: ...
|
|
48
|
+
|
|
49
|
+
@property
|
|
50
|
+
def se(self) -> np.ndarray: ...
|
|
51
|
+
|
|
52
|
+
@property
|
|
53
|
+
def eaf(self) -> np.ndarray: ...
|
|
54
|
+
|
|
55
|
+
@property
|
|
56
|
+
def imputation_score(self) -> np.ndarray: ...
|
|
57
|
+
|
|
58
|
+
@property
|
|
59
|
+
def imputation_status(self) -> np.ndarray: ...
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
class Gate:
|
|
63
|
+
"""One Analysis's rows judged batch by batch, and what it dropped counted and logged."""
|
|
64
|
+
|
|
65
|
+
def __init__(self, rules: AnalysisRules) -> None:
|
|
66
|
+
self.rules = rules
|
|
67
|
+
self._counts = AdmissionCounts()
|
|
68
|
+
self._dropped: list[tuple[str, Reason]] = []
|
|
69
|
+
|
|
70
|
+
@property
|
|
71
|
+
def counts(self) -> AdmissionCounts:
|
|
72
|
+
"""Everything this gate has judged so far."""
|
|
73
|
+
return self._counts
|
|
74
|
+
|
|
75
|
+
@property
|
|
76
|
+
def reads_imputation_scores(self) -> bool:
|
|
77
|
+
"""Whether the Analysis declared a score, so its rows' scores are judged."""
|
|
78
|
+
return self.rules.info.imputation_score_declaration is not None
|
|
79
|
+
|
|
80
|
+
@property
|
|
81
|
+
def dropped(self) -> list[tuple[str, Reason]]:
|
|
82
|
+
"""The logged drops so far, ``(source key, reason)``, in the order judged."""
|
|
83
|
+
return list(self._dropped)
|
|
84
|
+
|
|
85
|
+
def source_rows(self, rows: SourceRows) -> np.ndarray:
|
|
86
|
+
"""The rows of one batch Admission keeps; the rest are counted by reason."""
|
|
87
|
+
verdict = judge(rows, self.rules)
|
|
88
|
+
positions, reasons = verdict.logged()
|
|
89
|
+
if len(positions):
|
|
90
|
+
if rows.keys_at is None:
|
|
91
|
+
raise ValueError(
|
|
92
|
+
f"Analysis {self.rules.analysis_id}: {len(positions)} row(s) were dropped "
|
|
93
|
+
"for a logged reason from a batch that names no source keys"
|
|
94
|
+
)
|
|
95
|
+
self._dropped.extend(zip(rows.keys_at(positions), reasons, strict=True))
|
|
96
|
+
self._counts += verdict.counts
|
|
97
|
+
return verdict.keep
|
|
98
|
+
|
|
99
|
+
def unplaced(self, reason: Reason, keys: Sequence[str]) -> None:
|
|
100
|
+
"""Rows this gate admitted that were not placed, named by ``keys``.
|
|
101
|
+
|
|
102
|
+
They leave ``admitted`` for ``reason``, and are logged when it is a
|
|
103
|
+
logged reason: duplicates that disagree, and (D6) rows no axis holds.
|
|
104
|
+
"""
|
|
105
|
+
if not keys:
|
|
106
|
+
return
|
|
107
|
+
self._counts += AdmissionCounts(admitted=-len(keys), dropped={reason: len(keys)})
|
|
108
|
+
if reason in LOGGED:
|
|
109
|
+
self._dropped.extend((key, reason) for key in keys)
|
|
110
|
+
|
|
111
|
+
def unplaced_off_axis(self, keys: Sequence[str], lifted_off_primary: Collection[str]) -> None:
|
|
112
|
+
"""Rows no axis holds, named by ``keys``: `off_axis`, except those whose key
|
|
113
|
+
is in ``lifted_off_primary``, which a lift placed off the primary assembly
|
|
114
|
+
and are counted as `lift_off_primary` (#376)."""
|
|
115
|
+
if not lifted_off_primary:
|
|
116
|
+
self.unplaced(Reason.OFF_AXIS, keys)
|
|
117
|
+
return
|
|
118
|
+
self.unplaced(Reason.OFF_AXIS, [key for key in keys if key not in lifted_off_primary])
|
|
119
|
+
self.unplaced(Reason.LIFT_OFF_PRIMARY, [key for key in keys if key in lifted_off_primary])
|
|
120
|
+
|
|
121
|
+
def streamed(self, tally: StreamTally) -> None:
|
|
122
|
+
"""What the Source Reader did to this Analysis's rows as it read them.
|
|
123
|
+
|
|
124
|
+
Each row it dropped because no ALID can spell its identity (#365) is
|
|
125
|
+
examined, dropped and logged under its reason; each reported frequency
|
|
126
|
+
it made absent is counted (D7).
|
|
127
|
+
"""
|
|
128
|
+
rows = tally.faults.rows
|
|
129
|
+
if rows:
|
|
130
|
+
reasons = [Reason(reason) for _, reason in rows]
|
|
131
|
+
self._counts += AdmissionCounts(examined=len(rows), dropped=Counter(reasons))
|
|
132
|
+
self._dropped.extend(zip((key for key, _ in rows), reasons, strict=True))
|
|
133
|
+
made_absent = tally.frequencies.made_absent
|
|
134
|
+
if made_absent:
|
|
135
|
+
self._counts += AdmissionCounts(eaf_made_absent=made_absent)
|
|
136
|
+
if kept := tally.indel_rows_kept_by_hm_code:
|
|
137
|
+
self._counts += AdmissionCounts(indel_rows_kept_by_hm_code=kept)
|
|
138
|
+
|
|
139
|
+
def collapsed(self, n: int) -> None:
|
|
140
|
+
"""``n`` admitted rows agreed with an earlier row for their place."""
|
|
141
|
+
if n:
|
|
142
|
+
self._counts += AdmissionCounts(admitted=-n, dropped={Reason.DUPLICATE_COLLAPSED: n})
|
|
143
|
+
|
|
144
|
+
def duplicates(self, conflicting: Sequence[str], collapsed: int) -> None:
|
|
145
|
+
"""One batch of duplicates resolved: the keys of the rows dropped because
|
|
146
|
+
their (variant, Analysis) conflicts, and how many rows collapsed."""
|
|
147
|
+
self.unplaced(Reason.DUPLICATE_CONFLICTING, conflicting)
|
|
148
|
+
self.collapsed(collapsed)
|
|
149
|
+
|
|
150
|
+
def admit(self, batch: RoutedBatch) -> np.ndarray:
|
|
151
|
+
"""`source_rows` for one of Variant Routing's batches: its ``admit=``.
|
|
152
|
+
|
|
153
|
+
An Analysis that declared no score was read without one, so its rows'
|
|
154
|
+
scores are not handed on. A logged row is named by its source
|
|
155
|
+
coordinate, ``chromosome:position:ref:alt`` as the reader spelt it.
|
|
156
|
+
"""
|
|
157
|
+
declared = self.reads_imputation_scores
|
|
158
|
+
return self.source_rows(
|
|
159
|
+
SourceRows(
|
|
160
|
+
eaf=batch.eaf,
|
|
161
|
+
imputation_score=batch.imputation_score if declared else None,
|
|
162
|
+
imputation_status=batch.imputation_status if declared else None,
|
|
163
|
+
se=batch.se,
|
|
164
|
+
z=batch.z,
|
|
165
|
+
keys_at=lambda positions: [
|
|
166
|
+
f"{batch.chromosome[i]}:{batch.position[i]}:{batch.ref[i]}:{batch.alt[i]}"
|
|
167
|
+
for i in positions.tolist()
|
|
168
|
+
],
|
|
169
|
+
)
|
|
170
|
+
)
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def counting_gate(rules: AnalysisRules) -> Gate:
|
|
174
|
+
"""A gate that writes no record and keeps only its own counts: the resolver's."""
|
|
175
|
+
return Gate(rules)
|
|
@@ -0,0 +1,229 @@
|
|
|
1
|
+
"""A build's Admission record: what each Analysis examined, admitted and dropped.
|
|
2
|
+
|
|
3
|
+
Written beside the Store Release, never inside it, so a Store's bytes and its
|
|
4
|
+
`format_version` do not depend on it:
|
|
5
|
+
|
|
6
|
+
<directory>/admission.json the summary, written last
|
|
7
|
+
<directory>/parts/<index>.json one Analysis's counts, while building
|
|
8
|
+
<directory>/dropped/<index>.tsv.gz one Analysis's logged drops, if any:
|
|
9
|
+
`source_key`, `reason`, in source order
|
|
10
|
+
<directory>/parts/<index>.<stage>.json
|
|
11
|
+
<directory>/dropped/<index>.<stage>.tsv.gz
|
|
12
|
+
what a later stage dropped of the rows
|
|
13
|
+
the gate admitted (the Hybrid's fold)
|
|
14
|
+
|
|
15
|
+
A part is written by whichever process judged the Analysis -- a forked Pass 2
|
|
16
|
+
worker, typically -- temp-then-rename, so a part is either whole or absent.
|
|
17
|
+
`AdmissionRecord.finish` reads one part per Analysis, refuses a record that
|
|
18
|
+
lacks one, adds each stage's part to it, and writes `admission.json`. A
|
|
19
|
+
resumed build keeps the parts its first run wrote; any other build discards
|
|
20
|
+
whatever record it finds, because
|
|
21
|
+
the Store it described is the one this build replaces (the Store's own
|
|
22
|
+
destination check decides whether it may).
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
from __future__ import annotations
|
|
26
|
+
|
|
27
|
+
import gzip
|
|
28
|
+
import io
|
|
29
|
+
import json
|
|
30
|
+
import os
|
|
31
|
+
import shutil
|
|
32
|
+
from collections.abc import Iterator, Mapping, Sequence
|
|
33
|
+
from contextlib import contextmanager
|
|
34
|
+
from dataclasses import dataclass
|
|
35
|
+
from pathlib import Path
|
|
36
|
+
from typing import Any
|
|
37
|
+
|
|
38
|
+
from opengwasdb.admission.gate import Gate
|
|
39
|
+
from opengwasdb.admission.rules import AdmissionCounts, AnalysisRules, Reason
|
|
40
|
+
|
|
41
|
+
__all__ = [
|
|
42
|
+
"ADMISSION_RULES",
|
|
43
|
+
"AdmissionRecord",
|
|
44
|
+
"AdmissionSummary",
|
|
45
|
+
"RecordIncomplete",
|
|
46
|
+
"admission_record_for",
|
|
47
|
+
]
|
|
48
|
+
|
|
49
|
+
#: The rule set's version, recorded in every summary; each change to what a rule
|
|
50
|
+
#: admits moves it.
|
|
51
|
+
ADMISSION_RULES = "admission/14"
|
|
52
|
+
|
|
53
|
+
SUMMARY = "admission.json"
|
|
54
|
+
#: The Analyses whose Trait position a lifting build could not lift (#433); written
|
|
55
|
+
#: by the build beside the summary, and cleared with the rest of a record.
|
|
56
|
+
UNLIFTED_TRAIT_POSITIONS = "trait_positions_unlifted.tsv"
|
|
57
|
+
_PARTS = "parts"
|
|
58
|
+
_DROPPED = "dropped"
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
class RecordIncomplete(RuntimeError):
|
|
62
|
+
"""A record was finished without a part for every Analysis it was opened for."""
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def admission_record_for(destination: str | Path, named: str | Path | None = None) -> Path:
|
|
66
|
+
"""Where a build for ``destination`` writes its record: ``named``, or beside the Store."""
|
|
67
|
+
if named is not None:
|
|
68
|
+
return Path(named)
|
|
69
|
+
path = Path(destination)
|
|
70
|
+
return path.with_name(f"{path.name}.admission")
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
@dataclass(frozen=True)
|
|
74
|
+
class AdmissionSummary:
|
|
75
|
+
"""A finished record's counts, by Analysis, in the order it was opened with."""
|
|
76
|
+
|
|
77
|
+
counts: Mapping[str, AdmissionCounts]
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
class AdmissionRecord:
|
|
81
|
+
"""One build's record, opened for its Analyses in Manifest Row order."""
|
|
82
|
+
|
|
83
|
+
def __init__(self, directory: Path, analyses: Sequence[str], command: str) -> None:
|
|
84
|
+
if len(set(analyses)) != len(analyses):
|
|
85
|
+
raise ValueError("an Admission record's Analyses must be distinct")
|
|
86
|
+
self.directory = directory
|
|
87
|
+
self.analyses = tuple(analyses)
|
|
88
|
+
self.command = command
|
|
89
|
+
|
|
90
|
+
@classmethod
|
|
91
|
+
def open(
|
|
92
|
+
cls,
|
|
93
|
+
directory: str | Path,
|
|
94
|
+
analyses: Sequence[str],
|
|
95
|
+
*,
|
|
96
|
+
command: str,
|
|
97
|
+
resume: bool = False,
|
|
98
|
+
) -> AdmissionRecord:
|
|
99
|
+
"""Open a record for a build; ``resume`` keeps the parts already written,
|
|
100
|
+
and anything else starts the record afresh."""
|
|
101
|
+
record = cls(Path(directory), analyses, command)
|
|
102
|
+
if not resume:
|
|
103
|
+
record._clear()
|
|
104
|
+
(record.directory / _PARTS).mkdir(parents=True, exist_ok=True)
|
|
105
|
+
return record
|
|
106
|
+
|
|
107
|
+
@contextmanager
|
|
108
|
+
def gate(self, rules: AnalysisRules) -> Iterator[Gate]:
|
|
109
|
+
"""One Analysis's gate; its log and part are written when the block exits
|
|
110
|
+
cleanly, the part last, so a part's presence means its log is whole."""
|
|
111
|
+
index = self.analyses.index(rules.analysis_id)
|
|
112
|
+
gate = Gate(rules)
|
|
113
|
+
yield gate
|
|
114
|
+
self._write_log(index, gate.dropped)
|
|
115
|
+
self._write_part(index, gate.counts)
|
|
116
|
+
|
|
117
|
+
@contextmanager
|
|
118
|
+
def stage(self, analysis_id: str, name: str) -> Iterator[Gate]:
|
|
119
|
+
"""A later stage's gate for one Analysis: the rows the Analysis's gate
|
|
120
|
+
admitted that ``name`` then dropped, through `Gate.unplaced` and
|
|
121
|
+
`Gate.duplicates`.
|
|
122
|
+
|
|
123
|
+
Written as that stage's own part and log when the block exits cleanly,
|
|
124
|
+
replacing whatever an interrupted run of the stage wrote, so a stage
|
|
125
|
+
run again for an Analysis counts its rows once.
|
|
126
|
+
"""
|
|
127
|
+
if not name or not name.isidentifier():
|
|
128
|
+
raise ValueError(f"an Admission stage is named by an identifier, not {name!r}")
|
|
129
|
+
index = self.analyses.index(analysis_id)
|
|
130
|
+
gate = Gate(AnalysisRules(analysis_id))
|
|
131
|
+
yield gate
|
|
132
|
+
self._write_log(index, gate.dropped, name)
|
|
133
|
+
self._write_part(index, gate.counts, name)
|
|
134
|
+
|
|
135
|
+
def finish(self) -> AdmissionSummary:
|
|
136
|
+
"""Read every part, write the summary, and return the counts.
|
|
137
|
+
|
|
138
|
+
Raises `RecordIncomplete` naming the Analyses that have no part, before
|
|
139
|
+
anything is written.
|
|
140
|
+
"""
|
|
141
|
+
parts = {i: self._read_part(i) for i in range(len(self.analyses))}
|
|
142
|
+
missing = [self.analyses[i] for i, part in parts.items() if part is None]
|
|
143
|
+
if missing:
|
|
144
|
+
raise RecordIncomplete(
|
|
145
|
+
f"the Admission record at {self.directory} has no part for "
|
|
146
|
+
f"{len(missing)} Analysis/Analyses (first: {missing[:5]}); the build "
|
|
147
|
+
"that should have written them did not finish them"
|
|
148
|
+
)
|
|
149
|
+
counts = {
|
|
150
|
+
self.analyses[i]: sum(self._stages(i), part)
|
|
151
|
+
for i, part in parts.items()
|
|
152
|
+
if part is not None
|
|
153
|
+
}
|
|
154
|
+
_write_json(self.directory / SUMMARY, self._summary(counts))
|
|
155
|
+
return AdmissionSummary(counts=counts)
|
|
156
|
+
|
|
157
|
+
def _summary(self, counts: Mapping[str, AdmissionCounts]) -> dict[str, Any]:
|
|
158
|
+
return {
|
|
159
|
+
"rules": ADMISSION_RULES,
|
|
160
|
+
"command": self.command,
|
|
161
|
+
"analyses": [
|
|
162
|
+
{"analysis_id": analysis_id, **counts[analysis_id].as_dict()}
|
|
163
|
+
for analysis_id in self.analyses
|
|
164
|
+
],
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
def _clear(self) -> None:
|
|
168
|
+
(self.directory / SUMMARY).unlink(missing_ok=True)
|
|
169
|
+
(self.directory / UNLIFTED_TRAIT_POSITIONS).unlink(missing_ok=True)
|
|
170
|
+
for name in (_PARTS, _DROPPED):
|
|
171
|
+
shutil.rmtree(self.directory / name, ignore_errors=True)
|
|
172
|
+
|
|
173
|
+
def _part_path(self, index: int, stage: str | None = None) -> Path:
|
|
174
|
+
name = f"{index}.json" if stage is None else f"{index}.{stage}.json"
|
|
175
|
+
return self.directory / _PARTS / name
|
|
176
|
+
|
|
177
|
+
def _stages(self, index: int) -> list[AdmissionCounts]:
|
|
178
|
+
"""The parts later stages wrote for one Analysis, in name order."""
|
|
179
|
+
paths = sorted((self.directory / _PARTS).glob(f"{index}.*.json"))
|
|
180
|
+
return [self._read_counts(index, path) for path in paths]
|
|
181
|
+
|
|
182
|
+
def _write_log(
|
|
183
|
+
self, index: int, dropped: list[tuple[str, Reason]], stage: str | None = None
|
|
184
|
+
) -> None:
|
|
185
|
+
name = f"{index}.tsv.gz" if stage is None else f"{index}.{stage}.tsv.gz"
|
|
186
|
+
path = self.directory / _DROPPED / name
|
|
187
|
+
if not dropped:
|
|
188
|
+
path.unlink(missing_ok=True)
|
|
189
|
+
return
|
|
190
|
+
path.parent.mkdir(exist_ok=True)
|
|
191
|
+
text = "source_key\treason\n" + "".join(f"{key}\t{reason}\n" for key, reason in dropped)
|
|
192
|
+
buffer = io.BytesIO()
|
|
193
|
+
# No name and a zero mtime in the header: the same drops give the same bytes.
|
|
194
|
+
with gzip.GzipFile(filename="", mode="wb", fileobj=buffer, mtime=0) as fh:
|
|
195
|
+
fh.write(text.encode("utf-8"))
|
|
196
|
+
_write_bytes(path, buffer.getvalue())
|
|
197
|
+
|
|
198
|
+
def _write_part(self, index: int, counts: AdmissionCounts, stage: str | None = None) -> None:
|
|
199
|
+
_write_json(
|
|
200
|
+
self._part_path(index, stage),
|
|
201
|
+
{"analysis_id": self.analyses[index], **counts.as_dict()},
|
|
202
|
+
)
|
|
203
|
+
|
|
204
|
+
def _read_part(self, index: int) -> AdmissionCounts | None:
|
|
205
|
+
path = self._part_path(index)
|
|
206
|
+
if not path.exists():
|
|
207
|
+
return None
|
|
208
|
+
return self._read_counts(index, path)
|
|
209
|
+
|
|
210
|
+
def _read_counts(self, index: int, path: Path) -> AdmissionCounts:
|
|
211
|
+
data = json.loads(path.read_text(encoding="utf-8"))
|
|
212
|
+
if data.get("analysis_id") != self.analyses[index]:
|
|
213
|
+
raise RecordIncomplete(
|
|
214
|
+
f"{path} records Analysis {data.get('analysis_id')!r}, not "
|
|
215
|
+
f"{self.analyses[index]!r}: the record was written for another manifest"
|
|
216
|
+
)
|
|
217
|
+
return AdmissionCounts.from_dict(data)
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def _write_json(path: Path, payload: Mapping[str, Any]) -> None:
|
|
221
|
+
"""Write ``payload`` whole or not at all."""
|
|
222
|
+
_write_bytes(path, (json.dumps(payload, indent=1) + "\n").encode("utf-8"))
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def _write_bytes(path: Path, data: bytes) -> None:
|
|
226
|
+
"""Write ``data`` whole or not at all: a temp file renamed into place."""
|
|
227
|
+
tmp = path.with_name(f".{path.name}.{os.getpid()}.tmp")
|
|
228
|
+
tmp.write_bytes(data)
|
|
229
|
+
tmp.replace(path)
|