gnome3d 0.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- gnome3d/__init__.py +0 -0
- gnome3d/__main__.py +4 -0
- gnome3d/anchor_contacts.py +136 -0
- gnome3d/cli.py +198 -0
- gnome3d/data.py +529 -0
- gnome3d/extract.py +148 -0
- gnome3d/geometry.py +32 -0
- gnome3d/hierarchy.py +271 -0
- gnome3d/io.py +519 -0
- gnome3d/log.py +168 -0
- gnome3d/mc/__init__.py +33 -0
- gnome3d/mc/jax/__init__.py +19 -0
- gnome3d/mc/jax/arcs.py +1188 -0
- gnome3d/mc/jax/arcs_energy.py +213 -0
- gnome3d/mc/jax/memory.py +56 -0
- gnome3d/mc/jax/share.py +146 -0
- gnome3d/mc/jax/smooth.py +2536 -0
- gnome3d/mc/jax/util.py +204 -0
- gnome3d/mc/numba/__init__.py +28 -0
- gnome3d/mc/numba/arcs.py +158 -0
- gnome3d/mc/numba/arcs_solver.py +418 -0
- gnome3d/mc/numba/cells.py +70 -0
- gnome3d/mc/numba/common.py +370 -0
- gnome3d/mc/numba/heatmap.py +584 -0
- gnome3d/mc/numba/ib.py +168 -0
- gnome3d/mc/numba/smooth.py +521 -0
- gnome3d/mc/numba/terms.py +1124 -0
- gnome3d/pipeline/__init__.py +54 -0
- gnome3d/pipeline/coarse/__init__.py +59 -0
- gnome3d/pipeline/coarse/build.py +1224 -0
- gnome3d/pipeline/coarse/heatmap.py +148 -0
- gnome3d/pipeline/coarse/stages.py +197 -0
- gnome3d/pipeline/dag.py +67 -0
- gnome3d/pipeline/executor.py +385 -0
- gnome3d/pipeline/holes.py +261 -0
- gnome3d/pipeline/ib/__init__.py +35 -0
- gnome3d/pipeline/ib/arcs.py +332 -0
- gnome3d/pipeline/ib/buckets.py +24 -0
- gnome3d/pipeline/ib/chain.py +52 -0
- gnome3d/pipeline/ib/densify.py +190 -0
- gnome3d/pipeline/ib/estimate_dist.py +225 -0
- gnome3d/pipeline/ib/maxent.py +470 -0
- gnome3d/pipeline/ib/smooth.py +234 -0
- gnome3d/pipeline/ib/start.py +186 -0
- gnome3d/pipeline/multigpu.py +149 -0
- gnome3d/pipeline/registry.py +42 -0
- gnome3d/pipeline/relax.py +194 -0
- gnome3d/pipeline/stage.py +47 -0
- gnome3d/pipeline/state.py +113 -0
- gnome3d/pipeline/stitch.py +292 -0
- gnome3d/polymer.py +260 -0
- gnome3d/production.py +252 -0
- gnome3d/reconstruct.py +218 -0
- gnome3d/settings.py +1090 -0
- gnome3d/simulate.py +143 -0
- gnome3d/skeleton.py +328 -0
- gnome3d/tracks.py +237 -0
- gnome3d/types.py +213 -0
- gnome3d/util.py +104 -0
- gnome3d-0.1.1.dist-info/METADATA +157 -0
- gnome3d-0.1.1.dist-info/RECORD +64 -0
- gnome3d-0.1.1.dist-info/WHEEL +5 -0
- gnome3d-0.1.1.dist-info/entry_points.txt +2 -0
- gnome3d-0.1.1.dist-info/top_level.txt +1 -0
gnome3d/__init__.py
ADDED
|
File without changes
|
gnome3d/__main__.py
ADDED
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
"""Contact counts between anchor pairs, read from a full depth map at fine resolution.
|
|
2
|
+
|
|
3
|
+
The table holds, per chromosome, the anchors of the anchor file and the summed counts between
|
|
4
|
+
each pair. A pixel is shared between the anchors it overlaps in proportion to the overlap.
|
|
5
|
+
`slurm/ensemble/prep_anchor_contacts.py` writes the table from an mcool. A run matches its
|
|
6
|
+
anchors to the table's by interval. The table must come from the run's anchor file.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from dataclasses import dataclass
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
from typing import Any, cast
|
|
14
|
+
|
|
15
|
+
import numpy as np
|
|
16
|
+
import scipy.sparse as sp
|
|
17
|
+
|
|
18
|
+
from gnome3d.types import F32Array, F64Array, I32Array, I64Array
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@dataclass(frozen=True)
|
|
22
|
+
class AnchorContacts:
|
|
23
|
+
"""The table of one chromosome.
|
|
24
|
+
|
|
25
|
+
`starts` and `ends` are its anchors, sorted by start. `i`, `j` and `count` are the pairs with
|
|
26
|
+
a count, `i < j`.
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
starts: I64Array
|
|
30
|
+
ends: I64Array
|
|
31
|
+
i: I32Array
|
|
32
|
+
j: I32Array
|
|
33
|
+
count: F32Array
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
AnchorContactMap = dict[str, AnchorContacts]
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def overlap_weights(
|
|
40
|
+
bin_starts: I64Array, res: int, starts: I64Array, ends: I64Array
|
|
41
|
+
) -> sp.csr_matrix:
|
|
42
|
+
"""Returns the bins by anchors matrix of the share of each bin inside each anchor.
|
|
43
|
+
|
|
44
|
+
Parameters
|
|
45
|
+
----------
|
|
46
|
+
bin_starts
|
|
47
|
+
Left edges of a chromosome's bins, ascending, `res` apart.
|
|
48
|
+
res
|
|
49
|
+
Bin width in bp.
|
|
50
|
+
starts, ends
|
|
51
|
+
Anchor intervals, half open.
|
|
52
|
+
"""
|
|
53
|
+
first = np.searchsorted(bin_starts, starts, side="right") - 1
|
|
54
|
+
last = np.searchsorted(bin_starts, ends, side="left")
|
|
55
|
+
span = np.maximum(last - np.maximum(first, 0), 0)
|
|
56
|
+
rows = np.repeat(np.arange(len(starts)), span)
|
|
57
|
+
offs = np.arange(int(span.sum())) - np.repeat(np.cumsum(span) - span, span)
|
|
58
|
+
b = np.repeat(np.maximum(first, 0), span) + offs
|
|
59
|
+
lo = np.maximum(starts[rows], bin_starts[b])
|
|
60
|
+
hi = np.minimum(ends[rows], bin_starts[b] + res)
|
|
61
|
+
ov = (hi - lo).astype(np.float64) / res
|
|
62
|
+
keep = ov > 0
|
|
63
|
+
return sp.csr_matrix((ov[keep], (b[keep], rows[keep])), shape=(len(bin_starts), len(starts)))
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def aggregate_pixels(
|
|
67
|
+
w: sp.csr_matrix, bin1: I64Array, bin2: I64Array, count: F64Array
|
|
68
|
+
) -> sp.coo_matrix:
|
|
69
|
+
"""Sums the pixels `(bin1, bin2, count)` of an upper triangle map into anchor pair counts.
|
|
70
|
+
|
|
71
|
+
`w` is from `overlap_weights`. Returns the strict upper triangle. A pixel inside one anchor
|
|
72
|
+
and a pixel on the diagonal count for no pair.
|
|
73
|
+
"""
|
|
74
|
+
nb, na = int(w.shape[0]), int(w.shape[1]) # pyright: ignore[reportOptionalSubscript]
|
|
75
|
+
off = bin1 != bin2
|
|
76
|
+
m = sp.coo_matrix((count[off], (bin1[off], bin2[off])), shape=(nb, nb)).tocsr()
|
|
77
|
+
m = m + m.T
|
|
78
|
+
c = (w.T @ m @ w).tocoo()
|
|
79
|
+
up = c.row < c.col
|
|
80
|
+
return sp.coo_matrix((c.data[up], (c.row[up], c.col[up])), shape=(na, na))
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def save(path: Path, tables: AnchorContactMap, res: int, source: str) -> None:
|
|
84
|
+
"""Writes the tables as one npz, five arrays per chromosome."""
|
|
85
|
+
out: dict[str, np.ndarray] = {"res": np.array(res), "source": np.array(source)}
|
|
86
|
+
for chrom, t in tables.items():
|
|
87
|
+
for key in ("starts", "ends", "i", "j", "count"):
|
|
88
|
+
out[f"{chrom}/{key}"] = getattr(t, key)
|
|
89
|
+
cast(Any, np.savez_compressed)(path, **out)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def load(path: str, chr_set: set[str]) -> AnchorContactMap:
|
|
93
|
+
"""Loads the tables of the chromosomes in `chr_set`."""
|
|
94
|
+
out: AnchorContactMap = {}
|
|
95
|
+
with np.load(path) as z:
|
|
96
|
+
for chrom in sorted({k.split("/")[0] for k in z.files if "/" in k} & chr_set):
|
|
97
|
+
out[chrom] = AnchorContacts(
|
|
98
|
+
starts=z[f"{chrom}/starts"].astype(np.int64),
|
|
99
|
+
ends=z[f"{chrom}/ends"].astype(np.int64),
|
|
100
|
+
i=z[f"{chrom}/i"].astype(np.int32),
|
|
101
|
+
j=z[f"{chrom}/j"].astype(np.int32),
|
|
102
|
+
count=z[f"{chrom}/count"].astype(np.float32),
|
|
103
|
+
)
|
|
104
|
+
return out
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def match(table: AnchorContacts, starts: I64Array, ends: I64Array) -> I64Array:
|
|
108
|
+
"""Returns the table index of each given anchor, -1 for an anchor the table lacks."""
|
|
109
|
+
k = np.searchsorted(table.starts, starts, side="left")
|
|
110
|
+
idx = np.full(len(starts), -1, dtype=np.int64)
|
|
111
|
+
n = len(table.starts)
|
|
112
|
+
# Several table anchors may share a start; walk forward to the one with the same end.
|
|
113
|
+
for a in range(len(starts)):
|
|
114
|
+
p = int(k[a])
|
|
115
|
+
while p < n and table.starts[p] == starts[a]:
|
|
116
|
+
if table.ends[p] == ends[a]:
|
|
117
|
+
idx[a] = p
|
|
118
|
+
break
|
|
119
|
+
p += 1
|
|
120
|
+
return idx
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def dense_counts(table: AnchorContacts, starts: I64Array, ends: I64Array) -> F64Array:
|
|
124
|
+
"""Returns the symmetric counts between the given anchors.
|
|
125
|
+
|
|
126
|
+
The diagonal and any anchor the table lacks are zero.
|
|
127
|
+
"""
|
|
128
|
+
idx = match(table, starts, ends)
|
|
129
|
+
n = len(starts)
|
|
130
|
+
pos = np.full(len(table.starts), -1, dtype=np.int64)
|
|
131
|
+
pos[idx[idx >= 0]] = np.flatnonzero(idx >= 0)
|
|
132
|
+
a, b = pos[table.i], pos[table.j]
|
|
133
|
+
keep = (a >= 0) & (b >= 0)
|
|
134
|
+
out = np.zeros((n, n), dtype=np.float64)
|
|
135
|
+
np.add.at(out, (a[keep], b[keep]), table.count[keep].astype(np.float64))
|
|
136
|
+
return out + out.T
|
gnome3d/cli.py
ADDED
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Command-line interface for 3dgnome-ng.
|
|
3
|
+
|
|
4
|
+
Single region: gnome3d --config X.ini --region chr1:18288319-20307135
|
|
5
|
+
Single chr: gnome3d --config X.ini --region chr14
|
|
6
|
+
Chromosome list: gnome3d --config X.ini --region chr1,chr3,chrX
|
|
7
|
+
Range: gnome3d --config X.ini --region chr1-chr22,chrX
|
|
8
|
+
Whole genome: gnome3d --config X.ini (defaults to chr1-chr22,chrX)
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
import argparse
|
|
12
|
+
import contextlib
|
|
13
|
+
import sys
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
|
|
16
|
+
from gnome3d import log
|
|
17
|
+
from gnome3d.data import ContactData
|
|
18
|
+
from gnome3d.io import parse_chrs_arg, write_bead_table, write_cif
|
|
19
|
+
from gnome3d.pipeline.executor import Executor
|
|
20
|
+
from gnome3d.reconstruct import MEMBER_SEED_STRIDE, pick_executor, reconstruct
|
|
21
|
+
from gnome3d.settings import Settings
|
|
22
|
+
from gnome3d.types import BedRegion
|
|
23
|
+
|
|
24
|
+
LOG = log.get("main")
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def parse_members(spec: str) -> list[int]:
|
|
28
|
+
"""Parses a comma list of numbers and `lo-hi` ranges into structure indices, in order and
|
|
29
|
+
without repeats.
|
|
30
|
+
"""
|
|
31
|
+
out: list[int] = []
|
|
32
|
+
for part in spec.split(","):
|
|
33
|
+
part = part.strip()
|
|
34
|
+
if not part:
|
|
35
|
+
continue
|
|
36
|
+
if "-" in part.lstrip("-"):
|
|
37
|
+
lo, hi = part.split("-", 1)
|
|
38
|
+
out.extend(range(int(lo), int(hi) + 1))
|
|
39
|
+
else:
|
|
40
|
+
out.append(int(part))
|
|
41
|
+
return list(dict.fromkeys(out))
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _cif_name(entry_base: str, chr_: str, i: int, multi_chr: bool) -> str:
|
|
45
|
+
"""Returns the CIF file name of one structure, with the chromosome when the run has several."""
|
|
46
|
+
if multi_chr:
|
|
47
|
+
return f"{entry_base}_{chr_}_s{i + 1}.cif"
|
|
48
|
+
return f"{entry_base}_s{i + 1}.cif"
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _run_structure(
|
|
52
|
+
i: int,
|
|
53
|
+
pos: int,
|
|
54
|
+
n: int,
|
|
55
|
+
s: Settings,
|
|
56
|
+
data: ContactData,
|
|
57
|
+
chrs_list: list[str],
|
|
58
|
+
region: BedRegion | None,
|
|
59
|
+
out_dir: Path,
|
|
60
|
+
entry_base: str,
|
|
61
|
+
executor: Executor,
|
|
62
|
+
) -> int:
|
|
63
|
+
"""Reconstructs one structure, writes it and returns its bead count.
|
|
64
|
+
|
|
65
|
+
`i` is the member index, which sets the seed offset and the file name. `pos` is the 1-based
|
|
66
|
+
place in the run, which the progress label counts.
|
|
67
|
+
"""
|
|
68
|
+
label = f"structure {pos}/{n}" + (f" (member {i + 1})" if pos != i + 1 else "")
|
|
69
|
+
structure_ctx = log.step(LOG, label) if n > 1 else contextlib.nullcontext()
|
|
70
|
+
with structure_ctx:
|
|
71
|
+
per_chr = reconstruct(
|
|
72
|
+
s, data, chrs_list, region, executor=executor, seed_offset=i * MEMBER_SEED_STRIDE
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
multi_chr = len(chrs_list) > 1
|
|
76
|
+
total_beads = 0
|
|
77
|
+
for chr_ in chrs_list:
|
|
78
|
+
beads = per_chr.get(chr_)
|
|
79
|
+
if not beads:
|
|
80
|
+
LOG.warning("%s: no leaf beads (skipping)", chr_)
|
|
81
|
+
continue
|
|
82
|
+
|
|
83
|
+
cif_path = out_dir / _cif_name(entry_base, chr_, i, multi_chr)
|
|
84
|
+
entry_id = cif_path.stem
|
|
85
|
+
# The CIF is written last. Its presence marks the member as done for a later run.
|
|
86
|
+
write_bead_table(str(cif_path.with_suffix(".beads.tsv")), chr_, beads)
|
|
87
|
+
write_cif(str(cif_path), beads, entry_id=entry_id)
|
|
88
|
+
LOG.info("wrote %s (%d beads)", cif_path, len(beads))
|
|
89
|
+
total_beads += len(beads)
|
|
90
|
+
|
|
91
|
+
if total_beads == 0:
|
|
92
|
+
raise RuntimeError(f"Structure {i + 1}: no leaf beads from any chromosome")
|
|
93
|
+
|
|
94
|
+
return total_beads
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def main() -> None:
|
|
98
|
+
if sys.argv[1:2] == ["extract"]:
|
|
99
|
+
from gnome3d.extract import main as extract_main
|
|
100
|
+
|
|
101
|
+
sys.exit(extract_main(sys.argv[2:]))
|
|
102
|
+
parser = argparse.ArgumentParser(description="3dgnome-ng structure prediction")
|
|
103
|
+
parser.add_argument("--config", required=True, help="Path to config.ini")
|
|
104
|
+
parser.add_argument(
|
|
105
|
+
"--region",
|
|
106
|
+
default="",
|
|
107
|
+
help=(
|
|
108
|
+
"Chromosomes/region to reconstruct. Examples: "
|
|
109
|
+
"'chr14:18288319-20307135' (single region), 'chr14' (single chr), "
|
|
110
|
+
"'chr1,chr3,chrX' (comma list), 'chr1-chr22,chrX' (range + extras). "
|
|
111
|
+
"Default (empty): chr1-chr22,chrX (whole human genome)."
|
|
112
|
+
),
|
|
113
|
+
)
|
|
114
|
+
parser.add_argument(
|
|
115
|
+
"-n",
|
|
116
|
+
"--n-structures",
|
|
117
|
+
type=int,
|
|
118
|
+
default=1,
|
|
119
|
+
help="Number of independent structures to generate (default 1)",
|
|
120
|
+
)
|
|
121
|
+
parser.add_argument(
|
|
122
|
+
"--members",
|
|
123
|
+
default="",
|
|
124
|
+
help=(
|
|
125
|
+
"Explicit structure indices instead of -n, as a comma list of numbers and "
|
|
126
|
+
"ranges, e.g. '0-9' or '0,4,7-9'. One ensemble can then be split over several "
|
|
127
|
+
"jobs by giving each a disjoint set, and a member whose .cif already exists is "
|
|
128
|
+
"skipped, so a requeued job resumes instead of redoing its work. Index i keeps "
|
|
129
|
+
"the seed offset it would have had under -n, so the ensemble is the same "
|
|
130
|
+
"however it was split."
|
|
131
|
+
),
|
|
132
|
+
)
|
|
133
|
+
parser.add_argument("--data-dir", default=None, help="Override data_dir from config")
|
|
134
|
+
parser.add_argument("--out", default=".", help="Output directory (default: .)")
|
|
135
|
+
parser.add_argument(
|
|
136
|
+
"--log-file",
|
|
137
|
+
default=None,
|
|
138
|
+
help=(
|
|
139
|
+
"Write a full-detail (DEBUG) structured log here in addition to "
|
|
140
|
+
"stdout. Overrides [main] log_file. Handy for reconstructing "
|
|
141
|
+
"parallel (ib_workers>1 / -n>1) runs."
|
|
142
|
+
),
|
|
143
|
+
)
|
|
144
|
+
args = parser.parse_args()
|
|
145
|
+
|
|
146
|
+
chrs_list, bed_region = parse_chrs_arg(args.region)
|
|
147
|
+
if not chrs_list:
|
|
148
|
+
sys.exit(f"Cannot parse region: {args.region!r}")
|
|
149
|
+
|
|
150
|
+
s = Settings()
|
|
151
|
+
if not s.load_ini(args.config):
|
|
152
|
+
sys.exit(f"Failed to load config: {args.config!r}")
|
|
153
|
+
if args.data_dir:
|
|
154
|
+
s.data_dir = args.data_dir
|
|
155
|
+
|
|
156
|
+
log.setup(s.output_level, log_file=args.log_file or s.log_file or None)
|
|
157
|
+
log.status(LOG, "config: %s data_dir: %s", args.config, s.data_dir)
|
|
158
|
+
log.status(
|
|
159
|
+
LOG,
|
|
160
|
+
"chromosomes (%d): %s%s",
|
|
161
|
+
len(chrs_list),
|
|
162
|
+
",".join(chrs_list),
|
|
163
|
+
f" region={bed_region.chr}:{bed_region.start}-{bed_region.end}" if bed_region else "",
|
|
164
|
+
)
|
|
165
|
+
|
|
166
|
+
data = ContactData.from_files(s, chrs_list, bed_region)
|
|
167
|
+
|
|
168
|
+
out_dir = Path(args.out)
|
|
169
|
+
out_dir.mkdir(parents=True, exist_ok=True)
|
|
170
|
+
raw = args.region.strip() or "genome"
|
|
171
|
+
entry_base = raw.replace(":", "_").replace("-", "_").replace(",", "_")
|
|
172
|
+
|
|
173
|
+
# Structures run one after another. The seeding uses a global RNG and is not safe across
|
|
174
|
+
# threads.
|
|
175
|
+
members = parse_members(args.members) if args.members else list(range(args.n_structures))
|
|
176
|
+
if not members:
|
|
177
|
+
sys.exit(f"No structures selected: --members {args.members!r}")
|
|
178
|
+
|
|
179
|
+
executor = pick_executor(s)
|
|
180
|
+
n = len(members)
|
|
181
|
+
log.status(LOG, "running %d structure(s) [%s]", n, type(executor).__name__)
|
|
182
|
+
|
|
183
|
+
multi_chr = len(chrs_list) > 1
|
|
184
|
+
written = 0
|
|
185
|
+
for pos, i in enumerate(members, 1):
|
|
186
|
+
if args.members and all(
|
|
187
|
+
(out_dir / _cif_name(entry_base, c, i, multi_chr)).exists() for c in chrs_list
|
|
188
|
+
):
|
|
189
|
+
log.status(LOG, "structure %d already written, skipping", i + 1)
|
|
190
|
+
continue
|
|
191
|
+
_run_structure(i, pos, n, s, data, chrs_list, bed_region, out_dir, entry_base, executor)
|
|
192
|
+
written += 1
|
|
193
|
+
|
|
194
|
+
log.status(LOG, "%d structure(s) written to %s/", written, out_dir)
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
if __name__ == "__main__":
|
|
198
|
+
main()
|