consenrich 0.7.5b1__cp314-cp314-macosx_10_15_x86_64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of consenrich might be problematic. Click here for more details.

consenrich/core.py ADDED
@@ -0,0 +1,1438 @@
1
+ # -*- coding: utf-8 -*-
2
+ r"""
3
+ Consenrich core functions and classes.
4
+
5
+ """
6
+
7
+ import logging
8
+ import os
9
+ from tempfile import NamedTemporaryFile
10
+ from typing import (
11
+ Callable,
12
+ List,
13
+ Optional,
14
+ Tuple,
15
+ DefaultDict,
16
+ Any,
17
+ NamedTuple,
18
+ )
19
+
20
+ import numpy as np
21
+ import numpy.typing as npt
22
+ import pybedtools as bed
23
+ from scipy import signal, ndimage
24
+
25
+ from . import cconsenrich
26
+
27
+ logging.basicConfig(
28
+ level=logging.INFO,
29
+ format="%(asctime)s - %(module)s.%(funcName)s - %(levelname)s - %(message)s",
30
+ )
31
+
32
+ logger = logging.getLogger(__name__)
33
+
34
+
35
+ def resolveExtendBP(extendBP, bamFiles: List[str]) -> List[int]:
36
+ numFiles = len(bamFiles)
37
+
38
+ if isinstance(extendBP, str):
39
+ stringValue = extendBP.replace(" ", "")
40
+ try:
41
+ extendBP = (
42
+ [int(x) for x in stringValue.split(",")]
43
+ if stringValue
44
+ else []
45
+ )
46
+ except ValueError:
47
+ raise ValueError(
48
+ "`extendBP` string must be comma-separated values (castable to integers)"
49
+ )
50
+ if extendBP is None:
51
+ return [0] * numFiles
52
+ elif isinstance(extendBP, list):
53
+ valuesList = [int(x) for x in extendBP]
54
+ valuesLen = len(valuesList)
55
+ if valuesLen == 0:
56
+ return [0] * numFiles
57
+ if valuesLen == 1:
58
+ return [valuesList[0]] * numFiles
59
+ if valuesLen == numFiles:
60
+ return valuesList
61
+ raise ValueError(
62
+ f"extendBP length {valuesLen} does not match number of bamFiles {numFiles}; "
63
+ f"provide 0, 1, or {numFiles} values."
64
+ )
65
+ elif isinstance(extendBP, int) or isinstance(extendBP, float):
66
+ return [int(extendBP)] * numFiles
67
+ raise TypeError(
68
+ f"Invalid extendBP type: {type(extendBP).__name__}. "
69
+ "Expecting a single number (broadcast), a list of numbers matching `bamFiles`."
70
+ )
71
+
72
+
73
+ class processParams(NamedTuple):
74
+ r"""Parameters related to the process model of Consenrich.
75
+
76
+ The process model governs the signal and variance propagation
77
+ through the state transition :math:`\mathbf{F} \in \mathbb{R}^{2 \times 2}`
78
+ and process noise covariance :math:`\mathbf{Q}_{[i]} \in \mathbb{R}^{2 \times 2}`
79
+ matrices.
80
+
81
+ :param deltaF: Scales the signal and variance propagation between adjacent genomic intervals.
82
+ :param minQ: Minimum process noise level (diagonal in :math:`\mathbf{Q}_{[i]}`)
83
+ for each state variable.
84
+ :type minQ: float
85
+ :param dStatAlpha: Threshold on the deviation between the data and estimated signal -- used to determine whether the process noise is scaled up.
86
+ :type dStatAlpha: float
87
+ :param dStatd: Constant :math:`d` in the scaling expression :math:`\sqrt{d|D_{[i]} - \alpha_D| + c}`
88
+ that is used to up/down-scale the process noise covariance in the event of a model mismatch.
89
+ :type dStatd: float
90
+ :param dStatPC: Constant :math:`c` in the scaling expression :math:`\sqrt{d|D_{[i]} - \alpha_D| + c}`
91
+ that is used to up/down-scale the process noise covariance in the event of a model mismatch.
92
+ :type dStatPC: float
93
+ :param scaleResidualsByP11: If `True`, the primary state variances (posterior) :math:`\widetilde{P}_{[i], (11)}, i=1\ldots n` are included in the inverse-variance (precision) weighting of residuals :math:`\widetilde{\mathbf{y}}_{[i]}, i=1\ldots n`.
94
+ If `False`, only the per-sample *observation noise levels* will be used in the precision-weighting. Note that this does not affect `raw` residuals output (See :class:`outputParams`).
95
+ :type scaleResidualsByP11: Optional[bool]
96
+
97
+ """
98
+
99
+ deltaF: float
100
+ minQ: float
101
+ maxQ: float
102
+ offDiagQ: float
103
+ dStatAlpha: float
104
+ dStatd: float
105
+ dStatPC: float
106
+ scaleResidualsByP11: Optional[bool] = True
107
+
108
+
109
+ class observationParams(NamedTuple):
110
+ r"""Parameters related to the observation model of Consenrich.
111
+
112
+ The observation model is used to integrate sequence alignment count
113
+ data from the multiple input samples and account for region-and-sample-specific
114
+ noise processes corrupting data. The observation model matrix
115
+ :math:`\mathbf{H} \in \mathbb{R}^{m \times 2}` maps from the state dimension (2)
116
+ to the dimension of measurements/data (:math:`m`).
117
+
118
+ :param minR: The minimum observation noise level for each sample
119
+ :math:`j=1\ldots m` in the observation noise covariance
120
+ matrix :math:`\mathbf{R}_{[i, (11:mm)]}`.
121
+ :type minR: float
122
+ :param numNearest: The number of nearest nearby 'sparse' features to use for local
123
+ variance calculation. Ignored if `useALV` is True.
124
+ :type numNearest: int
125
+ :param localWeight: The coefficient for the local noise level (based on the local surrounding window / `numNearest` features) used in the weighted sum measuring sample-specific noise level at the current interval.
126
+ :type localWeight: float
127
+ :param globalWeight: The coefficient for the global noise level (based on all genomic intervals :math:`i=1\ldots n`) used in the weighted sum measuring sample-specific noise level at the current interval.
128
+ :type globalWeight: float
129
+ :param approximationWindowLengthBP: The length of the local variance approximation window in base pairs (BP)
130
+ for the local variance calculation.
131
+ :type approximationWindowLengthBP: int
132
+ :param sparseBedFile: The path to a BED file of 'sparse' regions for the local variance calculation. For genomes with default resources in `src/consenrich/data`, this may be left as `None`,
133
+ and a default annotation that is devoid of putative regulatory elements (ENCODE cCREs) will be used. Users can instead supply a custom BED file or set `observationParams.useALV` to `True`
134
+ to avoid predefined annotations.
135
+ :type sparseBedFile: str, optional
136
+ :param noGlobal: If True, only the 'local' variances are used to approximate observation noise
137
+ covariance :math:`\mathbf{R}_{[:, (11:mm)]}`.
138
+ :type noGlobal: bool
139
+ :param useALV: Whether to use average local variance (ALV) heuristic *exclusively* to approximate observation noise
140
+ covariances per-sample, per-interval. Note that unrestricted ALV (i.e., without masking previously annotated high-signal regions) is comparatively vulnerable to inflated noise estimates in large enriched genomic domains.
141
+ :type useALV: bool
142
+ :param lowPassFilterType: The type of low-pass filter to use (e.g., 'median', 'mean') in the ALV calculation (:func:`consenrich.core.getAverageLocalVarianceTrack`).
143
+ :type lowPassFilterType: Optional[str]
144
+ """
145
+
146
+ minR: float
147
+ maxR: float
148
+ useALV: bool
149
+ useConstantNoiseLevel: bool
150
+ noGlobal: bool
151
+ numNearest: int
152
+ localWeight: float
153
+ globalWeight: float
154
+ approximationWindowLengthBP: int
155
+ lowPassWindowLengthBP: int
156
+ lowPassFilterType: Optional[str]
157
+ returnCenter: bool
158
+
159
+
160
+ class stateParams(NamedTuple):
161
+ r"""Parameters related to state and uncertainty bounds and initialization.
162
+
163
+ :param stateInit: Initial value of the 'primary' state/signal at the first genomic interval: :math:`x_{[1]}`
164
+ :type stateInit: float
165
+ :param stateCovarInit: Initial state covariance (covariance) scale. Note, the *initial* state uncertainty :math:`\mathbf{P}_{[1]}` is a multiple of the identity matrix :math:`\mathbf{I}`. Final results are typically insensitive to this parameter, since the filter effectively 'forgets' its initialization after processing a moderate number of intervals and backward smoothing.
166
+ :type stateCovarInit: float
167
+ :param boundState: If True, the primary state estimate for :math:`x_{[i]}` is reported within `stateLowerBound` and `stateUpperBound`. Note that the internal filtering is unaffected.
168
+ :type boundState: bool
169
+ :param stateLowerBound: Lower bound for the state estimate.
170
+ :type stateLowerBound: float
171
+ :param stateUpperBound: Upper bound for the state estimate.
172
+ :type stateUpperBound: float
173
+ """
174
+
175
+ stateInit: float
176
+ stateCovarInit: float
177
+ boundState: bool
178
+ stateLowerBound: float
179
+ stateUpperBound: float
180
+
181
+
182
+ class samParams(NamedTuple):
183
+ r"""Parameters related to reading BAM files
184
+
185
+ :param samThreads: The number of threads to use for reading BAM files.
186
+ :type samThreads: int
187
+ :param samFlagExclude: The SAM flag to exclude certain reads.
188
+ :type samFlagExclude: int
189
+ :param oneReadPerBin: If 1, only the interval with the greatest read overlap is incremented.
190
+ :type oneReadPerBin: int
191
+ :param chunkSize: maximum number of intervals' data to hold in memory before flushing to disk.
192
+ :type chunkSize: int
193
+ :param offsetStr: A string of two comma-separated integers -- first for the 5' shift on forward strand, second for the 5' shift on reverse strand.
194
+ :type offsetStr: str
195
+ :param extendBP: A list of integers specifying the number of base pairs to extend reads for each BAM file after shifting per `offsetStr`.
196
+ If all BAM files share the same expected frag. length, can supply a single numeric value to be broadcasted. Ignored for PE reads.
197
+ :type extendBP: List[int]
198
+ :param maxInsertSize: Maximum frag length/insert for paired-end reads.
199
+ :type maxInsertSize: int
200
+ :param pairedEndMode: If > 0, only proper pairs are counted subject to `maxInsertSize`.
201
+ :type pairedEndMode: int
202
+ :param inferFragmentLength: Intended for single-end data: if > 0, the maximum correlation lag
203
+ (avg.) between *strand-specific* read tracks is taken as the fragment length estimate and used to
204
+ extend reads from 5'. Ignored if `pairedEndMode > 0` or `extendBP` set. This parameter is particularly
205
+ important when targeting broader marks (e.g., ChIP-seq H3K27me3).
206
+ :type inferFragmentLength: int
207
+ :param countEndsOnly: If True, only the 5' ends of reads are counted. Overrides `inferFragmentLength` and `pairedEndMode`.
208
+ :type countEndsOnly: Optional[bool]
209
+
210
+ .. tip::
211
+
212
+ For an overview of SAM flags, see https://broadinstitute.github.io/picard/explain-flags.html
213
+
214
+ """
215
+
216
+ samThreads: int
217
+ samFlagExclude: int
218
+ oneReadPerBin: int
219
+ chunkSize: int
220
+ offsetStr: Optional[str] = "0,0"
221
+ extendBP: Optional[List[int]] = []
222
+ maxInsertSize: Optional[int] = 1000
223
+ pairedEndMode: Optional[int] = 0
224
+ inferFragmentLength: Optional[int] = 0
225
+ countEndsOnly: Optional[bool] = False
226
+
227
+
228
+ class detrendParams(NamedTuple):
229
+ r"""Parameters related detrending and background-removal
230
+
231
+ :param useOrderStatFilter: Whether to use a local/moving order statistic (percentile filter) to model and remove trends in the read density data.
232
+ :type useOrderStatFilter: bool
233
+ :param usePolyFilter: Whether to use a low-degree polynomial fit to model and remove trends in the read density data.
234
+ :type usePolyFilter: bool
235
+ :param detrendSavitzkyGolayDegree: The polynomial degree of the Savitzky-Golay filter to use for detrending
236
+ :type detrendSavitzkyGolayDegree: int
237
+ :param detrendTrackPercentile: The percentile to use for the local/moving order-statistic filter.
238
+ Decrease for broad marks + sparse data if `useOrderStatFilter` is True.
239
+ :type detrendTrackPercentile: float
240
+ :param detrendWindowLengthBP: The length of the window in base pairs for detrending.
241
+ Increase for broader marks + sparse data.
242
+ :type detrendWindowLengthBP: int
243
+ """
244
+
245
+ useOrderStatFilter: bool
246
+ usePolyFilter: bool
247
+ detrendTrackPercentile: float
248
+ detrendSavitzkyGolayDegree: int
249
+ detrendWindowLengthBP: int
250
+
251
+
252
+ class inputParams(NamedTuple):
253
+ r"""Parameters related to the input data for Consenrich.
254
+
255
+ :param bamFiles: A list of paths to distinct coordinate-sorted and indexed BAM files.
256
+ :type bamFiles: List[str]
257
+
258
+ :param bamFilesControl: A list of paths to distinct coordinate-sorted and
259
+ indexed control BAM files. e.g., IgG control inputs for ChIP-seq.
260
+
261
+ :type bamFilesControl: List[str], optional
262
+
263
+ """
264
+
265
+ bamFiles: List[str]
266
+ bamFilesControl: Optional[List[str]]
267
+ pairedEnd: Optional[bool]
268
+
269
+
270
+ class genomeParams(NamedTuple):
271
+ r"""Specify assembly-specific resources, parameters.
272
+
273
+ :param genomeName: If supplied, default resources for the assembly (sizes file, blacklist, and 'sparse' regions) in `src/consenrich/data` are used.
274
+ ``ce10, ce11, dm6, hg19, hg38, mm10, mm39`` have default resources available.
275
+ :type genomeName: str
276
+ :param chromSizesFile: A two-column TSV-like file with chromosome names and sizes (in base pairs).
277
+ :type chromSizesFile: str
278
+ :param blacklistFile: A BED file with regions to exclude.
279
+ :type blacklistFile: str, optional
280
+ :param sparseBedFile: A BED file with 'sparse regions' used to estimate noise levels -- ignored if `observationParams.useALV` is True. 'Sparse regions' broadly refers to genomic intervals devoid of the targeted signal, based on prior annotations.
281
+ Users may supply a custom BED file and/or set `observationParams.useALV` to `True` to avoid relying on predefined annotations.
282
+ :type sparseBedFile: str, optional
283
+ :param chromosomes: A list of chromosome names to analyze. If None, all chromosomes in `chromSizesFile` are used.
284
+ :type chromosomes: List[str]
285
+ """
286
+
287
+ genomeName: str
288
+ chromSizesFile: str
289
+ blacklistFile: Optional[str]
290
+ sparseBedFile: Optional[str]
291
+ chromosomes: List[str]
292
+ excludeChroms: List[str]
293
+ excludeForNorm: List[str]
294
+
295
+
296
+ class countingParams(NamedTuple):
297
+ r"""Parameters related to counting reads in genomic intervals.
298
+
299
+ :param stepSize: Step size (bp) for the genomic intervals (AKA bin size, interval length, width, etc.)
300
+ :type stepSize: int
301
+ :param scaleDown: If using paired treatment and control BAM files, whether to
302
+ scale down the larger of the two before computing the difference/ratio
303
+ :type scaleDown: bool, optional
304
+ :param scaleFactors: Scale factors for the read counts.
305
+ :type scaleFactors: List[float], optional
306
+ :param scaleFactorsControl: Scale factors for the control read counts.
307
+ :type scaleFactorsControl: List[float], optional
308
+ :param numReads: Number of reads to sample.
309
+ :type numReads: int
310
+ :param applyAsinh: If true, :math:`\textsf{arsinh}(x)` applied to counts :math:`x` for each supplied BAM file (log-like for large values and linear near the origin).
311
+ :type applyAsinh: bool, optional
312
+ :param applyLog: If true, :math:`\textsf{log}(x + 1)` applied to counts :math:`x` for each supplied BAM file.
313
+ :type applyLog: bool, optional
314
+ """
315
+
316
+ stepSize: int
317
+ scaleDown: Optional[bool]
318
+ scaleFactors: Optional[List[float]]
319
+ scaleFactorsControl: Optional[List[float]]
320
+ numReads: int
321
+ applyAsinh: Optional[bool]
322
+ applyLog: Optional[bool]
323
+ rescaleToTreatmentCoverage: Optional[bool] = False # deprecated
324
+
325
+
326
+ class matchingParams(NamedTuple):
327
+ r"""Parameters related to the matching algorithm.
328
+
329
+ See :ref:`matching` for an overview of the approach.
330
+
331
+ :param templateNames: A list of str values -- each entry references a mother wavelet (or its corresponding scaling function). e.g., `[haar, db2]`
332
+ :type templateNames: List[str]
333
+ :param cascadeLevels: Number of cascade iterations used to approximate each template (wavelet or scaling function).
334
+ Must have the same length as `templateNames`, with each entry aligned to the
335
+ corresponding template. e.g., given templateNames `[haar, db2]`, then `[2,2]` would use 2 cascade levels for both templates.
336
+ :type cascadeLevels: List[int]
337
+ :param iters: Number of random blocks to sample in the response sequence while building
338
+ an empirical null to test significance. See :func:`cconsenrich.csampleBlockStats`.
339
+ :type iters: int
340
+ :param alpha: Primary significance threshold on detected matches. Specifically, the
341
+ minimum corrected empirical p-value approximated from randomly sampled blocks in the
342
+ response sequence.
343
+ :type alpha: float
344
+ :param minMatchLengthBP: Within a window of `minMatchLengthBP` length (bp), relative maxima in
345
+ the signal-template convolution must be greater in value than others to qualify as matches.
346
+ If set to a value less than 1, the minimum length is determined via :func:`consenrich.matching.autoMinLengthIntervals`.
347
+ If set to `None`, defaults to 250 bp.
348
+ :param minSignalAtMaxima: Secondary significance threshold coupled with `alpha`. Requires the *signal value*
349
+ at relative maxima in the response sequence to be greater than this threshold. Comparisons are made in log-scale
350
+ to temper genome-wide dynamic range. If a `float` value is provided, the minimum signal value must be greater
351
+ than this (absolute) value. *Set to a negative value to disable the threshold*.
352
+ If a `str` value is provided, looks for 'q:quantileValue', e.g., 'q:0.90'. The
353
+ threshold is then set to the corresponding quantile of the non-zero signal estimates.
354
+ :type minSignalAtMaxima: Optional[str | float]
355
+ :param useScalingFunction: If True, use (only) the scaling function to build the matching template.
356
+ If False, use (only) the wavelet function.
357
+ :type useScalingFunction: bool
358
+ :param excludeRegionsBedFile: A BED file with regions to exclude from matching
359
+ :type excludeRegionsBedFile: Optional[str]
360
+ :param penalizeBy: Specify a positional metric to scale/weight signal values by when matching.
361
+ `stateUncertainty` divides signal values by the square root of the primary state variance :math:`\sqrt{\widetilde{P}_{i,(11)}}` at each position :math:`i`,
362
+ thereby down-weighting positions where the posterior state uncertainty is high. 'muncTrace' divides signal values by
363
+ the square root of the *average* observation noise per interval :math:`\sqrt{\frac{\textsf{Trace}\left(\mathbf{R}_{[i]}\right)}{m}}` at each position :math:`i`,
364
+ :type penalizeBy: Optional[str]
365
+ :param eps: Tolerance parameter for relative maxima detection in the response sequence. Set to zero to enforce strict
366
+ inequalities when identifying discrete relative maxima.
367
+ :type eps: float
368
+ :seealso: :func:`cconsenrich.csampleBlockStats`, :ref:`matching`, :class:`outputParams`.
369
+ """
370
+
371
+ templateNames: List[str]
372
+ cascadeLevels: List[int]
373
+ iters: int
374
+ alpha: float
375
+ useScalingFunction: Optional[bool]
376
+ minMatchLengthBP: Optional[int]
377
+ maxNumMatches: Optional[int]
378
+ minSignalAtMaxima: Optional[str | float]
379
+ merge: Optional[bool]
380
+ mergeGapBP: Optional[int]
381
+ excludeRegionsBedFile: Optional[str]
382
+ penalizeBy: Optional[str]
383
+ randSeed: Optional[int] = 42
384
+ eps: Optional[float] = 1.0e-2
385
+
386
+
387
+ class outputParams(NamedTuple):
388
+ r"""Parameters related to output files.
389
+
390
+ :param convertToBigWig: If True, output bedGraph files are converted to bigWig format.
391
+ :type convertToBigWig: bool
392
+ :param roundDigits: Number of decimal places to round output values (bedGraph)
393
+ :type roundDigits: int
394
+ :param writeResiduals: If True, write to a separate bedGraph the mean of precision-weighted residuals at each interval. These may be interpreted as
395
+ a measure of model mismatch. Where these quantities are large (+-) the estimated signal and uncertainty do not explain the observed deviation from the data.
396
+ :type writeResiduals: bool
397
+ :param writeRawResiduals: If True, write to a separate bedGraph the pointwise avg. of post-fit residuals at each interval. These values are not 'precision-weighted'.
398
+ :type writeRawResiduals: bool
399
+ :param writeMuncTrace: If True, write to a separate bedGraph :math:`\sqrt{\frac{\textsf{Trace}\left(\mathbf{R}_{[i]}\right)}{m}}` -- that is, square root of the 'average' observation noise level at each interval :math:`i=1\ldots n`, where :math:`m` is the number of samples/tracks.
400
+ :type writeMuncTrace: bool
401
+ :param writeStateStd: If True, write to a separate bedGraph estimated 'standard deviation' of the primary state, :math:`\sqrt{\widetilde{P}_{i,(11)}}`, at each interval. Note that an absolute Gaussian interpretation of this metric depends on sample size, arguments in :class:`processParams` and :class:`observationParams`, etc.
402
+ In any case, this metric may be interpreted as a relative measure of uncertainty in the state estimate at each interval.
403
+ :type writeStateStd: bool
404
+ """
405
+
406
+ convertToBigWig: bool
407
+ roundDigits: int
408
+ writeResiduals: bool
409
+ writeRawResiduals: bool
410
+ writeMuncTrace: bool
411
+ writeStateStd: bool
412
+
413
+
414
+ def _numIntervals(start: int, end: int, step: int) -> int:
415
+ # helper for consistency
416
+ length = max(0, end - start)
417
+ return (length + step) // step
418
+
419
+
420
+ def getChromRanges(
421
+ bamFile: str,
422
+ chromosome: str,
423
+ chromLength: int,
424
+ samThreads: int,
425
+ samFlagExclude: int,
426
+ ) -> Tuple[int, int]:
427
+ r"""Get the start and end positions of reads in a chromosome from a BAM file.
428
+
429
+ :param bamFile: See :class:`inputParams`.
430
+ :type bamFile: str
431
+ :param chromosome: the chromosome to read in `bamFile`.
432
+ :type chromosome: str
433
+ :param chromLength: Base pair length of the chromosome.
434
+ :type chromLength: int
435
+ :param samThreads: See :class:`samParams`.
436
+ :type samThreads: int
437
+ :param samFlagExclude: See :class:`samParams`.
438
+ :type samFlagExclude: int
439
+ :return: Tuple of start and end positions (nucleotide coordinates) in the chromosome.
440
+ :rtype: Tuple[int, int]
441
+
442
+ :seealso: :func:`getChromRangesJoint`, :func:`cconsenrich.cgetFirstChromRead`, :func:`cconsenrich.cgetLastChromRead`
443
+ """
444
+ start: int = cconsenrich.cgetFirstChromRead(
445
+ bamFile, chromosome, chromLength, samThreads, samFlagExclude
446
+ )
447
+ end: int = cconsenrich.cgetLastChromRead(
448
+ bamFile, chromosome, chromLength, samThreads, samFlagExclude
449
+ )
450
+ return start, end
451
+
452
+
453
+ def getChromRangesJoint(
454
+ bamFiles: List[str],
455
+ chromosome: str,
456
+ chromSize: int,
457
+ samThreads: int,
458
+ samFlagExclude: int,
459
+ ) -> Tuple[int, int]:
460
+ r"""For multiple BAM files, reconcile a single start and end position over which to count reads,
461
+ where the start and end positions are defined by the first and last reads across all BAM files.
462
+
463
+ :param bamFiles: List of BAM files to read.
464
+ :type bamFiles: List[str]
465
+ :param chromosome: Chromosome to read.
466
+ :type chromosome: str
467
+ :param chromSize: Size of the chromosome.
468
+ :type chromSize: int
469
+ :param samThreads: Number of threads to use for reading the BAM files.
470
+ :type samThreads: int
471
+ :param samFlagExclude: SAM flag to exclude certain reads.
472
+ :type samFlagExclude: int
473
+ :return: Tuple of start and end positions.
474
+ :rtype: Tuple[int, int]
475
+
476
+ :seealso: :func:`getChromRanges`, :func:`cconsenrich.cgetFirstChromRead`, :func:`cconsenrich.cgetLastChromRead`
477
+ """
478
+ starts = []
479
+ ends = []
480
+ for bam_ in bamFiles:
481
+ start, end = getChromRanges(
482
+ bam_,
483
+ chromosome,
484
+ chromLength=chromSize,
485
+ samThreads=samThreads,
486
+ samFlagExclude=samFlagExclude,
487
+ )
488
+ starts.append(start)
489
+ ends.append(end)
490
+ return min(starts), max(ends)
491
+
492
+
493
+ def getReadLength(
494
+ bamFile: str,
495
+ numReads: int,
496
+ maxIterations: int,
497
+ samThreads: int,
498
+ samFlagExclude: int,
499
+ ) -> int:
500
+ r"""Infer read length from mapped reads in a BAM file.
501
+
502
+ Samples at least `numReads` reads passing criteria given by `samFlagExclude`
503
+ and returns the median read length.
504
+
505
+ :param bamFile: See :class:`inputParams`.
506
+ :type bamFile: str
507
+ :param numReads: Number of reads to sample.
508
+ :type numReads: int
509
+ :param maxIterations: Maximum number of iterations to perform.
510
+ :type maxIterations: int
511
+ :param samThreads: See :class:`samParams`.
512
+ :type samThreads: int
513
+ :param samFlagExclude: See :class:`samParams`.
514
+ :type samFlagExclude: int
515
+ :return: The median read length.
516
+ :rtype: int
517
+
518
+ :raises ValueError: If the read length cannot be determined after scanning `maxIterations` reads.
519
+
520
+ :seealso: :func:`cconsenrich.cgetReadLength`
521
+ """
522
+ init_rlen = cconsenrich.cgetReadLength(
523
+ bamFile, numReads, samThreads, maxIterations, samFlagExclude
524
+ )
525
+ if init_rlen == 0:
526
+ raise ValueError(
527
+ f"Failed to determine read length in {bamFile}. Revise `numReads`, and/or `samFlagExclude` parameters?"
528
+ )
529
+ return init_rlen
530
+
531
+
532
+ def getReadLengths(
533
+ bamFiles: List[str],
534
+ numReads: int,
535
+ maxIterations: int,
536
+ samThreads: int,
537
+ samFlagExclude: int,
538
+ ) -> List[int]:
539
+ r"""Get read lengths for a list of BAM files.
540
+
541
+ :seealso: :func:`getReadLength`
542
+ """
543
+ return [
544
+ getReadLength(
545
+ bamFile,
546
+ numReads=numReads,
547
+ maxIterations=maxIterations,
548
+ samThreads=samThreads,
549
+ samFlagExclude=samFlagExclude,
550
+ )
551
+ for bamFile in bamFiles
552
+ ]
553
+
554
+
555
+ def readBamSegments(
556
+ bamFiles: List[str],
557
+ chromosome: str,
558
+ start: int,
559
+ end: int,
560
+ stepSize: int,
561
+ readLengths: List[int],
562
+ scaleFactors: List[float],
563
+ oneReadPerBin: int,
564
+ samThreads: int,
565
+ samFlagExclude: int,
566
+ offsetStr: Optional[str] = "0,0",
567
+ applyAsinh: Optional[bool] = False,
568
+ applyLog: Optional[bool] = False,
569
+ extendBP: List[int] = [],
570
+ maxInsertSize: Optional[int] = 1000,
571
+ pairedEndMode: Optional[int] = 0,
572
+ inferFragmentLength: Optional[int] = 0,
573
+ countEndsOnly: Optional[bool] = False,
574
+ ) -> npt.NDArray[np.float32]:
575
+ r"""Calculate tracks of read counts (or a function thereof) for each BAM file.
576
+
577
+ See :func:`cconsenrich.creadBamSegment` for the underlying implementation in Cython.
578
+
579
+ :param bamFiles: See :class:`inputParams`.
580
+ :type bamFiles: List[str]
581
+ :param chromosome: Chromosome to read.
582
+ :type chromosome: str
583
+ :param start: Start position of the genomic segment.
584
+ :type start: int
585
+ :param end: End position of the genomic segment.
586
+ :type end: int
587
+ :param readLengths: List of read lengths for each BAM file.
588
+ :type readLengths: List[int]
589
+ :param scaleFactors: List of scale factors for each BAM file.
590
+ :type scaleFactors: List[float]
591
+ :param stepSize: See :class:`countingParams`.
592
+ :type stepSize: int
593
+ :param oneReadPerBin: See :class:`samParams`.
594
+ :type oneReadPerBin: int
595
+ :param samThreads: See :class:`samParams`.
596
+ :type samThreads: int
597
+ :param samFlagExclude: See :class:`samParams`.
598
+ :type samFlagExclude: int
599
+ :param offsetStr: See :class:`samParams`.
600
+ :type offsetStr: str
601
+ :param extendBP: See :class:`samParams`.
602
+ :type extendBP: int
603
+ :param maxInsertSize: See :class:`samParams`.
604
+ :type maxInsertSize: int
605
+ :param pairedEndMode: See :class:`samParams`.
606
+ :type pairedEndMode: int
607
+ :param inferFragmentLength: See :class:`samParams`.
608
+ :type inferFragmentLength: int
609
+ :param countEndsOnly: If True, only the 5' ends of reads are counted. This overrides `inferFragmentLength` and `pairedEndMode`.
610
+ :type countEndsOnly: Optional[bool]
611
+
612
+ """
613
+
614
+ if len(bamFiles) == 0:
615
+ raise ValueError("bamFiles list is empty")
616
+
617
+ if len(readLengths) != len(bamFiles) or len(scaleFactors) != len(
618
+ bamFiles
619
+ ):
620
+ raise ValueError(
621
+ "readLengths and scaleFactors must match bamFiles length"
622
+ )
623
+
624
+ extendBP = resolveExtendBP(extendBP, bamFiles)
625
+ offsetStr = ((str(offsetStr) or "0,0").replace(" ", "")).split(
626
+ ","
627
+ )
628
+ numIntervals = ((end - start) + stepSize - 1) // stepSize
629
+ counts = np.empty((len(bamFiles), numIntervals), dtype=np.float32)
630
+
631
+ if isinstance(countEndsOnly, bool) and countEndsOnly:
632
+ # note: setting this option ignores inferFragmentLength, pairedEndMode
633
+ inferFragmentLength = 0
634
+ pairedEndMode = 0
635
+
636
+ for j, bam in enumerate(bamFiles):
637
+ logger.info(f"Reading {chromosome}: {bam}")
638
+ arr = cconsenrich.creadBamSegment(
639
+ bam,
640
+ chromosome,
641
+ start,
642
+ end,
643
+ stepSize,
644
+ readLengths[j],
645
+ oneReadPerBin,
646
+ samThreads,
647
+ samFlagExclude,
648
+ int(offsetStr[0]),
649
+ int(offsetStr[1]),
650
+ extendBP[j],
651
+ maxInsertSize,
652
+ pairedEndMode,
653
+ inferFragmentLength,
654
+ )
655
+ # FFR: use ufuncs?
656
+ counts[j, :] = arr
657
+ counts[j, :] *= np.float32(scaleFactors[j])
658
+ if applyAsinh:
659
+ counts[j, :] = np.arcsinh(counts[j, :])
660
+ elif applyLog:
661
+ counts[j, :] = np.log1p(counts[j, :])
662
+ return counts
663
+
664
+
665
+
666
+ def getAverageLocalVarianceTrack(
667
+ values: np.ndarray,
668
+ stepSize: int,
669
+ approximationWindowLengthBP: int,
670
+ lowPassWindowLengthBP: int,
671
+ minR: float,
672
+ maxR: float,
673
+ lowPassFilterType: Optional[str] = "median",
674
+ ) -> npt.NDArray[np.float32]:
675
+ r"""Generate positional noise-level tracks with first and second moments approximated from local windows
676
+
677
+ First, computes a moving average of ``values`` using a bp-length window
678
+ ``approximationWindowLengthBP`` and a moving average of ``values**2`` over the
679
+ same window. Their difference is used to approximate the local variance. A low-pass filter
680
+ (median or mean) with window ``lowPassWindowLengthBP`` then smooths the variance track.
681
+ Finally, the track is clipped to ``[minR, maxR]`` to yield the local noise level track.
682
+
683
+ :param values: 1D array of read-density-based values for a single sample.
684
+ :type values: np.ndarray
685
+ :param stepSize: Bin size (bp).
686
+ :type stepSize: int
687
+ :param approximationWindowLengthBP: Window (bp) for local mean and second-moment. See :class:`observationParams`.
688
+ :type approximationWindowLengthBP: int
689
+ :param lowPassWindowLengthBP: Window (bp) for the low-pass filter on the variance track. See :class:`observationParams`.
690
+ :type lowPassWindowLengthBP: int
691
+ :param minR: Lower clip for the returned noise level. See :class:`observationParams`.
692
+ :type minR: float
693
+ :param maxR: Upper clip for the returned noise level. See :class:`observationParams`.
694
+ :type maxR: float
695
+ :param lowPassFilterType: ``"median"`` (default) or ``"mean"``. Type of low-pass filter to use for smoothing the local variance track. See :class:`observationParams`.
696
+ :type lowPassFilterType: Optional[str]
697
+ :return: Local noise level per interval.
698
+ :rtype: npt.NDArray[np.float32]
699
+
700
+ :seealso: :class:`observationParams`
701
+ """
702
+ values = np.asarray(values, dtype=np.float32)
703
+ windowLength = int(approximationWindowLengthBP / stepSize)
704
+ if windowLength % 2 == 0:
705
+ windowLength += 1
706
+ if len(values) < 3:
707
+ constVar = np.var(values)
708
+ if constVar < minR:
709
+ return np.full_like(values, minR, dtype=np.float32)
710
+ return np.full_like(values, constVar, dtype=np.float32)
711
+
712
+ # first get a simple moving average of the values
713
+ localMeanTrack: npt.NDArray[np.float32] = ndimage.uniform_filter(
714
+ values, size=windowLength, mode="nearest"
715
+ )
716
+
717
+ # ~ E[X_i^2] - E[X_i]^2 ~
718
+ localVarTrack: npt.NDArray[np.float32] = (
719
+ ndimage.uniform_filter(
720
+ values**2, size=windowLength, mode="nearest"
721
+ )
722
+ - localMeanTrack**2
723
+ )
724
+
725
+ # safe-guard: difference of convolutions returns negative values.
726
+ # shouldn't actually happen, but just in case there are some
727
+ # ...potential artifacts i'm unaware of edge effects, etc.
728
+ localVarTrack = np.maximum(localVarTrack, 0.0)
729
+
730
+ # low-pass filter on the local variance track: positional 'noise level' track
731
+ lpassWindowLength = int(lowPassWindowLengthBP / stepSize)
732
+ if lpassWindowLength % 2 == 0:
733
+ lpassWindowLength += 1
734
+
735
+ noiseLevel: npt.NDArray[np.float32] = np.zeros_like(
736
+ localVarTrack, dtype=np.float32
737
+ )
738
+ if lowPassFilterType is None or (
739
+ isinstance(lowPassFilterType, str)
740
+ and lowPassFilterType.lower() == "median"
741
+ ):
742
+ noiseLevel = ndimage.median_filter(
743
+ localVarTrack, size=lpassWindowLength
744
+ )
745
+ elif (
746
+ isinstance(lowPassFilterType, str)
747
+ and lowPassFilterType.lower() == "mean"
748
+ ):
749
+ noiseLevel = ndimage.uniform_filter(
750
+ localVarTrack, size=lpassWindowLength
751
+ )
752
+
753
+ return np.clip(noiseLevel, minR, maxR).astype(np.float32)
754
+
755
+
756
+ def constructMatrixF(deltaF: float) -> npt.NDArray[np.float32]:
757
+ r"""Build the state transition matrix for the process model
758
+
759
+ :param deltaF: See :class:`processParams`.
760
+ :type deltaF: float
761
+ :return: The state transition matrix :math:`\mathbf{F}`
762
+ :rtype: npt.NDArray[np.float32]
763
+
764
+ :seealso: :class:`processParams`
765
+ """
766
+ initMatrixF: npt.NDArray[np.float32] = np.eye(2, dtype=np.float32)
767
+ initMatrixF[0, 1] = np.float32(deltaF)
768
+ return initMatrixF
769
+
770
+
771
+ def constructMatrixQ(
772
+ minDiagQ: float, offDiagQ: float = 0.0
773
+ ) -> npt.NDArray[np.float32]:
774
+ r"""Build the initial process noise covariance matrix :math:`\mathbf{Q}_{[1]}`.
775
+
776
+ :param minDiagQ: See :class:`processParams`.
777
+ :type minDiagQ: float
778
+ :param offDiagQ: See :class:`processParams`.
779
+ :type offDiagQ: float
780
+ :return: The initial process noise covariance matrix :math:`\mathbf{Q}_{[1]}`.
781
+ :rtype: npt.NDArray[np.float32]
782
+
783
+ :seealso: :class:`processParams`
784
+ """
785
+ minDiagQ = np.float32(minDiagQ)
786
+ offDiagQ = np.float32(offDiagQ)
787
+ initMatrixQ: npt.NDArray[np.float32] = np.zeros(
788
+ (2, 2), dtype=np.float32
789
+ )
790
+ initMatrixQ[0, 0] = minDiagQ
791
+ initMatrixQ[1, 1] = minDiagQ
792
+ initMatrixQ[0, 1] = offDiagQ
793
+ initMatrixQ[1, 0] = offDiagQ
794
+ return initMatrixQ
795
+
796
+
797
+ def constructMatrixH(
798
+ m: int, coefficients: Optional[np.ndarray] = None
799
+ ) -> npt.NDArray[np.float32]:
800
+ r"""Build the observation model matrix :math:`\mathbf{H}`.
801
+
802
+ :param m: Number of observations.
803
+ :type m: int
804
+ :param coefficients: Optional coefficients for the observation model,
805
+ which can be used to weight the observations manually.
806
+ :type coefficients: Optional[np.ndarray]
807
+ :return: The observation model matrix :math:`\mathbf{H}`.
808
+ :rtype: npt.NDArray[np.float32]
809
+
810
+ :seealso: :class:`observationParams`, class:`inputParams`
811
+ """
812
+ if coefficients is None:
813
+ coefficients = np.ones(m, dtype=np.float32)
814
+ elif isinstance(coefficients, list):
815
+ coefficients = np.array(coefficients, dtype=np.float32)
816
+ initMatrixH = np.empty((m, 2), dtype=np.float32)
817
+ initMatrixH[:, 0] = coefficients.astype(np.float32)
818
+ initMatrixH[:, 1] = np.zeros(m, dtype=np.float32)
819
+ return initMatrixH
820
+
821
+
822
+ def runConsenrich(
823
+ matrixData: np.ndarray,
824
+ matrixMunc: np.ndarray,
825
+ deltaF: float,
826
+ minQ: float,
827
+ maxQ: float,
828
+ offDiagQ: float,
829
+ dStatAlpha: float,
830
+ dStatd: float,
831
+ dStatPC: float,
832
+ stateInit: float,
833
+ stateCovarInit: float,
834
+ boundState: bool,
835
+ stateLowerBound: float,
836
+ stateUpperBound: float,
837
+ chunkSize: int,
838
+ progressIter: int,
839
+ coefficientsH: Optional[np.ndarray] = None,
840
+ residualCovarInversionFunc: Optional[Callable] = None,
841
+ adjustProcessNoiseFunc: Optional[Callable] = None,
842
+ ) -> Tuple[
843
+ npt.NDArray[np.float32],
844
+ npt.NDArray[np.float32],
845
+ npt.NDArray[np.float32],
846
+ ]:
847
+ r"""Run consenrich on a contiguous segment (e.g. a chromosome) of read-density-based data.
848
+ Completes the forward and backward passes given data and approximated observation noise
849
+ covariance matrices :math:`\mathbf{R}_{[1:n, (11:mm)]}`.
850
+
851
+ This is the primary function implementing the core Consenrich algorithm. Users requiring specialized
852
+ preprocessing may prefer to call this function programmatically on their own preprocessed data rather
853
+ than using the command-line interface.
854
+
855
+
856
+ :param matrixData: Read density data for a single chromosome or general contiguous segment,
857
+ possibly preprocessed. Two-dimensional array of shape :math:`m \times n` where :math:`m`
858
+ is the number of samples/tracks and :math:`n` the number of genomic intervals.
859
+ :type matrixData: np.ndarray
860
+ :param matrixMunc: Uncertainty estimates for the read coverage data.
861
+ Two-dimensional array of shape :math:`m \times n` where :math:`m` is the number of samples/tracks
862
+ and :math:`n` the number of genomic intervals. See :func:`getMuncTrack`.
863
+ :type matrixMunc: np.ndarray
864
+ :param deltaF: See :class:`processParams`.
865
+ :type deltaF: float
866
+ :param minQ: See :class:`processParams`.
867
+ :type minQ: float
868
+ :param maxQ: See :class:`processParams`.
869
+ :type maxQ: float
870
+ :param offDiagQ: See :class:`processParams`.
871
+ :type offDiagQ: float
872
+ :param dStatAlpha: See :class:`processParams`.
873
+ :type dStatAlpha: float
874
+ :param dStatd: See :class:`processParams`.
875
+ :type dStatd: float
876
+ :param dStatPC: See :class:`processParams`.
877
+ :type dStatPC: float
878
+ :param stateInit: See :class:`stateParams`.
879
+ :type stateInit: float
880
+ :param stateCovarInit: See :class:`stateParams`.
881
+ :type stateCovarInit: float
882
+ :param chunkSize: Number of genomic intervals' data to keep in memory before flushing to disk.
883
+ :type chunkSize: int
884
+ :param progressIter: The number of iterations after which to log progress.
885
+ :type progressIter: int
886
+ :param coefficientsH: Optional coefficients for the observation model matrix :math:`\mathbf{H}`.
887
+ If None, the coefficients are set to 1.0 for all samples.
888
+ :type coefficientsH: Optional[np.ndarray]
889
+ :param residualCovarInversionFunc: Callable function to invert the observation covariance matrix :math:`\mathbf{E}_{[i]}`.
890
+ If None, defaults to :func:`cconsenrich.cinvertMatrixE`.
891
+ :type residualCovarInversionFunc: Optional[Callable]
892
+ :param adjustProcessNoiseFunc: Function to adjust the process noise covariance matrix :math:`\mathbf{Q}_{[i]}`.
893
+ If None, defaults to :func:`cconsenrich.updateProcessNoiseCovariance`.
894
+ :type adjustProcessNoiseFunc: Optional[Callable]
895
+ :return: Tuple of three numpy arrays:
896
+ - state estimates :math:`\widetilde{\mathbf{x}}_{[i]}` of shape :math:`n \times 2`
897
+ - state covariance estimates :math:`\widetilde{\mathbf{P}}_{[i]}` of shape :math:`n \times 2 \times 2`
898
+ - post-fit residuals :math:`\widetilde{\mathbf{y}}_{[i]}` of shape :math:`n \times m`
899
+ :rtype: Tuple[np.ndarray, np.ndarray, np.ndarray]
900
+
901
+ :raises ValueError: If the number of samples in `matrixData` is not equal to the number of samples in `matrixMunc`.
902
+ :seealso: :class:`observationParams`, :class:`processParams`, :class:`stateParams`
903
+ """
904
+ matrixData = np.ascontiguousarray(matrixData, dtype=np.float32)
905
+ matrixMunc = np.ascontiguousarray(matrixMunc, dtype=np.float32)
906
+ m: int = 1 if matrixData.ndim == 1 else matrixData.shape[0]
907
+ n: int = 1 if matrixData.ndim == 1 else matrixData.shape[1]
908
+ inflatedQ: bool = False
909
+ dStat: float = np.float32(0.0)
910
+ countAdjustments: int = 0
911
+
912
+ IKH: np.ndarray = np.zeros(shape=(2, 2), dtype=np.float32)
913
+ matrixEInverse: np.ndarray = np.zeros(
914
+ shape=(m, m), dtype=np.float32
915
+ )
916
+ matrixF: np.ndarray = constructMatrixF(deltaF)
917
+ matrixQ: np.ndarray = constructMatrixQ(minQ, offDiagQ=offDiagQ)
918
+ matrixQCopy: np.ndarray = matrixQ.copy()
919
+ matrixP: np.ndarray = np.eye(2, dtype=np.float32) * np.float32(
920
+ stateCovarInit
921
+ )
922
+ matrixH: np.ndarray = constructMatrixH(
923
+ m, coefficients=coefficientsH
924
+ )
925
+ matrixK: np.ndarray = np.zeros((2, m), dtype=np.float32)
926
+ vectorX: np.ndarray = np.array([stateInit, 0.0], dtype=np.float32)
927
+ vectorY: np.ndarray = np.zeros(m, dtype=np.float32)
928
+ matrixI2: np.ndarray = np.eye(2, dtype=np.float32)
929
+
930
+ if residualCovarInversionFunc is None:
931
+ residualCovarInversionFunc = cconsenrich.cinvertMatrixE
932
+ if adjustProcessNoiseFunc is None:
933
+ adjustProcessNoiseFunc = (
934
+ cconsenrich.updateProcessNoiseCovariance
935
+ )
936
+
937
+ # ==========================
938
+ # forward: 0,1,2,...,n-1
939
+ # ==========================
940
+ stateForward = np.memmap(
941
+ NamedTemporaryFile(delete=True),
942
+ dtype=np.float32,
943
+ mode="w+",
944
+ shape=(n, 2),
945
+ )
946
+ stateCovarForward = np.memmap(
947
+ NamedTemporaryFile(delete=True),
948
+ dtype=np.float32,
949
+ mode="w+",
950
+ shape=(n, 2, 2),
951
+ )
952
+ pNoiseForward = np.memmap(
953
+ NamedTemporaryFile(delete=True),
954
+ dtype=np.float32,
955
+ mode="w+",
956
+ shape=(n, 2, 2),
957
+ )
958
+ progressIter = max(1, progressIter)
959
+ for i in range(n):
960
+ if i % progressIter == 0:
961
+ logger.info(f"Forward pass interval: {i + 1}/{n}")
962
+ vectorZ = matrixData[:, i]
963
+ vectorX = matrixF @ vectorX
964
+ matrixP = matrixF @ matrixP @ matrixF.T + matrixQ
965
+ vectorY = vectorZ - (matrixH @ vectorX)
966
+ matrixEInverse = residualCovarInversionFunc(
967
+ matrixMunc[:, i], np.float32(matrixP[0, 0])
968
+ )
969
+ Einv_diag = np.diag(matrixEInverse)
970
+ dStat = np.median((vectorY**2) * Einv_diag)
971
+ countAdjustments = countAdjustments + int(dStat > dStatAlpha)
972
+ matrixQ, inflatedQ = adjustProcessNoiseFunc(
973
+ matrixQ,
974
+ matrixQCopy,
975
+ dStat,
976
+ dStatAlpha,
977
+ dStatd,
978
+ dStatPC,
979
+ inflatedQ,
980
+ maxQ,
981
+ minQ,
982
+ )
983
+ matrixK = (matrixP @ matrixH.T) @ matrixEInverse
984
+ IKH = matrixI2 - (matrixK @ matrixH)
985
+
986
+ vectorX = vectorX + (matrixK @ vectorY)
987
+ matrixP = (IKH) @ matrixP @ (IKH).T + (
988
+ matrixK * matrixMunc[:, i]
989
+ ) @ matrixK.T
990
+ stateForward[i] = vectorX.astype(np.float32)
991
+ stateCovarForward[i] = matrixP.astype(np.float32)
992
+ pNoiseForward[i] = matrixQ.astype(np.float32)
993
+
994
+ if i % chunkSize == 0 and i > 0:
995
+ stateForward.flush()
996
+ stateCovarForward.flush()
997
+ pNoiseForward.flush()
998
+
999
+ stateForward.flush()
1000
+ stateCovarForward.flush()
1001
+ pNoiseForward.flush()
1002
+ stateForwardArr = stateForward
1003
+ stateCovarForwardArr = stateCovarForward
1004
+ pNoiseForwardArr = pNoiseForward
1005
+
1006
+ # log num. times process noise was adjusted
1007
+ logger.info(
1008
+ f"`Median(normedInnovations) > α_D` triggered the adaptive procedure at [{round(((1.0 * countAdjustments) / n) * 100.0, 4)}%] of intervals"
1009
+ )
1010
+
1011
+
1012
+ # ==========================
1013
+ # backward: n,n-1,n-2,...,0
1014
+ # ==========================
1015
+ stateSmoothed = np.memmap(
1016
+ NamedTemporaryFile(delete=True),
1017
+ dtype=np.float32,
1018
+ mode="w+",
1019
+ shape=(n, 2),
1020
+ )
1021
+ stateCovarSmoothed = np.memmap(
1022
+ NamedTemporaryFile(delete=True),
1023
+ dtype=np.float32,
1024
+ mode="w+",
1025
+ shape=(n, 2, 2),
1026
+ )
1027
+ postFitResiduals = np.memmap(
1028
+ NamedTemporaryFile(delete=True),
1029
+ dtype=np.float32,
1030
+ mode="w+",
1031
+ shape=(n, m),
1032
+ )
1033
+
1034
+ stateSmoothed[-1] = np.float32(stateForwardArr[-1])
1035
+ stateCovarSmoothed[-1] = np.float32(stateCovarForwardArr[-1])
1036
+ postFitResiduals[-1] = np.float32(
1037
+ matrixData[:, -1] - (matrixH @ stateSmoothed[-1])
1038
+ )
1039
+
1040
+ for k in range(n - 2, -1, -1):
1041
+ if k % progressIter == 0:
1042
+ logger.info(f"Backward pass interval: {k + 1}/{n}")
1043
+ forwardStatePosterior = stateForwardArr[k]
1044
+ forwardCovariancePosterior = stateCovarForwardArr[k]
1045
+ backwardInitialState = matrixF @ forwardStatePosterior
1046
+ backwardInitialCovariance = (
1047
+ matrixF @ forwardCovariancePosterior @ matrixF.T
1048
+ + pNoiseForwardArr[k + 1]
1049
+ )
1050
+
1051
+ smootherGain = np.linalg.solve(
1052
+ backwardInitialCovariance.T,
1053
+ (forwardCovariancePosterior @ matrixF.T).T,
1054
+ ).T
1055
+ stateSmoothed[k] = (
1056
+ forwardStatePosterior
1057
+ + smootherGain
1058
+ @ (stateSmoothed[k + 1] - backwardInitialState)
1059
+ ).astype(np.float32)
1060
+
1061
+ stateCovarSmoothed[k] = (
1062
+ forwardCovariancePosterior
1063
+ + smootherGain
1064
+ @ (stateCovarSmoothed[k + 1] - backwardInitialCovariance)
1065
+ @ smootherGain.T
1066
+ ).astype(np.float32)
1067
+ postFitResiduals[k] = np.float32(
1068
+ matrixData[:, k] - matrixH @ stateSmoothed[k]
1069
+ )
1070
+
1071
+ if k % chunkSize == 0 and k > 0:
1072
+ stateSmoothed.flush()
1073
+ stateCovarSmoothed.flush()
1074
+ postFitResiduals.flush()
1075
+
1076
+ stateSmoothed.flush()
1077
+ stateCovarSmoothed.flush()
1078
+ postFitResiduals.flush()
1079
+ if boundState:
1080
+ stateSmoothed[:, 0] = np.clip(
1081
+ stateSmoothed[:, 0], stateLowerBound, stateUpperBound
1082
+ ).astype(np.float32)
1083
+
1084
+ return (
1085
+ stateSmoothed[:],
1086
+ stateCovarSmoothed[:],
1087
+ postFitResiduals[:],
1088
+ )
1089
+
1090
+
1091
+ def getPrimaryState(
1092
+ stateVectors: np.ndarray, roundPrecision: int = 3
1093
+ ) -> npt.NDArray[np.float32]:
1094
+ r"""Get the primary state estimate from each vector after running Consenrich.
1095
+
1096
+ :param stateVectors: State vectors from :func:`runConsenrich`.
1097
+ :type stateVectors: npt.NDArray[np.float32]
1098
+ :return: A one-dimensional numpy array of the primary state estimates.
1099
+ :rtype: npt.NDArray[np.float32]
1100
+ """
1101
+ out_ = np.ascontiguousarray(stateVectors[:, 0], dtype=np.float32)
1102
+ np.round(out_, decimals=roundPrecision, out=out_)
1103
+ return out_
1104
+
1105
+
1106
+ def getStateCovarTrace(
1107
+ stateCovarMatrices: np.ndarray, roundPrecision: int = 3
1108
+ ) -> npt.NDArray[np.float32]:
1109
+ r"""Get a one-dimensional array of state covariance traces after running Consenrich
1110
+
1111
+ :param stateCovarMatrices: Estimated state covariance matrices :math:`\widetilde{\mathbf{P}}_{[i]}`
1112
+ :type stateCovarMatrices: np.ndarray
1113
+ :return: A one-dimensional numpy array of the traces of the state covariance matrices.
1114
+ :rtype: npt.NDArray[np.float32]
1115
+ """
1116
+ stateCovarMatrices = np.ascontiguousarray(
1117
+ stateCovarMatrices, dtype=np.float32
1118
+ )
1119
+ out_ = cconsenrich.cgetStateCovarTrace(stateCovarMatrices)
1120
+ np.round(out_, decimals=roundPrecision, out=out_)
1121
+ return out_
1122
+
1123
+
1124
+ def getPrecisionWeightedResidual(
1125
+ postFitResiduals: np.ndarray,
1126
+ matrixMunc: np.ndarray,
1127
+ roundPrecision: int = 3,
1128
+ stateCovarSmoothed: Optional[np.ndarray] = None,
1129
+ ) -> npt.NDArray[np.float32]:
1130
+ r"""Get a one-dimensional precision-weighted array residuals after running Consenrich.
1131
+
1132
+ Applies an inverse-variance weighting of the post-fit residuals :math:`\widetilde{\mathbf{y}}_{[i]}` and
1133
+ returns a one-dimensional array of "precision-weighted residuals". The state-level uncertainty can also be
1134
+ incorporated given `stateCovarSmoothed`.
1135
+
1136
+ :param postFitResiduals: Post-fit residuals :math:`\widetilde{\mathbf{y}}_{[i]}` from :func:`runConsenrich`.
1137
+ :type postFitResiduals: np.ndarray
1138
+ :param matrixMunc: An :math:`m \times n` sample-by-interval matrix -- At genomic intervals :math:`i = 1,2,\ldots,n`, the respective length-:math:`m` column is :math:`\mathbf{R}_{[i,11:mm]}`.
1139
+ That is, the observation noise levels for each sample :math:`j=1,2,\ldots,m` at interval :math:`i`. To keep memory usage minimal `matrixMunc` is not returned in full or computed in
1140
+ in :func:`runConsenrich`. If using Consenrich programmatically, run :func:`consenrich.core.getMuncTrack` for each sample's count data (rows in the matrix output of :func:`readBamSegments`).
1141
+ :type matrixMunc: np.ndarray
1142
+ :param stateCovarSmoothed: Smoothed state covariance matrices :math:`\widetilde{\mathbf{P}}_{[i]}` from :func:`runConsenrich`.
1143
+ :type stateCovarSmoothed: Optional[np.ndarray]
1144
+ :return: A one-dimensional array of "precision-weighted residuals"
1145
+ :rtype: npt.NDArray[np.float32]
1146
+ """
1147
+
1148
+ n, m = postFitResiduals.shape
1149
+ if matrixMunc.shape != (m, n):
1150
+ raise ValueError(
1151
+ f"matrixMunc should be (m,n)=({m}, {n}): observed {matrixMunc.shape}"
1152
+ )
1153
+ if stateCovarSmoothed is not None and (
1154
+ stateCovarSmoothed.ndim < 3 or len(stateCovarSmoothed) != n
1155
+ ):
1156
+ raise ValueError(
1157
+ "stateCovarSmoothed must be shape (n) x (2,2) (if provided)"
1158
+ )
1159
+
1160
+ postFitResiduals_CContig = np.ascontiguousarray(
1161
+ postFitResiduals, dtype=np.float32
1162
+ )
1163
+
1164
+ needsCopy = (
1165
+ (stateCovarSmoothed is not None)
1166
+ and len(stateCovarSmoothed) == n
1167
+ ) or (not matrixMunc.flags.writeable)
1168
+
1169
+ matrixMunc_CContig = np.array(
1170
+ matrixMunc, dtype=np.float32, order="C", copy=needsCopy
1171
+ )
1172
+
1173
+ if needsCopy:
1174
+ # adds the 'primary' state uncertainty to observation noise covariance :math:`\mathbf{R}_{[i,:]}`
1175
+ # primary state uncertainty (0,0) :math:`\mathbf{P}_{[i]} \in \mathbb{R}^{2 \times 2}`
1176
+ stateCovarArr00 = np.asarray(
1177
+ stateCovarSmoothed[:, 0, 0], dtype=np.float32
1178
+ )
1179
+ matrixMunc_CContig += stateCovarArr00
1180
+
1181
+ np.maximum(
1182
+ matrixMunc_CContig, np.float32(1e-8), out=matrixMunc_CContig
1183
+ )
1184
+ out = cconsenrich.cgetPrecisionWeightedResidual(
1185
+ postFitResiduals_CContig, matrixMunc_CContig
1186
+ )
1187
+ np.round(out, decimals=roundPrecision, out=out)
1188
+ return out
1189
+
1190
+
1191
+ def getMuncTrack(
1192
+ chromosome: str,
1193
+ intervals: np.ndarray,
1194
+ stepSize: int,
1195
+ rowValues: np.ndarray,
1196
+ minR: float,
1197
+ maxR: float,
1198
+ useALV: bool,
1199
+ useConstantNoiseLevel: bool,
1200
+ noGlobal: bool,
1201
+ localWeight: float,
1202
+ globalWeight: float,
1203
+ approximationWindowLengthBP: int,
1204
+ lowPassWindowLengthBP: int,
1205
+ returnCenter: bool,
1206
+ sparseMap: Optional[dict[int, int]] = None,
1207
+ lowPassFilterType: Optional[str] = "median",
1208
+ ) -> npt.NDArray[np.float32]:
1209
+ r"""Get observation noise variance :math:`R_{[:,jj]}` for the sample :math:`j`.
1210
+
1211
+ Combines a local ALV estimate (see :func:`getAverageLocalVarianceTrack`) with an
1212
+ optional global component. If ``useALV`` is True, *only* the ALV is used. If
1213
+ ``useConstantNoiseLevel`` is True, a constant track set to the global mean is used.
1214
+ When a ``sparseMap`` is provided, local values are aggregated over nearby 'sparse'
1215
+ regions before mixing with the global component.
1216
+
1217
+ For heterochromatic or repressive marks (H3K9me3, H3K27me3, MNase-seq, etc.), consider setting
1218
+ `useALV=True` to prevent inflated sample-level noise estimates.
1219
+
1220
+ :param chromosome: Tracks are approximated for this chromosome.
1221
+ :type chromosome: str
1222
+ :param intervals: Genomic intervals for which to compute the noise track.
1223
+ :param stepSize: See :class:`countingParams`.
1224
+ :type stepSize: int
1225
+ :param rowValues: Read-density-based values for the sample :math:`j` at the genomic intervals :math:`i=1,2,\ldots,n`.
1226
+ :type rowValues: np.ndarray
1227
+ :param minR: See :class:`observationParams`.
1228
+ :type minR: float
1229
+ :param maxR: See :class:`observationParams`.
1230
+ :type maxR: float
1231
+ :param useALV: See :class:`observationParams`.
1232
+ :type useALV: bool
1233
+ :param useConstantNoiseLevel: See :class:`observationParams`.
1234
+ :type useConstantNoiseLevel: bool
1235
+ :param noGlobal: See :class:`observationParams`.
1236
+ :type noGlobal: bool
1237
+ :param localWeight: See :class:`observationParams`.
1238
+ :type localWeight: float
1239
+ :param globalWeight: See :class:`observationParams`.
1240
+ :type globalWeight: float
1241
+ :param approximationWindowLengthBP: See :class:`observationParams` and/or :func:`getAverageLocalVarianceTrack`.
1242
+ :type approximationWindowLengthBP: int
1243
+ :param lowPassWindowLengthBP: See :class:`observationParams` and/or :func:`getAverageLocalVarianceTrack`.
1244
+ :type lowPassWindowLengthBP: int
1245
+ :param sparseMap: Optional mapping (dictionary) of interval indices to the nearest sparse regions. See :func:`getSparseMap`.
1246
+ :type sparseMap: Optional[dict[int, int]]
1247
+ :param lowPassFilterType: The type of low-pass filter to use in average local variance track (e.g., 'median', 'mean').
1248
+ :type lowPassFilterType: Optional[str]
1249
+ :return: A one-dimensional numpy array of the observation noise track for the sample :math:`j`.
1250
+ :rtype: npt.NDArray[np.float32]
1251
+
1252
+ """
1253
+ trackALV = getAverageLocalVarianceTrack(
1254
+ rowValues,
1255
+ stepSize,
1256
+ approximationWindowLengthBP,
1257
+ lowPassWindowLengthBP,
1258
+ minR,
1259
+ maxR,
1260
+ lowPassFilterType,
1261
+ ).astype(np.float32)
1262
+
1263
+ globalNoise: float = np.float32(np.mean(trackALV))
1264
+ if noGlobal or globalWeight == 0 or useALV:
1265
+ return np.clip(trackALV, minR, maxR).astype(np.float32)
1266
+
1267
+ if (
1268
+ useConstantNoiseLevel
1269
+ or localWeight == 0
1270
+ and sparseMap is None
1271
+ ):
1272
+ return np.clip(
1273
+ globalNoise * np.ones_like(rowValues), minR, maxR
1274
+ ).astype(np.float32)
1275
+
1276
+ if sparseMap is not None:
1277
+ trackALV = cconsenrich.cSparseAvg(trackALV, sparseMap)
1278
+
1279
+ return np.clip(
1280
+ trackALV * localWeight + np.mean(trackALV) * globalWeight,
1281
+ minR,
1282
+ maxR,
1283
+ ).astype(np.float32)
1284
+
1285
+
1286
+ def sparseIntersection(
1287
+ chromosome: str, intervals: np.ndarray, sparseBedFile: str
1288
+ ) -> npt.NDArray[np.int64]:
1289
+ r"""Returns intervals in the chromosome that overlap with the sparse features.
1290
+
1291
+ Not relevant if `observationParams.useALV` is True.
1292
+
1293
+ :param chromosome: The chromosome name.
1294
+ :type chromosome: str
1295
+ :param intervals: The genomic intervals to consider.
1296
+ :type intervals: np.ndarray
1297
+ :param sparseBedFile: Path to the sparse BED file.
1298
+ :type sparseBedFile: str
1299
+ :return: A numpy array of start positions of the sparse features that overlap with the intervals
1300
+ :rtype: np.ndarray[Tuple[Any], np.dtype[Any]]
1301
+ """
1302
+
1303
+ stepSize: int = intervals[1] - intervals[0]
1304
+ chromFeatures: bed.BedTool = (
1305
+ bed.BedTool(sparseBedFile)
1306
+ .sort()
1307
+ .merge()
1308
+ .filter(
1309
+ lambda b: (
1310
+ b.chrom == chromosome
1311
+ and b.start > intervals[0]
1312
+ and b.end < intervals[-1]
1313
+ and (b.end - b.start) >= stepSize
1314
+ )
1315
+ )
1316
+ )
1317
+ centeredFeatures: bed.BedTool = chromFeatures.each(
1318
+ adjustFeatureBounds, stepSize=stepSize
1319
+ )
1320
+
1321
+ start0: int = int(intervals[0])
1322
+ last: int = int(intervals[-1])
1323
+ chromFeatures: bed.BedTool = (
1324
+ bed.BedTool(sparseBedFile)
1325
+ .sort()
1326
+ .merge()
1327
+ .filter(
1328
+ lambda b: (
1329
+ b.chrom == chromosome
1330
+ and b.start > start0
1331
+ and b.end < last
1332
+ and (b.end - b.start) >= stepSize
1333
+ )
1334
+ )
1335
+ )
1336
+ centeredFeatures: bed.BedTool = chromFeatures.each(
1337
+ adjustFeatureBounds, stepSize=stepSize
1338
+ )
1339
+ centeredStarts = []
1340
+ for f in centeredFeatures:
1341
+ s = int(f.start)
1342
+ if start0 <= s <= last and (s - start0) % stepSize == 0:
1343
+ centeredStarts.append(s)
1344
+ return np.asarray(centeredStarts, dtype=np.int64)
1345
+
1346
+
1347
+ def adjustFeatureBounds(
1348
+ feature: bed.Interval, stepSize: int
1349
+ ) -> bed.Interval:
1350
+ r"""Adjust the start and end positions of a BED feature to be centered around a step."""
1351
+ feature.start = cconsenrich.stepAdjustment(
1352
+ (feature.start + feature.end) // 2, stepSize
1353
+ )
1354
+ feature.end = feature.start + stepSize
1355
+ return feature
1356
+
1357
+
1358
+ def getSparseMap(
1359
+ chromosome: str,
1360
+ intervals: np.ndarray,
1361
+ numNearest: int,
1362
+ sparseBedFile: str,
1363
+ ) -> dict:
1364
+ r"""Build a map between each genomic interval and numNearest sparse features
1365
+
1366
+ :param chromosome: The chromosome name. Note, this function only needs to be run once per chromosome.
1367
+ :type chromosome: str
1368
+ :param intervals: The genomic intervals to map.
1369
+ :type intervals: np.ndarray
1370
+ :param numNearest: The number of nearest sparse features to consider
1371
+ :type numNearest: int
1372
+ :param sparseBedFile: path to the sparse BED file.
1373
+ :type sparseBedFile: str
1374
+ :return: A dictionary mapping each interval index to the indices of the nearest sparse regions.
1375
+ :rtype: dict[int, np.ndarray]
1376
+
1377
+ """
1378
+ numNearest = numNearest
1379
+ sparseStarts = sparseIntersection(
1380
+ chromosome, intervals, sparseBedFile
1381
+ )
1382
+ idxSparseInIntervals = np.searchsorted(
1383
+ intervals, sparseStarts, side="left"
1384
+ )
1385
+ centers = np.searchsorted(sparseStarts, intervals, side="left")
1386
+ sparseMap: dict = {}
1387
+ for i, (interval, center) in enumerate(zip(intervals, centers)):
1388
+ left = max(0, center - numNearest)
1389
+ right = min(len(sparseStarts), center + numNearest)
1390
+ candidates = np.arange(left, right)
1391
+ dists = np.abs(sparseStarts[candidates] - interval)
1392
+ take = np.argsort(dists)[:numNearest]
1393
+ sparseMap[i] = idxSparseInIntervals[candidates[take]]
1394
+ return sparseMap
1395
+
1396
+
1397
+ def getBedMask(
1398
+ chromosome: str,
1399
+ bedFile: str,
1400
+ intervals: np.ndarray,
1401
+ ) -> np.ndarray:
1402
+ r"""Return a 1/0 mask for intervals overlapping a sorted and merged BED file.
1403
+
1404
+ This function is a wrapper for :func:`cconsenrich.cbedMask`.
1405
+
1406
+ :param chromosome: The chromosome name.
1407
+ :type chromosome: str
1408
+ :param intervals: chromosome-specific, sorted, non-overlapping start positions of genomic intervals.
1409
+ Each interval is assumed `stepSize`.
1410
+ :type intervals: np.ndarray
1411
+ :param bedFile: Path to a sorted and merged BED file
1412
+ :type bedFile: str
1413
+ :return: An `intervals`-length mask s.t. True indicates the interval overlaps a feature in the BED file.
1414
+ :rtype: np.ndarray
1415
+ """
1416
+ if not os.path.exists(bedFile):
1417
+ raise ValueError(f"Could not find {bedFile}")
1418
+ if len(intervals) < 2:
1419
+ raise ValueError(
1420
+ "intervals must contain at least two positions"
1421
+ )
1422
+ bedFile_ = str(bedFile)
1423
+
1424
+ # (possibly redundant) creation of uint32 version
1425
+ # + quick check for constant steps
1426
+ intervals_ = np.asarray(intervals, dtype=np.uint32)
1427
+ if (intervals_[1] - intervals_[0]) != (
1428
+ intervals_[-1] - intervals_[-2]
1429
+ ):
1430
+ raise ValueError("Intervals are not fixed in size")
1431
+
1432
+ stepSize_: int = intervals[1] - intervals[0]
1433
+ return cconsenrich.cbedMask(
1434
+ chromosome,
1435
+ bedFile_,
1436
+ intervals_,
1437
+ stepSize_,
1438
+ ).astype(np.bool_)