ldcov 0.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ldcov/__init__.py +49 -0
- ldcov/cli/__init__.py +3 -0
- ldcov/cli/commands.py +310 -0
- ldcov/cli/main.py +34 -0
- ldcov/compute/__init__.py +10 -0
- ldcov/compute/correlation.py +239 -0
- ldcov/compute/covariate.py +315 -0
- ldcov/compute/projection.py +335 -0
- ldcov/io/__init__.py +14 -0
- ldcov/io/bcor_file_handle.py +152 -0
- ldcov/io/bcor_index.py +331 -0
- ldcov/io/bcor_reader.py +794 -0
- ldcov/io/bcor_writer.py +394 -0
- ldcov/io/blockmatrix/__init__.py +6 -0
- ldcov/io/blockmatrix/block_codec.py +48 -0
- ldcov/io/blockmatrix/partitioner.py +40 -0
- ldcov/io/blockmatrix/reader.py +62 -0
- ldcov/io/correlation_io.py +380 -0
- ldcov/io/covariate_loader.py +120 -0
- ldcov/io/fs_utils.py +21 -0
- ldcov/io/variant_index.py +50 -0
- ldcov/ld_bm/__init__.py +5 -0
- ldcov/ld_bm/cli.py +106 -0
- ldcov/ld_bm/extract.py +300 -0
- ldcov/utils/__init__.py +7 -0
- ldcov/utils/categorical_utils.py +88 -0
- ldcov/utils/region_utils.py +35 -0
- ldcov/utils/variant_filter.py +179 -0
- ldcov-0.4.0.dist-info/METADATA +450 -0
- ldcov-0.4.0.dist-info/RECORD +36 -0
- ldcov-0.4.0.dist-info/WHEEL +5 -0
- ldcov-0.4.0.dist-info/entry_points.txt +2 -0
- ldcov-0.4.0.dist-info/licenses/LICENSE +21 -0
- ldcov-0.4.0.dist-info/scm_file_list.json +86 -0
- ldcov-0.4.0.dist-info/scm_version.json +8 -0
- ldcov-0.4.0.dist-info/top_level.txt +1 -0
ldcov/__init__.py
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
"""
|
|
2
|
+
ldcov: A Python package for efficient linkage disequilibrium calculation with covariate adjustment.
|
|
3
|
+
|
|
4
|
+
This package provides tools for:
|
|
5
|
+
1. LD calculation for large BGEN genotype datasets
|
|
6
|
+
2. Covariate adjustment of genotypes using FWL projection
|
|
7
|
+
3. Efficient handling of BGEN files with support for regions and indexing
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from importlib.metadata import version as _version, PackageNotFoundError as _PackageNotFoundError
|
|
11
|
+
|
|
12
|
+
try:
|
|
13
|
+
__version__ = _version("ldcov")
|
|
14
|
+
except _PackageNotFoundError: # package not installed (e.g. running from a source tree)
|
|
15
|
+
__version__ = "0.0.0+unknown"
|
|
16
|
+
|
|
17
|
+
# Lazy import support for faster startup
|
|
18
|
+
_LAZY_IMPORTS = {
|
|
19
|
+
"load_and_adjust_genotypes": ".compute.correlation",
|
|
20
|
+
"compute_ld_from_standardized": ".compute.correlation",
|
|
21
|
+
"compute_correlation_matrix": ".compute.correlation",
|
|
22
|
+
"regress_out_covariates": ".compute.covariate",
|
|
23
|
+
"standardize_genotypes": ".compute.covariate",
|
|
24
|
+
"load_bgen": ".io",
|
|
25
|
+
"load_covariates": ".io.covariate_loader",
|
|
26
|
+
"save_correlation_matrix": ".io.correlation_io",
|
|
27
|
+
"load_correlation_matrix": ".io.correlation_io",
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def __getattr__(name):
|
|
32
|
+
"""Lazy loading of module attributes."""
|
|
33
|
+
if name in _LAZY_IMPORTS:
|
|
34
|
+
module_path = _LAZY_IMPORTS[name]
|
|
35
|
+
if module_path.startswith("."):
|
|
36
|
+
# Relative import
|
|
37
|
+
import importlib
|
|
38
|
+
|
|
39
|
+
module = importlib.import_module(module_path, package=__name__)
|
|
40
|
+
else:
|
|
41
|
+
# Absolute import
|
|
42
|
+
import importlib
|
|
43
|
+
|
|
44
|
+
module = importlib.import_module(module_path)
|
|
45
|
+
attr = getattr(module, name)
|
|
46
|
+
# Cache it for future use
|
|
47
|
+
globals()[name] = attr
|
|
48
|
+
return attr
|
|
49
|
+
raise AttributeError(f"module '{__name__}' has no attribute '{name}'")
|
ldcov/cli/__init__.py
ADDED
ldcov/cli/commands.py
ADDED
|
@@ -0,0 +1,310 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Command-line interface for the ldcov package.
|
|
3
|
+
|
|
4
|
+
This module provides a command-line interface for computing
|
|
5
|
+
linkage disequilibrium and adjusting genotypes with ldcov.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import argparse
|
|
9
|
+
import logging
|
|
10
|
+
import sys
|
|
11
|
+
import os
|
|
12
|
+
from typing import Optional, List # noqa: F401
|
|
13
|
+
|
|
14
|
+
from ..compute.correlation import (
|
|
15
|
+
load_and_adjust_genotypes,
|
|
16
|
+
compute_ld_from_standardized,
|
|
17
|
+
)
|
|
18
|
+
from ldcov.ld_bm.cli import add_ld_bm_arguments, run_ld_bm
|
|
19
|
+
|
|
20
|
+
# Configure logging
|
|
21
|
+
logging.basicConfig(
|
|
22
|
+
level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
|
|
23
|
+
)
|
|
24
|
+
logger = logging.getLogger(__name__)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def parse_args():
|
|
28
|
+
"""
|
|
29
|
+
Parse command-line arguments.
|
|
30
|
+
|
|
31
|
+
Returns:
|
|
32
|
+
--------
|
|
33
|
+
argparse.Namespace
|
|
34
|
+
Parsed arguments
|
|
35
|
+
"""
|
|
36
|
+
parser = argparse.ArgumentParser(
|
|
37
|
+
description="ldcov: Compute linkage disequilibrium with optional covariate adjustment.",
|
|
38
|
+
epilog="""
|
|
39
|
+
Examples:
|
|
40
|
+
# Compute LD only (no covariate adjustment)
|
|
41
|
+
ldcov --bgen input.bgen --out output --compute-ld
|
|
42
|
+
|
|
43
|
+
# Compute LD with covariate adjustment
|
|
44
|
+
ldcov --bgen input.bgen --out output --compute-ld -c covariates.txt
|
|
45
|
+
|
|
46
|
+
# Use specific columns as covariates
|
|
47
|
+
ldcov --bgen input.bgen --out output --compute-ld -c covariates.txt --covariate-cols PC1 PC2 PC3
|
|
48
|
+
|
|
49
|
+
# Pre-compute projection matrix for later use
|
|
50
|
+
ldcov --precompute-projection -c covariates.txt --sample data.sample --out myproject
|
|
51
|
+
|
|
52
|
+
# Use pre-computed projection matrix
|
|
53
|
+
ldcov --bgen input.bgen --projection-matrix myproject.proj.npz --compute-ld --out output
|
|
54
|
+
|
|
55
|
+
# Compute LD and save projection matrix for future use
|
|
56
|
+
ldcov --bgen input.bgen -c covariates.txt --compute-ld --save-projection --out output
|
|
57
|
+
""",
|
|
58
|
+
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
# Required arguments (bgen is conditionally required)
|
|
62
|
+
parser.add_argument("--bgen", help="Input BGEN genotype file")
|
|
63
|
+
|
|
64
|
+
parser.add_argument(
|
|
65
|
+
"--out",
|
|
66
|
+
required=True,
|
|
67
|
+
help="Output file prefix (extensions will be added automatically)",
|
|
68
|
+
)
|
|
69
|
+
|
|
70
|
+
# Mode flags (at least one required)
|
|
71
|
+
parser.add_argument(
|
|
72
|
+
"--compute-ld", action="store_true", help="Compute linkage disequilibrium matrix"
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
parser.add_argument(
|
|
76
|
+
"--precompute-projection",
|
|
77
|
+
action="store_true",
|
|
78
|
+
help="Pre-compute projection matrix from covariates for later use",
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
# Optional arguments
|
|
82
|
+
parser.add_argument("--bgi", help="Path to BGEN index file (.bgi, optional)")
|
|
83
|
+
|
|
84
|
+
parser.add_argument("--covariates", "-c", help="Covariate file for adjustment")
|
|
85
|
+
|
|
86
|
+
parser.add_argument(
|
|
87
|
+
"--projection-matrix",
|
|
88
|
+
help="Pre-computed projection matrix file (.proj.npz) to use instead of covariates",
|
|
89
|
+
)
|
|
90
|
+
|
|
91
|
+
parser.add_argument(
|
|
92
|
+
"--save-projection",
|
|
93
|
+
action="store_true",
|
|
94
|
+
help="Save projection matrix when computing LD or adjusting genotypes",
|
|
95
|
+
)
|
|
96
|
+
|
|
97
|
+
parser.add_argument(
|
|
98
|
+
"--covariate-id-col",
|
|
99
|
+
default="IID",
|
|
100
|
+
help="Column name for sample IDs in covariate file (default: IID)",
|
|
101
|
+
)
|
|
102
|
+
|
|
103
|
+
parser.add_argument(
|
|
104
|
+
"--covariate-cols",
|
|
105
|
+
nargs="+",
|
|
106
|
+
help="Specific columns to use as covariates (default: all columns except ID)",
|
|
107
|
+
)
|
|
108
|
+
|
|
109
|
+
parser.add_argument("--region", "-r", help='Genomic region in format "chr:start-end"')
|
|
110
|
+
|
|
111
|
+
parser.add_argument(
|
|
112
|
+
"--output-format",
|
|
113
|
+
choices=["matrix", "long", "bcor", "npz", "both"],
|
|
114
|
+
default="matrix",
|
|
115
|
+
help="Output format for LD matrix (default: matrix)",
|
|
116
|
+
)
|
|
117
|
+
|
|
118
|
+
parser.add_argument(
|
|
119
|
+
"--no-bcor-idx",
|
|
120
|
+
dest="bcor_index",
|
|
121
|
+
action="store_false",
|
|
122
|
+
default=True,
|
|
123
|
+
help="Skip writing .bcor.idx sidecar (only meaningful with --output-format bcor).",
|
|
124
|
+
)
|
|
125
|
+
|
|
126
|
+
parser.add_argument("--sample", help="Path to sample file (.sample, optional)")
|
|
127
|
+
|
|
128
|
+
parser.add_argument(
|
|
129
|
+
"--z",
|
|
130
|
+
"--z-file",
|
|
131
|
+
help="Path to .z file specifying variants to extract and their order (optional)",
|
|
132
|
+
)
|
|
133
|
+
|
|
134
|
+
parser.add_argument("--verbose", "-v", action="store_true", help="Enable verbose logging")
|
|
135
|
+
|
|
136
|
+
parser.add_argument(
|
|
137
|
+
"--progress",
|
|
138
|
+
action="store_true",
|
|
139
|
+
help="Enable progress bars during loading (disabled by default)",
|
|
140
|
+
)
|
|
141
|
+
|
|
142
|
+
parser.add_argument(
|
|
143
|
+
"--nan-action",
|
|
144
|
+
choices=["error", "mean", "omit"],
|
|
145
|
+
default="error",
|
|
146
|
+
help="Action for handling NaN values in genotypes: "
|
|
147
|
+
"'error' (default) - raise error if NaN values are found; "
|
|
148
|
+
"'mean' - impute NaN values with variant mean; "
|
|
149
|
+
"'omit' - remove samples with NaN values (with warnings)",
|
|
150
|
+
)
|
|
151
|
+
|
|
152
|
+
add_ld_bm_arguments(parser)
|
|
153
|
+
return parser.parse_args()
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def validate_args(args):
|
|
157
|
+
"""
|
|
158
|
+
Validate command-line arguments based on flags.
|
|
159
|
+
|
|
160
|
+
Parameters:
|
|
161
|
+
-----------
|
|
162
|
+
args : argparse.Namespace
|
|
163
|
+
Parsed arguments
|
|
164
|
+
|
|
165
|
+
Raises:
|
|
166
|
+
-------
|
|
167
|
+
ValueError
|
|
168
|
+
If arguments are invalid for the selected options
|
|
169
|
+
"""
|
|
170
|
+
# At least one mode flag must be specified (--ld-bm is its own mode, skip this check)
|
|
171
|
+
if not getattr(args, "ld_bm", False) and not args.compute_ld and not args.precompute_projection:
|
|
172
|
+
raise ValueError(
|
|
173
|
+
"At least one of --compute-ld or --precompute-projection must be specified"
|
|
174
|
+
)
|
|
175
|
+
|
|
176
|
+
# npz/both output formats are only valid for --ld-bm (the BGEN LD flow supports matrix/long/bcor)
|
|
177
|
+
if not getattr(args, "ld_bm", False) and args.output_format in ("npz", "both"):
|
|
178
|
+
raise ValueError(
|
|
179
|
+
f"--output-format {args.output_format} is only valid with --ld-bm; "
|
|
180
|
+
"use matrix, long, or bcor for --compute-ld"
|
|
181
|
+
)
|
|
182
|
+
|
|
183
|
+
# Precompute projection requires covariates
|
|
184
|
+
if args.precompute_projection:
|
|
185
|
+
if not args.covariates:
|
|
186
|
+
raise ValueError("--covariates is required when using --precompute-projection")
|
|
187
|
+
if args.bgen:
|
|
188
|
+
raise ValueError("--bgen should not be specified with --precompute-projection")
|
|
189
|
+
|
|
190
|
+
# Cannot use both covariates and projection matrix
|
|
191
|
+
if args.covariates and args.projection_matrix:
|
|
192
|
+
raise ValueError("Cannot specify both --covariates and --projection-matrix")
|
|
193
|
+
|
|
194
|
+
# Save projection only makes sense with covariates
|
|
195
|
+
if args.save_projection and not args.covariates:
|
|
196
|
+
raise ValueError("--save-projection requires --covariates")
|
|
197
|
+
|
|
198
|
+
# BGEN is required for compute-ld
|
|
199
|
+
if args.compute_ld and not args.bgen:
|
|
200
|
+
raise ValueError("--bgen is required for --compute-ld")
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def run_cli():
|
|
204
|
+
"""
|
|
205
|
+
Main CLI entry point.
|
|
206
|
+
"""
|
|
207
|
+
# Parse arguments
|
|
208
|
+
args = parse_args()
|
|
209
|
+
|
|
210
|
+
# Configure logging
|
|
211
|
+
if args.verbose:
|
|
212
|
+
logging.getLogger().setLevel(logging.DEBUG)
|
|
213
|
+
|
|
214
|
+
# Validate arguments
|
|
215
|
+
try:
|
|
216
|
+
validate_args(args)
|
|
217
|
+
except ValueError as e:
|
|
218
|
+
logger.error(f"Invalid arguments: {e}")
|
|
219
|
+
sys.exit(1)
|
|
220
|
+
|
|
221
|
+
# Create output directory if it doesn't exist
|
|
222
|
+
output_dir = os.path.dirname(os.path.abspath(args.out))
|
|
223
|
+
if output_dir and not os.path.exists(output_dir):
|
|
224
|
+
os.makedirs(output_dir, exist_ok=True)
|
|
225
|
+
|
|
226
|
+
if getattr(args, "ld_bm", False):
|
|
227
|
+
run_ld_bm(args)
|
|
228
|
+
return
|
|
229
|
+
|
|
230
|
+
# Handle precompute projection mode
|
|
231
|
+
if args.precompute_projection:
|
|
232
|
+
from ..compute.projection import compute_projection_matrix, save_projection_matrix
|
|
233
|
+
|
|
234
|
+
logger.info("Pre-computing projection matrix from covariates")
|
|
235
|
+
projection_data = compute_projection_matrix(
|
|
236
|
+
covariate_file=args.covariates,
|
|
237
|
+
sample_file=args.sample,
|
|
238
|
+
covariate_id_col=args.covariate_id_col,
|
|
239
|
+
covariate_cols=args.covariate_cols,
|
|
240
|
+
)
|
|
241
|
+
|
|
242
|
+
output_file = f"{args.out}.proj.npz"
|
|
243
|
+
save_projection_matrix(projection_data, output_file)
|
|
244
|
+
logger.info(f"Projection matrix saved to {output_file}")
|
|
245
|
+
return
|
|
246
|
+
|
|
247
|
+
# For other modes, we need BGEN file
|
|
248
|
+
# Auto-detect BGI file if not specified
|
|
249
|
+
bgi_file = args.bgi
|
|
250
|
+
if bgi_file is None and args.bgen:
|
|
251
|
+
potential_bgi = f"{args.bgen}.bgi"
|
|
252
|
+
if os.path.exists(potential_bgi):
|
|
253
|
+
logger.info(f"Using auto-detected BGI file: {potential_bgi}")
|
|
254
|
+
bgi_file = potential_bgi
|
|
255
|
+
|
|
256
|
+
# Load and adjust genotypes (common step for compute-ld and export-adjusted-bgen)
|
|
257
|
+
result = load_and_adjust_genotypes(
|
|
258
|
+
genotype_file=args.bgen,
|
|
259
|
+
covariate_file=args.covariates,
|
|
260
|
+
projection_matrix_file=args.projection_matrix,
|
|
261
|
+
region=args.region,
|
|
262
|
+
index_file=bgi_file,
|
|
263
|
+
sample_file=args.sample,
|
|
264
|
+
z_file=args.z,
|
|
265
|
+
covariate_id_col=args.covariate_id_col,
|
|
266
|
+
covariate_cols=args.covariate_cols,
|
|
267
|
+
show_progress=args.progress,
|
|
268
|
+
nan_action=args.nan_action,
|
|
269
|
+
)
|
|
270
|
+
standardized_genotypes, variant_info, sample_ids, means, norms = result
|
|
271
|
+
|
|
272
|
+
# Save projection matrix if requested and covariates were used
|
|
273
|
+
if args.save_projection and args.covariates:
|
|
274
|
+
from ..compute.projection import compute_projection_matrix, save_projection_matrix
|
|
275
|
+
|
|
276
|
+
logger.info("Computing and saving projection matrix")
|
|
277
|
+
projection_data = compute_projection_matrix(
|
|
278
|
+
covariate_file=args.covariates,
|
|
279
|
+
sample_ids=sample_ids,
|
|
280
|
+
covariate_id_col=args.covariate_id_col,
|
|
281
|
+
covariate_cols=args.covariate_cols,
|
|
282
|
+
)
|
|
283
|
+
|
|
284
|
+
projection_output_file = f"{args.out}.proj.npz"
|
|
285
|
+
save_projection_matrix(projection_data, projection_output_file)
|
|
286
|
+
logger.info(f"Projection matrix saved to {projection_output_file}")
|
|
287
|
+
|
|
288
|
+
# Compute LD if requested
|
|
289
|
+
if args.compute_ld:
|
|
290
|
+
# Determine LD output file extension based on format
|
|
291
|
+
if args.output_format == "matrix":
|
|
292
|
+
ld_output_file = f"{args.out}.ld"
|
|
293
|
+
elif args.output_format == "long":
|
|
294
|
+
ld_output_file = f"{args.out}.ld.gz"
|
|
295
|
+
elif args.output_format == "bcor":
|
|
296
|
+
ld_output_file = f"{args.out}.bcor"
|
|
297
|
+
|
|
298
|
+
logger.info("Computing LD matrix")
|
|
299
|
+
compute_ld_from_standardized(
|
|
300
|
+
standardized_genotypes=standardized_genotypes,
|
|
301
|
+
variant_info=variant_info,
|
|
302
|
+
output_file=ld_output_file,
|
|
303
|
+
output_format=args.output_format,
|
|
304
|
+
write_index=args.bcor_index,
|
|
305
|
+
)
|
|
306
|
+
logger.info(f"LD matrix saved to {ld_output_file}")
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
if __name__ == "__main__":
|
|
310
|
+
run_cli()
|
ldcov/cli/main.py
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Main entry point for the ldcov command-line interface.
|
|
3
|
+
|
|
4
|
+
This module provides the main entry point for the ldcov command-line interface.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import sys
|
|
8
|
+
import logging
|
|
9
|
+
from typing import List, Optional
|
|
10
|
+
|
|
11
|
+
from .commands import run_cli
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def main(args: Optional[List[str]] = None):
|
|
15
|
+
"""
|
|
16
|
+
Main entry point for the ldcov command-line interface.
|
|
17
|
+
|
|
18
|
+
Parameters:
|
|
19
|
+
-----------
|
|
20
|
+
args : list, optional
|
|
21
|
+
Command-line arguments (for testing)
|
|
22
|
+
"""
|
|
23
|
+
if args:
|
|
24
|
+
sys.argv = args
|
|
25
|
+
|
|
26
|
+
try:
|
|
27
|
+
run_cli()
|
|
28
|
+
except Exception as e:
|
|
29
|
+
logging.error(f"Error: {e}")
|
|
30
|
+
sys.exit(1)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
if __name__ == "__main__":
|
|
34
|
+
main()
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Computation modules for LD calculation and covariate adjustment.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from .correlation import (
|
|
6
|
+
compute_correlation_matrix,
|
|
7
|
+
load_and_adjust_genotypes,
|
|
8
|
+
compute_ld_from_standardized,
|
|
9
|
+
)
|
|
10
|
+
from .covariate import regress_out_covariates, standardize_genotypes
|
|
@@ -0,0 +1,239 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Core correlation computation module for calculating linkage disequilibrium.
|
|
3
|
+
|
|
4
|
+
This module provides functions for computing correlation between genetic variants,
|
|
5
|
+
which can be used as a measure of linkage disequilibrium.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import numpy as np
|
|
9
|
+
import pandas as pd
|
|
10
|
+
from typing import Optional, List, Tuple
|
|
11
|
+
import logging
|
|
12
|
+
|
|
13
|
+
# Local compute imports
|
|
14
|
+
from .covariate import regress_out_covariates, standardize_genotypes
|
|
15
|
+
|
|
16
|
+
# IO imports
|
|
17
|
+
from ..io import load_bgen
|
|
18
|
+
from ..io.covariate_loader import load_covariates
|
|
19
|
+
from ..io.correlation_io import save_correlation_matrix
|
|
20
|
+
|
|
21
|
+
logger = logging.getLogger(__name__)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def load_and_adjust_genotypes(
|
|
25
|
+
genotype_file: str,
|
|
26
|
+
covariate_file: Optional[str] = None,
|
|
27
|
+
projection_matrix_file: Optional[str] = None,
|
|
28
|
+
region: Optional[str] = None,
|
|
29
|
+
index_file: Optional[str] = None,
|
|
30
|
+
sample_file: Optional[str] = None,
|
|
31
|
+
z_file: Optional[str] = None,
|
|
32
|
+
covariate_id_col: str = "IID",
|
|
33
|
+
covariate_cols: Optional[List[str]] = None,
|
|
34
|
+
show_progress: bool = False,
|
|
35
|
+
nan_action: str = "error",
|
|
36
|
+
) -> Tuple[np.ndarray, pd.DataFrame, List[str], np.ndarray, np.ndarray]:
|
|
37
|
+
"""
|
|
38
|
+
Load genotypes, standardize them, and optionally adjust for covariates.
|
|
39
|
+
|
|
40
|
+
Parameters:
|
|
41
|
+
-----------
|
|
42
|
+
genotype_file : str
|
|
43
|
+
Path to BGEN genotype file
|
|
44
|
+
covariate_file : str, optional
|
|
45
|
+
Path to covariate file for adjustment. Ignored if projection_matrix_file is provided.
|
|
46
|
+
projection_matrix_file : str, optional
|
|
47
|
+
Path to pre-computed projection matrix file (.proj.npz). If provided, uses this
|
|
48
|
+
instead of computing from covariates.
|
|
49
|
+
region : str, optional
|
|
50
|
+
Genomic region in format "chr:start-end"
|
|
51
|
+
index_file : str, optional
|
|
52
|
+
Path to BGEN index file (.bgi)
|
|
53
|
+
sample_file : str, optional
|
|
54
|
+
Path to sample file
|
|
55
|
+
z_file : str, optional
|
|
56
|
+
Path to .z file specifying which variants to load and their order
|
|
57
|
+
covariate_id_col : str, optional
|
|
58
|
+
Column name for sample IDs in covariate file (default: "IID")
|
|
59
|
+
covariate_cols : list of str, optional
|
|
60
|
+
Specific columns to use as covariates. If None, all columns except ID are used
|
|
61
|
+
show_progress : bool, optional
|
|
62
|
+
Whether to show progress bars during loading (default: False)
|
|
63
|
+
nan_action : str, optional
|
|
64
|
+
Action for handling NaN values: 'error' (default), 'mean', or 'omit'
|
|
65
|
+
|
|
66
|
+
Returns:
|
|
67
|
+
--------
|
|
68
|
+
tuple
|
|
69
|
+
(standardized_genotypes, variant_info, sample_ids, means, norms)
|
|
70
|
+
- standardized_genotypes: Standardized (and possibly adjusted) genotypes
|
|
71
|
+
- variant_info: DataFrame with variant information (ordered by .z file if provided)
|
|
72
|
+
- sample_ids: List of sample IDs
|
|
73
|
+
- means: Original means before standardization
|
|
74
|
+
- norms: Original norms before standardization
|
|
75
|
+
"""
|
|
76
|
+
# Load genotype data
|
|
77
|
+
logger.info(f"Loading genotype data from BGEN file: {genotype_file}")
|
|
78
|
+
|
|
79
|
+
# Process .z file if provided
|
|
80
|
+
variant_filter = None
|
|
81
|
+
if z_file is not None:
|
|
82
|
+
from ..utils.variant_filter import load_variant_filter
|
|
83
|
+
|
|
84
|
+
logger.info(f"Reading variant filter from .z file: {z_file}")
|
|
85
|
+
variant_filter = load_variant_filter(z_file)
|
|
86
|
+
|
|
87
|
+
# Determine which samples to load based on covariate/projection data
|
|
88
|
+
samples_to_load = None
|
|
89
|
+
|
|
90
|
+
if projection_matrix_file:
|
|
91
|
+
# If using projection matrix, we need to load only samples in the projection
|
|
92
|
+
logger.info("Pre-loading projection matrix to determine samples")
|
|
93
|
+
from .projection import load_projection_matrix
|
|
94
|
+
|
|
95
|
+
projection_data = load_projection_matrix(projection_matrix_file)
|
|
96
|
+
samples_to_load = projection_data.sample_ids
|
|
97
|
+
logger.info(f"Will load {len(samples_to_load)} samples from projection matrix")
|
|
98
|
+
|
|
99
|
+
elif covariate_file:
|
|
100
|
+
# If using covariates, we can pre-load the sample IDs to filter early
|
|
101
|
+
logger.info("Pre-loading covariate sample IDs for early filtering")
|
|
102
|
+
# Quick load just to get sample IDs
|
|
103
|
+
import pandas as pd
|
|
104
|
+
|
|
105
|
+
# Try to read with automatic delimiter detection
|
|
106
|
+
try:
|
|
107
|
+
cov_df = pd.read_csv(
|
|
108
|
+
covariate_file,
|
|
109
|
+
sep=None,
|
|
110
|
+
engine="python",
|
|
111
|
+
usecols=[covariate_id_col],
|
|
112
|
+
dtype={covariate_id_col: str},
|
|
113
|
+
)
|
|
114
|
+
except Exception:
|
|
115
|
+
# Fallback to reading all columns if usecols fails
|
|
116
|
+
cov_df = pd.read_csv(covariate_file, sep=None, engine="python", dtype=str)
|
|
117
|
+
if covariate_id_col not in cov_df.columns:
|
|
118
|
+
raise ValueError(f"Column '{covariate_id_col}' not found in covariate file")
|
|
119
|
+
cov_df = cov_df[[covariate_id_col]]
|
|
120
|
+
|
|
121
|
+
samples_to_load = cov_df[covariate_id_col].astype(str).tolist()
|
|
122
|
+
logger.info(f"Will load {len(samples_to_load)} samples from covariate file")
|
|
123
|
+
|
|
124
|
+
# Load genotypes with sample filtering if applicable
|
|
125
|
+
genotypes, variant_info, sample_ids = load_bgen(
|
|
126
|
+
file_path=genotype_file,
|
|
127
|
+
index_path=index_file,
|
|
128
|
+
sample_path=sample_file,
|
|
129
|
+
region=region,
|
|
130
|
+
variant_filter=variant_filter,
|
|
131
|
+
sample_ids=samples_to_load,
|
|
132
|
+
show_progress=show_progress,
|
|
133
|
+
nan_action=nan_action,
|
|
134
|
+
)
|
|
135
|
+
|
|
136
|
+
# Standardize genotypes
|
|
137
|
+
logger.info("Standardizing genotypes")
|
|
138
|
+
standardized_genotypes, means, norms = standardize_genotypes(
|
|
139
|
+
genotypes, center=True, scale=True, inplace=True
|
|
140
|
+
)
|
|
141
|
+
|
|
142
|
+
# Apply covariate adjustment if provided
|
|
143
|
+
if projection_matrix_file:
|
|
144
|
+
# Use pre-computed projection matrix
|
|
145
|
+
# Note: samples were already filtered during loading
|
|
146
|
+
logger.info("Using pre-computed projection matrix for adjustment")
|
|
147
|
+
|
|
148
|
+
# Projection data was already loaded above, but we load it again
|
|
149
|
+
# to avoid variable scope issues
|
|
150
|
+
if "projection_data" not in locals():
|
|
151
|
+
from .projection import load_projection_matrix
|
|
152
|
+
|
|
153
|
+
projection_data = load_projection_matrix(projection_matrix_file)
|
|
154
|
+
|
|
155
|
+
logger.info("Adjusting genotypes using pre-computed projection matrix")
|
|
156
|
+
standardized_genotypes = regress_out_covariates(
|
|
157
|
+
standardized_genotypes, projection_matrix_Q=projection_data.Q, inplace=True
|
|
158
|
+
)
|
|
159
|
+
|
|
160
|
+
elif covariate_file:
|
|
161
|
+
# Original workflow: load covariates and compute projection
|
|
162
|
+
# Note: samples were already filtered during loading if covariate file was provided
|
|
163
|
+
logger.info(f"Loading full covariates from {covariate_file}")
|
|
164
|
+
covariates = load_covariates(
|
|
165
|
+
covariate_file, sample_ids, id_col=covariate_id_col, cols_to_use=covariate_cols
|
|
166
|
+
)
|
|
167
|
+
|
|
168
|
+
# Since we already filtered during loading, all samples should match
|
|
169
|
+
if len(covariates) != len(sample_ids):
|
|
170
|
+
logger.warning(
|
|
171
|
+
f"Unexpected sample mismatch after pre-filtering: "
|
|
172
|
+
f"{len(covariates)} covariate samples vs {len(sample_ids)} genotype samples"
|
|
173
|
+
)
|
|
174
|
+
|
|
175
|
+
logger.info("Adjusting genotypes for covariates")
|
|
176
|
+
standardized_genotypes = regress_out_covariates(
|
|
177
|
+
standardized_genotypes, covariates=covariates, inplace=True
|
|
178
|
+
)
|
|
179
|
+
|
|
180
|
+
return standardized_genotypes, variant_info, sample_ids, means, norms
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def compute_ld_from_standardized(
|
|
184
|
+
standardized_genotypes: np.ndarray,
|
|
185
|
+
variant_info: pd.DataFrame,
|
|
186
|
+
output_file: str,
|
|
187
|
+
output_format: str = "matrix",
|
|
188
|
+
write_index: bool = True,
|
|
189
|
+
) -> None:
|
|
190
|
+
"""
|
|
191
|
+
Compute LD matrix from standardized genotypes and save to file.
|
|
192
|
+
|
|
193
|
+
Parameters:
|
|
194
|
+
-----------
|
|
195
|
+
standardized_genotypes : numpy.ndarray
|
|
196
|
+
Standardized genotype matrix (samples x variants)
|
|
197
|
+
variant_info : pandas.DataFrame
|
|
198
|
+
Variant information
|
|
199
|
+
output_file : str
|
|
200
|
+
Path to output LD file
|
|
201
|
+
output_format : str
|
|
202
|
+
Output format ("matrix", "long", "bcor")
|
|
203
|
+
write_index : bool
|
|
204
|
+
When output_format=='bcor', also emit a .bcor.idx sidecar (default True).
|
|
205
|
+
"""
|
|
206
|
+
logger.info("Computing LD matrix")
|
|
207
|
+
corr_matrix = compute_correlation_matrix(standardized_genotypes)
|
|
208
|
+
|
|
209
|
+
# Save to file
|
|
210
|
+
save_correlation_matrix(
|
|
211
|
+
corr_matrix,
|
|
212
|
+
output_file,
|
|
213
|
+
variant_info=variant_info,
|
|
214
|
+
output_format=output_format,
|
|
215
|
+
write_index=write_index,
|
|
216
|
+
)
|
|
217
|
+
|
|
218
|
+
logger.info(f"LD matrix saved to {output_file}")
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def compute_correlation_matrix(standardized_genotypes: np.ndarray) -> np.ndarray:
|
|
222
|
+
"""
|
|
223
|
+
Compute correlation matrix from standardized genotype data.
|
|
224
|
+
|
|
225
|
+
Parameters:
|
|
226
|
+
-----------
|
|
227
|
+
standardized_genotypes : numpy.ndarray
|
|
228
|
+
Standardized genotype matrix (samples x variants). Genotypes should already be
|
|
229
|
+
standardized (centered and scaled) before calling this function.
|
|
230
|
+
|
|
231
|
+
Returns:
|
|
232
|
+
--------
|
|
233
|
+
numpy.ndarray
|
|
234
|
+
Correlation matrix (variants x variants).
|
|
235
|
+
"""
|
|
236
|
+
# Calculate correlations using dot product of standardized genotypes: t(X_scaled) %*% X_scaled
|
|
237
|
+
corr_matrix = np.dot(standardized_genotypes.T, standardized_genotypes)
|
|
238
|
+
|
|
239
|
+
return corr_matrix
|