strkit 0.13.1__tar.gz → 0.15.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. {strkit-0.13.1/strkit.egg-info → strkit-0.15.0}/PKG-INFO +50 -39
  2. {strkit-0.13.1 → strkit-0.15.0}/README.md +46 -33
  3. {strkit-0.13.1 → strkit-0.15.0}/setup.py +3 -8
  4. strkit-0.15.0/strkit/VERSION +1 -0
  5. {strkit-0.13.1 → strkit-0.15.0}/strkit/call/__init__.py +2 -0
  6. {strkit-0.13.1 → strkit-0.15.0}/strkit/call/allele.py +41 -38
  7. strkit-0.15.0/strkit/call/call_locus.py +1529 -0
  8. strkit-0.15.0/strkit/call/call_sample.py +482 -0
  9. strkit-0.15.0/strkit/call/cigar.py +27 -0
  10. strkit-0.15.0/strkit/call/consensus.py +76 -0
  11. strkit-0.15.0/strkit/call/output/__init__.py +12 -0
  12. strkit-0.15.0/strkit/call/output/json_report.py +83 -0
  13. {strkit-0.13.1 → strkit-0.15.0}/strkit/call/output/tsv.py +5 -2
  14. strkit-0.15.0/strkit/call/output/vcf.py +209 -0
  15. strkit-0.15.0/strkit/call/params.py +141 -0
  16. strkit-0.15.0/strkit/call/realign.py +123 -0
  17. strkit-0.15.0/strkit/call/repeats.py +243 -0
  18. strkit-0.15.0/strkit/call/snvs.py +173 -0
  19. strkit-0.15.0/strkit/call/types.py +131 -0
  20. strkit-0.15.0/strkit/call/utils.py +54 -0
  21. {strkit-0.13.1 → strkit-0.15.0}/strkit/entry.py +46 -68
  22. strkit-0.15.0/strkit/json.py +21 -0
  23. strkit-0.15.0/strkit/logger.py +40 -0
  24. {strkit-0.13.1 → strkit-0.15.0}/strkit/mi/base.py +6 -4
  25. {strkit-0.13.1 → strkit-0.15.0}/strkit/mi/expansionhunter.py +1 -1
  26. {strkit-0.13.1 → strkit-0.15.0}/strkit/mi/repeathmm.py +2 -0
  27. {strkit-0.13.1 → strkit-0.15.0}/strkit/mi/result.py +12 -12
  28. {strkit-0.13.1 → strkit-0.15.0}/strkit/mi/strkit.py +5 -6
  29. {strkit-0.13.1 → strkit-0.15.0}/strkit/utils.py +8 -0
  30. {strkit-0.13.1 → strkit-0.15.0}/strkit/viz/server.py +13 -7
  31. {strkit-0.13.1 → strkit-0.15.0}/strkit/viz/templates/browser.html +24 -20
  32. {strkit-0.13.1 → strkit-0.15.0/strkit.egg-info}/PKG-INFO +50 -39
  33. {strkit-0.13.1 → strkit-0.15.0}/strkit.egg-info/SOURCES.txt +1 -0
  34. {strkit-0.13.1 → strkit-0.15.0}/strkit.egg-info/requires.txt +3 -6
  35. strkit-0.15.0/tests/test_caller_utils.py +20 -0
  36. strkit-0.13.1/strkit/VERSION +0 -1
  37. strkit-0.13.1/strkit/call/call_locus.py +0 -1118
  38. strkit-0.13.1/strkit/call/call_sample.py +0 -344
  39. strkit-0.13.1/strkit/call/cigar.py +0 -93
  40. strkit-0.13.1/strkit/call/consensus.py +0 -36
  41. strkit-0.13.1/strkit/call/output/__init__.py +0 -9
  42. strkit-0.13.1/strkit/call/output/json_report.py +0 -46
  43. strkit-0.13.1/strkit/call/output/vcf.py +0 -190
  44. strkit-0.13.1/strkit/call/realign.py +0 -64
  45. strkit-0.13.1/strkit/call/repeats.py +0 -288
  46. strkit-0.13.1/strkit/call/snvs.py +0 -475
  47. strkit-0.13.1/strkit/call/types.py +0 -56
  48. strkit-0.13.1/strkit/call/utils.py +0 -63
  49. strkit-0.13.1/strkit/json.py +0 -20
  50. strkit-0.13.1/strkit/logger.py +0 -27
  51. strkit-0.13.1/tests/test_caller_utils.py +0 -33
  52. {strkit-0.13.1 → strkit-0.15.0}/LICENSE +0 -0
  53. {strkit-0.13.1 → strkit-0.15.0}/MANIFEST.in +0 -0
  54. {strkit-0.13.1 → strkit-0.15.0}/pyproject.toml +0 -0
  55. {strkit-0.13.1 → strkit-0.15.0}/setup.cfg +0 -0
  56. {strkit-0.13.1 → strkit-0.15.0}/strkit/__init__.py +0 -0
  57. {strkit-0.13.1 → strkit-0.15.0}/strkit/call/align_matrix.py +0 -0
  58. {strkit-0.13.1 → strkit-0.15.0}/strkit/call/non_daemonic_pool.py +0 -0
  59. {strkit-0.13.1 → strkit-0.15.0}/strkit/catalog/__init__.py +0 -0
  60. {strkit-0.13.1 → strkit-0.15.0}/strkit/catalog/combine.py +0 -0
  61. {strkit-0.13.1 → strkit-0.15.0}/strkit/constants.py +0 -0
  62. {strkit-0.13.1 → strkit-0.15.0}/strkit/convert/__init__.py +0 -0
  63. {strkit-0.13.1 → strkit-0.15.0}/strkit/convert/_bed_4.py +0 -0
  64. {strkit-0.13.1 → strkit-0.15.0}/strkit/convert/converter.py +0 -0
  65. {strkit-0.13.1 → strkit-0.15.0}/strkit/convert/expansionhunter.py +0 -0
  66. {strkit-0.13.1 → strkit-0.15.0}/strkit/convert/gangstr.py +0 -0
  67. {strkit-0.13.1 → strkit-0.15.0}/strkit/convert/hipstr.py +0 -0
  68. {strkit-0.13.1 → strkit-0.15.0}/strkit/convert/straglr.py +0 -0
  69. {strkit-0.13.1 → strkit-0.15.0}/strkit/convert/tandem_genotypes.py +0 -0
  70. {strkit-0.13.1 → strkit-0.15.0}/strkit/exceptions.py +0 -0
  71. {strkit-0.13.1 → strkit-0.15.0}/strkit/mi/__init__.py +0 -0
  72. {strkit-0.13.1 → strkit-0.15.0}/strkit/mi/gangstr.py +0 -0
  73. {strkit-0.13.1 → strkit-0.15.0}/strkit/mi/straglr.py +0 -0
  74. {strkit-0.13.1 → strkit-0.15.0}/strkit/mi/tandem_genotypes.py +0 -0
  75. {strkit-0.13.1 → strkit-0.15.0}/strkit/mi/vcf_utils.py +0 -0
  76. {strkit-0.13.1 → strkit-0.15.0}/strkit/viz/__init__.py +0 -0
  77. {strkit-0.13.1 → strkit-0.15.0}/strkit/viz/static/logo.png +0 -0
  78. {strkit-0.13.1 → strkit-0.15.0}/strkit.egg-info/dependency_links.txt +0 -0
  79. {strkit-0.13.1 → strkit-0.15.0}/strkit.egg-info/entry_points.txt +0 -0
  80. {strkit-0.13.1 → strkit-0.15.0}/strkit.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: strkit
3
- Version: 0.13.1
3
+ Version: 0.15.0
4
4
  Summary: A toolkit for analyzing variation in short(ish) tandem repeats.
5
5
  Home-page: https://github.com/davidlougheed/strkit
6
6
  Author: David Lougheed
@@ -13,22 +13,22 @@ Requires-Python: ~=3.9
13
13
  Description-Content-Type: text/markdown
14
14
  License-File: LICENSE
15
15
  Requires-Dist: Flask<3.1,>=2.2.5
16
+ Requires-Dist: orjson<4,>=3.9.15
16
17
  Requires-Dist: pysam<0.23,>=0.19
17
18
  Requires-Dist: numpy<1.27,>=1.23.4
18
19
  Requires-Dist: parasail<1.4,>=1.2.4
19
- Requires-Dist: pyfamsa<0.4,>=0.3.0
20
20
  Requires-Dist: scikit-learn<1.5,>=1.2.1
21
- Requires-Dist: scipy<1.13,>=1.10
21
+ Requires-Dist: scipy<1.14,>=1.10
22
22
  Requires-Dist: statsmodels<0.15,>=0.14.0
23
- Provides-Extra: rustdeps
24
- Requires-Dist: orjson<4,>=3.8.5; extra == "rustdeps"
25
- Requires-Dist: strkit_rust_ext==0.5.0; extra == "rustdeps"
23
+ Requires-Dist: strkit_rust_ext==0.12.2
26
24
 
27
- # STRkit
25
+ # STRkit - short tandem repeat genotyping with long reads
28
26
 
29
27
  [![PyPI version](https://badge.fury.io/py/strkit.svg)](https://badge.fury.io/py/strkit)
30
28
 
31
- A genotyping and analysis toolkit for short(ish) tandem repeats.
29
+ STRkit is a short tandem repeat (STR) genotyping and analysis toolkit for long read sequencing data, especially
30
+ PacBio HiFi data. The STRkit software package is written in Python and is available in the PyPI package registry or as
31
+ a Docker container.
32
32
 
33
33
  <img src="./docs/images/strkit_logo_small.png" alt="" width="500" height="324" />
34
34
 
@@ -39,8 +39,8 @@ A genotyping and analysis toolkit for short(ish) tandem repeats.
39
39
  created during graduate research by DL.
40
40
  * 2023+: (versions beyond `0.8.0a1`):
41
41
  * Portions &copy; DL and McGill University 2021-2023
42
- * Portions &copy; McGill University 2023
43
- * Portions &copy; DL 2023
42
+ * Portions &copy; McGill University 2024
43
+ * Portions &copy; DL 2024
44
44
 
45
45
 
46
46
  ### Notice
@@ -75,6 +75,8 @@ Used under the terms of the ISC license.
75
75
 
76
76
  ## Installation
77
77
 
78
+ ### Via PyPI
79
+
78
80
  STRkit requires Python 3.9+ and can be installed from PyPI via `pip`
79
81
  with the following command:
80
82
 
@@ -82,18 +84,38 @@ with the following command:
82
84
  python -m pip install strkit
83
85
  ```
84
86
 
85
- For **faster genotyping** (via some optimized utility functions) and the faster `orjson` library
86
- for JSON parsing and serialization, specify the `rustdeps` extra:
87
+ You may need to install the [Rust toolchain](https://www.rust-lang.org/tools/install)
88
+ to compile the `strkit_rust_ext` wheels, although prebuilt wheels for this module are available
89
+ for some platforms.
90
+
91
+ On Digital Research Alliance of Canada/Compute Canada clusters, this involves loading a few modules:
92
+
93
+ ```bash
94
+ module load rust/1.76.0 clang/17.0.6 python/3.11 scipy-stack/2023b parasail/2.6.2
95
+ python -m pip install strkit
96
+ ```
97
+
98
+ STRkit should then be available in your Python environment as a command-line tool:
87
99
 
88
100
  ```bash
89
- python -m pip install strkit[rustdeps]
101
+ strkit --help
90
102
  ```
91
103
 
92
- On some Alliance (DRAC/CC) clusters, this may require loading Rust first, to compile the wheels:
104
+ ### As a Docker container
105
+
106
+ STRkit is also available as a [Docker container](https://github.com/davidlougheed/strkit/pkgs/container/strkit), stored
107
+ in the GitHub Container Registry.
108
+
109
+ It can be pulled using the following command:
93
110
 
94
111
  ```bash
95
- module load rust/1.65.0
96
- python -m pip install strkit[rustdeps]
112
+ docker pull ghcr.io/davidlougheed/strkit:latest
113
+ ```
114
+
115
+ Then, STRkit commands can be run mostly as normal using the Docker image:
116
+
117
+ ```bash
118
+ docker run -it ghcr.io/davidlougheed/strkit --help
97
119
  ```
98
120
 
99
121
 
@@ -116,8 +138,8 @@ long read data should still work.
116
138
  * Whole-genome and targeted genotyping modes to adjust this re-weighting.
117
139
  * Incorporation of single-nucleotide variation (SNVs) for better and faster calling plus
118
140
  additional downstream analysis possibilities.
119
- * Recommended for **HiFi data only**. In my testing, this worsens runtime and call quality for
120
- ONT-UL data, but speeds up the tool and improves call quality for HiFi data.
141
+ * Recommended for **HiFi data and ONT duplex data only**. In my testing, this worsens runtime and call quality for
142
+ ONT ultra-long-read data, but speeds up the tool and improves call quality for HiFi/ONT duplex data.
121
143
  * Parallelized for faster computing on clusters and for ad-hoc fast analysis of single samples.
122
144
  * 95% confidence intervals on calls via a user-configurable optional parametric bootstrapping process.
123
145
 
@@ -126,11 +148,11 @@ long read data should still work.
126
148
 
127
149
  ```bash
128
150
  strkit call \
129
- path/to/read/file.bam \ # [REQUIRED] At least one indexed read file (BAM/CRAM)
130
- --hq \ # If using PacBio HiFi reads, enable this to get better genotyping & more robust expansion detection
131
- --realign \ # If using PacBio HiFi reads, enable this to enable local realignment / read recovery
151
+ path/to/read/file.bam \ # [REQUIRED] One indexed read file (BAM/CRAM)
152
+ --hq \ # If using PacBio HiFi or ONT duplex reads, enable this to get better genotyping & more robust expansion detection
153
+ --realign \ # If using PacBio HiFi reads, enable this to enable local realignment / read recovery. Good for detecting expansions, but slows down calling.
132
154
  --ref path/to/reference.fa.gz \ # [REQUIRED] Indexed FASTA-formatted reference genome
133
- --loci path/to/loci.bed \ # [REQUIRED] TRF-formatted (or 4-col, with motif as last column) list of loci to genotype
155
+ --loci path/to/loci.bed \ # [REQUIRED] TRF-formatted (or 4-col, with motif as last column) sorted list of loci to genotype
134
156
  --incorporate-snvs path/to/dbsnp.vcf.gz \ # [RECOMMENDED FOR HIFI ONLY] If you want SNV calls to help phase STRs & speed up calling
135
157
  --min-reads 4 \ # Minimum number of supporting reads needed to make a call
136
158
  --min-allele-reads 2 \ # Minimum number of supporting reads needed to call a specific allele size
@@ -149,9 +171,9 @@ If you're using HiFi reads as input, **use the `--hq` and `--realign` options**
149
171
  genotype calculation and a greater proportion of reads incorporated into the computed genotypes,
150
172
  respectively. These should not add much performance overhead.
151
173
 
152
- If more than one read file is specified, the reads will be pooled. This can come in handy if you
153
- have e.g. multiple flow cells of the same sample split into different BAM files, or the reads are
154
- split by chromosome.
174
+ If you want to **incorporate haplotagging from an alignment file (`HP` tags)** into the
175
+ process, which should speed up runtime and potentially improve calling results, you must pass
176
+ the `--use-hp` flag. **This flag is experimental, and has not been tested extensively.**
155
177
 
156
178
  If you want to **incorporate SNV calling** into the process, which speeds up runtime and gives
157
179
  marginally better calling results, you must provide an indexed, `bgzip`-compressed SNV catalog
@@ -165,9 +187,11 @@ If you want to output a full call report, you can use the `--json output-file.js
165
187
  specify a path to output a more detailed JSON document to. This document contains 99% CIs, peak
166
188
  labels, and some other information that isn't included in the normal TSV file. If you want this
167
189
  file to be indented and human-readable, use the `--indent-json` flag in addition to `--json ...`.
190
+ Note that **memory usage will be higher** when JSON output is enabled.
168
191
 
169
192
  If you want to output a VCF file (STRs and SNVs if called; currently not phased), use the
170
- `--vcf ...` argument.
193
+ `--vcf ...` argument. If you pass `--vcf stdout`, the VCF will be written to `stdout` instead of a
194
+ file.
171
195
 
172
196
  See the '[Caller catalog format & choosing a catalog](./docs/caller_catalog.md)' page for more on
173
197
  how to format a locus catalog or choose from existing available catalogs.
@@ -180,19 +204,6 @@ bgzip my-reference.fa # Replaces .fa with a .fa.gz file
180
204
  samtools faidx my-reference.fa.gz # Generates a .fai index file
181
205
  ```
182
206
 
183
- ##### Note on OpenMP Threading
184
-
185
- Slow performance can result from running `strkit call` on a system with OpenMP,
186
- due to a misguided attempt at multithreading under the hood somewhere in Numpy/Scipy (which doesn't work
187
- here due to repeated initializations of the Gaussian mixture model.) To fix this, the following
188
- environment variable is auto-set (hardcoded) before running:
189
-
190
- ```bash
191
- export OMP_NUM_THREADS=1
192
- ```
193
-
194
- If this hard-coded value interferes with your use case, please open an issue.
195
-
196
207
 
197
208
  #### Further documentation on the STRkit caller, including output format:
198
209
 
@@ -1,8 +1,10 @@
1
- # STRkit
1
+ # STRkit - short tandem repeat genotyping with long reads
2
2
 
3
3
  [![PyPI version](https://badge.fury.io/py/strkit.svg)](https://badge.fury.io/py/strkit)
4
4
 
5
- A genotyping and analysis toolkit for short(ish) tandem repeats.
5
+ STRkit is a short tandem repeat (STR) genotyping and analysis toolkit for long read sequencing data, especially
6
+ PacBio HiFi data. The STRkit software package is written in Python and is available in the PyPI package registry or as
7
+ a Docker container.
6
8
 
7
9
  <img src="./docs/images/strkit_logo_small.png" alt="" width="500" height="324" />
8
10
 
@@ -13,8 +15,8 @@ A genotyping and analysis toolkit for short(ish) tandem repeats.
13
15
  created during graduate research by DL.
14
16
  * 2023+: (versions beyond `0.8.0a1`):
15
17
  * Portions &copy; DL and McGill University 2021-2023
16
- * Portions &copy; McGill University 2023
17
- * Portions &copy; DL 2023
18
+ * Portions &copy; McGill University 2024
19
+ * Portions &copy; DL 2024
18
20
 
19
21
 
20
22
  ### Notice
@@ -49,6 +51,8 @@ Used under the terms of the ISC license.
49
51
 
50
52
  ## Installation
51
53
 
54
+ ### Via PyPI
55
+
52
56
  STRkit requires Python 3.9+ and can be installed from PyPI via `pip`
53
57
  with the following command:
54
58
 
@@ -56,18 +60,38 @@ with the following command:
56
60
  python -m pip install strkit
57
61
  ```
58
62
 
59
- For **faster genotyping** (via some optimized utility functions) and the faster `orjson` library
60
- for JSON parsing and serialization, specify the `rustdeps` extra:
63
+ You may need to install the [Rust toolchain](https://www.rust-lang.org/tools/install)
64
+ to compile the `strkit_rust_ext` wheels, although prebuilt wheels for this module are available
65
+ for some platforms.
66
+
67
+ On Digital Research Alliance of Canada/Compute Canada clusters, this involves loading a few modules:
68
+
69
+ ```bash
70
+ module load rust/1.76.0 clang/17.0.6 python/3.11 scipy-stack/2023b parasail/2.6.2
71
+ python -m pip install strkit
72
+ ```
73
+
74
+ STRkit should then be available in your Python environment as a command-line tool:
61
75
 
62
76
  ```bash
63
- python -m pip install strkit[rustdeps]
77
+ strkit --help
64
78
  ```
65
79
 
66
- On some Alliance (DRAC/CC) clusters, this may require loading Rust first, to compile the wheels:
80
+ ### As a Docker container
81
+
82
+ STRkit is also available as a [Docker container](https://github.com/davidlougheed/strkit/pkgs/container/strkit), stored
83
+ in the GitHub Container Registry.
84
+
85
+ It can be pulled using the following command:
67
86
 
68
87
  ```bash
69
- module load rust/1.65.0
70
- python -m pip install strkit[rustdeps]
88
+ docker pull ghcr.io/davidlougheed/strkit:latest
89
+ ```
90
+
91
+ Then, STRkit commands can be run mostly as normal using the Docker image:
92
+
93
+ ```bash
94
+ docker run -it ghcr.io/davidlougheed/strkit --help
71
95
  ```
72
96
 
73
97
 
@@ -90,8 +114,8 @@ long read data should still work.
90
114
  * Whole-genome and targeted genotyping modes to adjust this re-weighting.
91
115
  * Incorporation of single-nucleotide variation (SNVs) for better and faster calling plus
92
116
  additional downstream analysis possibilities.
93
- * Recommended for **HiFi data only**. In my testing, this worsens runtime and call quality for
94
- ONT-UL data, but speeds up the tool and improves call quality for HiFi data.
117
+ * Recommended for **HiFi data and ONT duplex data only**. In my testing, this worsens runtime and call quality for
118
+ ONT ultra-long-read data, but speeds up the tool and improves call quality for HiFi/ONT duplex data.
95
119
  * Parallelized for faster computing on clusters and for ad-hoc fast analysis of single samples.
96
120
  * 95% confidence intervals on calls via a user-configurable optional parametric bootstrapping process.
97
121
 
@@ -100,11 +124,11 @@ long read data should still work.
100
124
 
101
125
  ```bash
102
126
  strkit call \
103
- path/to/read/file.bam \ # [REQUIRED] At least one indexed read file (BAM/CRAM)
104
- --hq \ # If using PacBio HiFi reads, enable this to get better genotyping & more robust expansion detection
105
- --realign \ # If using PacBio HiFi reads, enable this to enable local realignment / read recovery
127
+ path/to/read/file.bam \ # [REQUIRED] One indexed read file (BAM/CRAM)
128
+ --hq \ # If using PacBio HiFi or ONT duplex reads, enable this to get better genotyping & more robust expansion detection
129
+ --realign \ # If using PacBio HiFi reads, enable this to enable local realignment / read recovery. Good for detecting expansions, but slows down calling.
106
130
  --ref path/to/reference.fa.gz \ # [REQUIRED] Indexed FASTA-formatted reference genome
107
- --loci path/to/loci.bed \ # [REQUIRED] TRF-formatted (or 4-col, with motif as last column) list of loci to genotype
131
+ --loci path/to/loci.bed \ # [REQUIRED] TRF-formatted (or 4-col, with motif as last column) sorted list of loci to genotype
108
132
  --incorporate-snvs path/to/dbsnp.vcf.gz \ # [RECOMMENDED FOR HIFI ONLY] If you want SNV calls to help phase STRs & speed up calling
109
133
  --min-reads 4 \ # Minimum number of supporting reads needed to make a call
110
134
  --min-allele-reads 2 \ # Minimum number of supporting reads needed to call a specific allele size
@@ -123,9 +147,9 @@ If you're using HiFi reads as input, **use the `--hq` and `--realign` options**
123
147
  genotype calculation and a greater proportion of reads incorporated into the computed genotypes,
124
148
  respectively. These should not add much performance overhead.
125
149
 
126
- If more than one read file is specified, the reads will be pooled. This can come in handy if you
127
- have e.g. multiple flow cells of the same sample split into different BAM files, or the reads are
128
- split by chromosome.
150
+ If you want to **incorporate haplotagging from an alignment file (`HP` tags)** into the
151
+ process, which should speed up runtime and potentially improve calling results, you must pass
152
+ the `--use-hp` flag. **This flag is experimental, and has not been tested extensively.**
129
153
 
130
154
  If you want to **incorporate SNV calling** into the process, which speeds up runtime and gives
131
155
  marginally better calling results, you must provide an indexed, `bgzip`-compressed SNV catalog
@@ -139,9 +163,11 @@ If you want to output a full call report, you can use the `--json output-file.js
139
163
  specify a path to output a more detailed JSON document to. This document contains 99% CIs, peak
140
164
  labels, and some other information that isn't included in the normal TSV file. If you want this
141
165
  file to be indented and human-readable, use the `--indent-json` flag in addition to `--json ...`.
166
+ Note that **memory usage will be higher** when JSON output is enabled.
142
167
 
143
168
  If you want to output a VCF file (STRs and SNVs if called; currently not phased), use the
144
- `--vcf ...` argument.
169
+ `--vcf ...` argument. If you pass `--vcf stdout`, the VCF will be written to `stdout` instead of a
170
+ file.
145
171
 
146
172
  See the '[Caller catalog format & choosing a catalog](./docs/caller_catalog.md)' page for more on
147
173
  how to format a locus catalog or choose from existing available catalogs.
@@ -154,19 +180,6 @@ bgzip my-reference.fa # Replaces .fa with a .fa.gz file
154
180
  samtools faidx my-reference.fa.gz # Generates a .fai index file
155
181
  ```
156
182
 
157
- ##### Note on OpenMP Threading
158
-
159
- Slow performance can result from running `strkit call` on a system with OpenMP,
160
- due to a misguided attempt at multithreading under the hood somewhere in Numpy/Scipy (which doesn't work
161
- here due to repeated initializations of the Gaussian mixture model.) To fix this, the following
162
- environment variable is auto-set (hardcoded) before running:
163
-
164
- ```bash
165
- export OMP_NUM_THREADS=1
166
- ```
167
-
168
- If this hard-coded value interferes with your use case, please open an issue.
169
-
170
183
 
171
184
  #### Further documentation on the STRkit caller, including output format:
172
185
 
@@ -15,20 +15,15 @@ setup(
15
15
  python_requires="~=3.9",
16
16
  install_requires=[
17
17
  "Flask>=2.2.5,<3.1",
18
+ "orjson>=3.9.15,<4",
18
19
  "pysam>=0.19,<0.23",
19
20
  "numpy>=1.23.4,<1.27",
20
21
  "parasail>=1.2.4,<1.4",
21
- "pyfamsa>=0.3.0,<0.4",
22
22
  "scikit-learn>=1.2.1,<1.5",
23
- "scipy>=1.10,<1.13",
23
+ "scipy>=1.10,<1.14",
24
24
  "statsmodels>=0.14.0,<0.15",
25
+ "strkit_rust_ext==0.12.2",
25
26
  ],
26
- extras_require={
27
- "rustdeps": [
28
- "orjson>=3.8.5,<4",
29
- "strkit_rust_ext==0.5.0",
30
- ],
31
- },
32
27
 
33
28
  description="A toolkit for analyzing variation in short(ish) tandem repeats.",
34
29
  long_description=long_description,
@@ -0,0 +1 @@
1
+ 0.15.0
@@ -2,8 +2,10 @@ from __future__ import annotations
2
2
 
3
3
  from .allele import call_alleles
4
4
  from .call_sample import call_sample
5
+ from .params import CallParams
5
6
 
6
7
  __all__ = [
7
8
  "call_alleles",
8
9
  "call_sample",
10
+ "CallParams",
9
11
  ]
@@ -1,8 +1,12 @@
1
1
  from __future__ import annotations
2
2
 
3
- # Disable OpenMP multithreading since it adds enormous overhead when multiprocessing
3
+ # Disable OpenMP/other multithreading since it adds enormous overhead when multiprocessing
4
4
  import os
5
5
  os.environ["OMP_NUM_THREADS"] = "1"
6
+ os.environ["OPENBLAS_NUM_THREADS"] = "1"
7
+ os.environ["MKL_NUM_THREADS"] = "1"
8
+ os.environ["VECLIB_MAXIMUM_THREADS"] = "1"
9
+ os.environ["NUMEXPR_NUM_THREADS"] = "1"
6
10
 
7
11
  # ----------------------------------------------------------------------------------------------------------------------
8
12
 
@@ -20,6 +24,7 @@ from typing import Iterable, Literal, Optional, TypedDict, Union
20
24
 
21
25
  import strkit.constants as cc
22
26
 
27
+ from .params import CallParams
23
28
  from .utils import get_new_seed
24
29
 
25
30
  __all__ = [
@@ -40,6 +45,7 @@ small_allele_min = 8
40
45
  expansion_ratio = 5
41
46
  N_GM_INIT = 3
42
47
 
48
+ WEIGHT_1_0 = np.array([[1.0]])
43
49
  FLOAT_32_EPSILON = np.finfo(np.float32).eps
44
50
 
45
51
  CI_PERCENTILE_RANGES = {
@@ -52,15 +58,11 @@ def _array_as_int(n: Union[NDArray[np.int_], NDArray[np.float_]]) -> NDArray[np.
52
58
  return np.rint(n).astype(np.int32)
53
59
 
54
60
 
55
- def _calculate_cis(
56
- samples,
57
- force_int: bool = False,
58
- ci: str = Literal["95", "99"],
59
- ) -> Union[NDArray[np.int32], NDArray[np.float_]]:
61
+ def _calculate_cis(samples, ci: str = Literal["95", "99"]) -> Union[NDArray[np.int32], NDArray[np.float_]]:
60
62
  percentiles = np.percentile(
61
63
  samples, CI_PERCENTILE_RANGES[ci], axis=1, method="interpolated_inverted_cdf"
62
64
  ).transpose()
63
- return _array_as_int(percentiles) if force_int else percentiles
65
+ return _array_as_int(percentiles)
64
66
 
65
67
 
66
68
  def get_n_alleles(default_n_alleles: int, sample_sex_chroms: Optional[str], contig: str) -> Optional[int]:
@@ -100,7 +102,7 @@ def fit_gmm(
100
102
  # I've confirmed this gives an ~identical result to fitting a GMM with one parameter.
101
103
  fake_g: object = type("", (), {})()
102
104
  fake_g.means_ = np.array([[np.mean(sample_rs)]])
103
- fake_g.weights_ = np.array([[1.0]])
105
+ fake_g.weights_ = WEIGHT_1_0
104
106
  fake_g.covariances_ = np.array([[np.var(sample_rs)]])
105
107
  return fake_g
106
108
 
@@ -143,7 +145,7 @@ def fit_gmm(
143
145
  return g
144
146
 
145
147
 
146
- class CallDict(TypedDict):
148
+ class BaseCallDict(TypedDict):
147
149
  call: Union[NDArray[np.int32], NDArray[np.float_]]
148
150
  call_95_cis: Union[NDArray[np.int32], NDArray[np.float_]] # 2D arrays
149
151
  call_99_cis: Union[NDArray[np.int32], NDArray[np.float_]] # 2D arrays
@@ -153,6 +155,10 @@ class CallDict(TypedDict):
153
155
  modal_n_peaks: int
154
156
 
155
157
 
158
+ class CallDict(BaseCallDict, total=False):
159
+ ps: int
160
+
161
+
156
162
  def make_read_weights(read_weights: Optional[Iterable[float]], num_reads: int) -> NDArray[np.float_]:
157
163
  return np.array(
158
164
  read_weights if read_weights is not None else np.array(([1/num_reads] * num_reads) if num_reads else []))
@@ -163,15 +169,12 @@ def call_alleles(
163
169
  repeats_rev: RepeatCounts,
164
170
  read_weights_fwd: Optional[Iterable[float]],
165
171
  read_weights_rev: Optional[Iterable[float]],
166
- bootstrap_iterations: int,
172
+ params: CallParams,
167
173
  min_reads: int,
168
- min_allele_reads: int,
169
174
  n_alleles: int,
170
175
  separate_strands: bool,
171
176
  read_bias_corr_min: int,
172
177
  gm_filter_factor: int,
173
- hq: bool,
174
- force_int: bool,
175
178
  seed: Optional[int],
176
179
  logger_: logging.Logger,
177
180
  debug_str: str,
@@ -200,18 +203,18 @@ def call_alleles(
200
203
  logger_.debug(f"{debug_str} - skipping bootstrap / GMM fitting for allele(s) (single value)")
201
204
  cn = combined_reads[0]
202
205
 
203
- call = np.array([cn] * n_alleles)
204
- call_cis = np.array([[cn, cn] for _ in range(n_alleles)])
206
+ call = _array_as_int(np.full(n_alleles, cn))
207
+ call_cis = _array_as_int(np.full((n_alleles, 2), cn))
205
208
 
206
- peaks: NDArray[np.float_] = np.array([cn] * n_alleles, dtype=np.float_)
209
+ peaks: NDArray[np.float_] = call.astype(np.float_)
207
210
 
208
211
  return {
209
- "call": _array_as_int(call) if force_int else call,
210
- "call_95_cis": _array_as_int(call_cis) if force_int else call,
211
- "call_99_cis": _array_as_int(call_cis) if force_int else call,
212
+ "call": call,
213
+ "call_95_cis": call_cis,
214
+ "call_99_cis": call_cis,
212
215
  "peaks": peaks,
213
- "peak_weights": np.array([1.0] * n_alleles) / n_alleles,
214
- "peak_stdevs": np.array([0.0] * n_alleles),
216
+ "peak_weights": np.full(n_alleles, 1.0 / n_alleles),
217
+ "peak_stdevs": np.full(n_alleles, 0.0),
215
218
  "modal_n_peaks": 1, # 1 peak, since we have 1 value
216
219
  }
217
220
 
@@ -234,14 +237,14 @@ def call_alleles(
234
237
 
235
238
  fwd_strand_samples = rng.choice(
236
239
  fwd_strand_reads,
237
- size=(bootstrap_iterations, target_length),
240
+ size=(params.num_bootstrap, target_length),
238
241
  replace=True,
239
242
  p=fwd_strand_weights,
240
243
  )
241
244
 
242
245
  rev_strand_samples = rng.choice(
243
246
  rev_strand_reads,
244
- size=(bootstrap_iterations, target_length),
247
+ size=(params.num_bootstrap, target_length),
245
248
  replace=True,
246
249
  p=rev_strand_weights,
247
250
  )
@@ -254,26 +257,26 @@ def call_alleles(
254
257
  concat_samples = np.sort(
255
258
  rng.choice(
256
259
  combined_reads,
257
- size=(bootstrap_iterations, combined_len),
260
+ size=(params.num_bootstrap, combined_len),
258
261
  replace=True,
259
262
  p=combined_weights,
260
- ) if bootstrap_iterations > 1 else np.array([combined_reads]),
263
+ ) if params.num_bootstrap > 1 else np.array([combined_reads]),
261
264
  kind="stable")
262
265
 
263
266
  gmm_cache = {}
264
267
 
265
- def _get_fitted_gmm(s: NDArray[np.int_] | NDArray[np.float_]) -> Optional[object]:
266
- if (s_t := tuple(s)) not in gmm_cache:
268
+ def _get_fitted_gmm(s: Union[NDArray[np.int_], NDArray[np.float_]]) -> Optional[object]:
269
+ if (s_t := s.tobytes()) not in gmm_cache:
267
270
  # Fit Gaussian mixture model to the resampled data
268
- gmm_cache[s_t] = fit_gmm(rng, s, n_alleles, allele_filter, hq, gm_filter_factor)
271
+ gmm_cache[s_t] = fit_gmm(rng, s, n_alleles, allele_filter, params.hq, gm_filter_factor)
269
272
 
270
273
  return gmm_cache[s_t]
271
274
 
272
275
  # Filter out peaks that aren't supported by ~min_allele_reads reads by probability, with some delta to
273
276
  # allow for peaks supported by "most of a read".
274
- allele_filter = (min_allele_reads - 0.1) / concat_samples.shape[0]
277
+ allele_filter = (params.min_allele_reads - 0.1) / concat_samples.shape[0]
275
278
 
276
- for i in range(bootstrap_iterations):
279
+ for i in range(params.num_bootstrap):
277
280
  sample = concat_samples[i, :]
278
281
 
279
282
  g: Optional[object] = _get_fitted_gmm(sample)
@@ -320,8 +323,8 @@ def call_alleles(
320
323
  # Calculate 95% and 99% confidence intervals for each allele from the bootstrap distributions.
321
324
  allele_samples_argsort = allele_samples.argsort(axis=1, kind="stable")
322
325
  allele_samples = np.take_along_axis(allele_samples, allele_samples_argsort, axis=1)
323
- allele_cis_95 = _calculate_cis(allele_samples, force_int=force_int, ci="95")
324
- allele_cis_99 = _calculate_cis(allele_samples, force_int=force_int, ci="99")
326
+ allele_cis_95 = _calculate_cis(allele_samples, ci="95")
327
+ allele_cis_99 = _calculate_cis(allele_samples, ci="99")
325
328
  allele_weight_samples = np.take_along_axis(allele_weight_samples, allele_samples_argsort, axis=1)
326
329
  allele_stdev_samples = np.take_along_axis(allele_stdev_samples, allele_samples_argsort, axis=1)
327
330
 
@@ -335,20 +338,20 @@ def call_alleles(
335
338
 
336
339
  median_idx = allele_samples.shape[1] // 2 #
337
340
  medians_of_means = allele_samples[:, median_idx]
338
- medians_of_means_final = medians_of_means
339
- if force_int:
340
- medians_of_means_final = np.rint(medians_of_means).astype(np.int32)
341
- peak_weights = allele_weight_samples[:, median_idx]
341
+ medians_of_means_final = np.rint(medians_of_means).astype(np.int32)
342
+ peak_weights = allele_weight_samples[:, median_idx].flatten()
342
343
  peak_stdevs = allele_stdev_samples[:, median_idx]
343
344
  modal_n_peaks: int = statistics.mode(sample_peaks).item()
344
345
 
346
+ peak_weights /= peak_weights.sum() # re-normalize weights
347
+
345
348
  return {
346
349
  "call": medians_of_means_final.flatten(),
347
350
  "call_95_cis": allele_cis_95,
348
351
  "call_99_cis": allele_cis_99,
349
352
 
350
- "peaks": medians_of_means_final.flatten(), # Don't round, so we can recover original Gaussian model
351
- "peak_weights": peak_weights.flatten(),
353
+ "peaks": medians_of_means.flatten(), # Don't round, so we can recover original Gaussian model
354
+ "peak_weights": peak_weights,
352
355
  "peak_stdevs": peak_stdevs.flatten(),
353
356
  # TODO: should be ok to use this, because resample gets put at end, vertically (3rd allele in a 3-ploid case)
354
357
  # so taking the first 2 alleles still works in terms of stdev/mean estimates? I think?