mafutils 0.2.0__tar.gz → 0.2.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {mafutils-0.2.0 → mafutils-0.2.1}/.gitignore +4 -1
- {mafutils-0.2.0 → mafutils-0.2.1}/DEVELOPMENT.md +34 -0
- {mafutils-0.2.0/mafutils.egg-info → mafutils-0.2.1}/PKG-INFO +23 -49
- {mafutils-0.2.0 → mafutils-0.2.1}/README.md +22 -48
- {mafutils-0.2.0 → mafutils-0.2.1}/mafutils/_version.py +3 -3
- {mafutils-0.2.0 → mafutils-0.2.1}/mafutils/fetch.py +8 -4
- {mafutils-0.2.0 → mafutils-0.2.1/mafutils.egg-info}/PKG-INFO +23 -49
- {mafutils-0.2.0 → mafutils-0.2.1}/mafutils.egg-info/SOURCES.txt +0 -2
- {mafutils-0.2.0 → mafutils-0.2.1}/mafutils.egg-info/scm_file_list.json +0 -2
- mafutils-0.2.1/mafutils.egg-info/scm_version.json +8 -0
- mafutils-0.2.0/benchmarks/maf_fetch.log +0 -516
- mafutils-0.2.0/maf_fetch.log +0 -5565
- mafutils-0.2.0/mafutils.egg-info/scm_version.json +0 -8
- {mafutils-0.2.0 → mafutils-0.2.1}/.codex +0 -0
- {mafutils-0.2.0 → mafutils-0.2.1}/.github/workflows/publish-pypi.yml +0 -0
- {mafutils-0.2.0 → mafutils-0.2.1}/AGENTS.md +0 -0
- {mafutils-0.2.0 → mafutils-0.2.1}/LICENSE +0 -0
- {mafutils-0.2.0 → mafutils-0.2.1}/THIRD_PARTY_LICENSES.md +0 -0
- {mafutils-0.2.0 → mafutils-0.2.1}/benchmarks/README.md +0 -0
- {mafutils-0.2.0 → mafutils-0.2.1}/benchmarks/Snakefile +0 -0
- {mafutils-0.2.0 → mafutils-0.2.1}/benchmarks/baseline-manual-timings.txt +0 -0
- {mafutils-0.2.0 → mafutils-0.2.1}/benchmarks/config.yaml +0 -0
- {mafutils-0.2.0 → mafutils-0.2.1}/benchmarks/history.csv +0 -0
- {mafutils-0.2.0 → mafutils-0.2.1}/benchmarks/notebook.ipynb +0 -0
- {mafutils-0.2.0 → mafutils-0.2.1}/benchmarks/parse_benchmarks.py +0 -0
- {mafutils-0.2.0 → mafutils-0.2.1}/benchmarks/profiles/slurm/config.yaml +0 -0
- {mafutils-0.2.0 → mafutils-0.2.1}/benchmarks/rulegraph.png +0 -0
- {mafutils-0.2.0 → mafutils-0.2.1}/mafutils/__init__.py +0 -0
- {mafutils-0.2.0 → mafutils-0.2.1}/mafutils/__main__.py +0 -0
- {mafutils-0.2.0 → mafutils-0.2.1}/mafutils/cli.py +0 -0
- {mafutils-0.2.0 → mafutils-0.2.1}/mafutils/gc.py +0 -0
- {mafutils-0.2.0 → mafutils-0.2.1}/mafutils/index.py +0 -0
- {mafutils-0.2.0 → mafutils-0.2.1}/mafutils/lib/__init__.py +0 -0
- {mafutils-0.2.0 → mafutils-0.2.1}/mafutils/lib/bgzf.py +0 -0
- {mafutils-0.2.0 → mafutils-0.2.1}/mafutils/lib/common.py +0 -0
- {mafutils-0.2.0 → mafutils-0.2.1}/mafutils/lib/loginit.py +0 -0
- {mafutils-0.2.0 → mafutils-0.2.1}/mafutils/stats.py +0 -0
- {mafutils-0.2.0 → mafutils-0.2.1}/mafutils/validate.py +0 -0
- {mafutils-0.2.0 → mafutils-0.2.1}/mafutils.egg-info/dependency_links.txt +0 -0
- {mafutils-0.2.0 → mafutils-0.2.1}/mafutils.egg-info/entry_points.txt +0 -0
- {mafutils-0.2.0 → mafutils-0.2.1}/mafutils.egg-info/requires.txt +0 -0
- {mafutils-0.2.0 → mafutils-0.2.1}/mafutils.egg-info/top_level.txt +0 -0
- {mafutils-0.2.0 → mafutils-0.2.1}/pyproject.toml +0 -0
- {mafutils-0.2.0 → mafutils-0.2.1}/setup.cfg +0 -0
- {mafutils-0.2.0 → mafutils-0.2.1}/tests/crossblocks-missing-species.bed +0 -0
- {mafutils-0.2.0 → mafutils-0.2.1}/tests/crossblocks.bed +0 -0
- {mafutils-0.2.0 → mafutils-0.2.1}/tests/example-tabs.maf +0 -0
- {mafutils-0.2.0 → mafutils-0.2.1}/tests/example-tabs.maf.block.idx +0 -0
- {mafutils-0.2.0 → mafutils-0.2.1}/tests/example-tabs.maf.scaffold.idx +0 -0
- {mafutils-0.2.0 → mafutils-0.2.1}/tests/example.bed +0 -0
- {mafutils-0.2.0 → mafutils-0.2.1}/tests/example.maf +0 -0
- {mafutils-0.2.0 → mafutils-0.2.1}/tests/example.maf.bgz +0 -0
- {mafutils-0.2.0 → mafutils-0.2.1}/tests/example.maf.bgz.block.idx +0 -0
- {mafutils-0.2.0 → mafutils-0.2.1}/tests/example.maf.bgz.scaffold.idx +0 -0
- {mafutils-0.2.0 → mafutils-0.2.1}/tests/example.maf.block.idx +0 -0
- {mafutils-0.2.0 → mafutils-0.2.1}/tests/example.maf.gz +0 -0
- {mafutils-0.2.0 → mafutils-0.2.1}/tests/example.maf.gz.block.idx +0 -0
- {mafutils-0.2.0 → mafutils-0.2.1}/tests/example.maf.gz.scaffold.idx +0 -0
- {mafutils-0.2.0 → mafutils-0.2.1}/tests/example.maf.scaffold.idx +0 -0
- {mafutils-0.2.0 → mafutils-0.2.1}/tests/example.maf.scaffold.regenerated.idx +0 -0
- {mafutils-0.2.0 → mafutils-0.2.1}/tests/expected-fasta/01-singleblock-gap.fa +0 -0
- {mafutils-0.2.0 → mafutils-0.2.1}/tests/expected-fasta/02-truncatedblock.fa +0 -0
- {mafutils-0.2.0 → mafutils-0.2.1}/tests/expected-fasta/03-crossblocks-missing.fa +0 -0
- {mafutils-0.2.0 → mafutils-0.2.1}/tests/expected-fasta/04-singleblock.fa +0 -0
- {mafutils-0.2.0 → mafutils-0.2.1}/tests/expected-fasta/05-crossblocks.fa +0 -0
- {mafutils-0.2.0 → mafutils-0.2.1}/tests/expected-fasta/06-noalign.fa +0 -0
- {mafutils-0.2.0 → mafutils-0.2.1}/tests/expected-fasta/07-negstrand.fa +0 -0
- {mafutils-0.2.0 → mafutils-0.2.1}/tests/expected-fasta/08-span-multiple-gaps.fa +0 -0
- {mafutils-0.2.0 → mafutils-0.2.1}/tests/expected-fasta/09-missingchrom.fa +0 -0
- {mafutils-0.2.0 → mafutils-0.2.1}/tests/expected-fasta/10-crossblocks-missing-species.fa +0 -0
- {mafutils-0.2.0 → mafutils-0.2.1}/tests/expected-maf/01-singleblock-gap.maf +0 -0
- {mafutils-0.2.0 → mafutils-0.2.1}/tests/expected-maf/02-truncatedblock.maf +0 -0
- {mafutils-0.2.0 → mafutils-0.2.1}/tests/expected-maf/03-crossblocks-missing.maf +0 -0
- {mafutils-0.2.0 → mafutils-0.2.1}/tests/expected-maf/04-singleblock.maf +0 -0
- {mafutils-0.2.0 → mafutils-0.2.1}/tests/expected-maf/05-crossblocks.maf +0 -0
- {mafutils-0.2.0 → mafutils-0.2.1}/tests/expected-maf/06-noalign.maf +0 -0
- {mafutils-0.2.0 → mafutils-0.2.1}/tests/expected-maf/07-negstrand.maf +0 -0
- {mafutils-0.2.0 → mafutils-0.2.1}/tests/expected-maf/08-span-multiple-gaps.maf +0 -0
- {mafutils-0.2.0 → mafutils-0.2.1}/tests/expected-maf/09-missingchrom.maf +0 -0
- {mafutils-0.2.0 → mafutils-0.2.1}/tests/expected-maf/10-crossblocks-missing-species.maf +0 -0
- {mafutils-0.2.0 → mafutils-0.2.1}/tests/test_compression.py +0 -0
- {mafutils-0.2.0 → mafutils-0.2.1}/tests/test_fetch.py +0 -0
- {mafutils-0.2.0 → mafutils-0.2.1}/tests/test_gc.py +0 -0
- {mafutils-0.2.0 → mafutils-0.2.1}/tests/test_validate.py +0 -0
|
@@ -1,5 +1,9 @@
|
|
|
1
1
|
# mafutils Development Notes
|
|
2
2
|
|
|
3
|
+
Internal implementation notes, gotchas, and maintainer workflows (testing,
|
|
4
|
+
releasing) for developing `mafutils` itself — see [`README.md`](README.md)
|
|
5
|
+
for user-facing installation/usage docs.
|
|
6
|
+
|
|
3
7
|
- `tests/example.maf.scaffold.idx` is preserved as an older, headerless
|
|
4
8
|
scaffold-index fixture for comparison — it deliberately sits at
|
|
5
9
|
`example.maf`'s *default* scaffold-index path
|
|
@@ -155,6 +159,36 @@
|
|
|
155
159
|
backward-seek risk entirely rather than relying on any per-region
|
|
156
160
|
processing order.
|
|
157
161
|
|
|
162
|
+
## Development setup
|
|
163
|
+
|
|
164
|
+
From a source checkout, an editable install picks up code changes without
|
|
165
|
+
reinstalling:
|
|
166
|
+
|
|
167
|
+
```bash
|
|
168
|
+
pip install -e .
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
Or run directly against the checkout without installing at all:
|
|
172
|
+
|
|
173
|
+
```bash
|
|
174
|
+
python -m mafutils --help
|
|
175
|
+
```
|
|
176
|
+
|
|
177
|
+
## Testing
|
|
178
|
+
|
|
179
|
+
From inside this `mafutils/` directory:
|
|
180
|
+
|
|
181
|
+
```bash
|
|
182
|
+
pytest tests/
|
|
183
|
+
```
|
|
184
|
+
|
|
185
|
+
`tests/test_compression.py` covers cross-cutting compression behavior
|
|
186
|
+
(bgzip/gzip detection, index headers, index auto-derivation, and the
|
|
187
|
+
per-command parallel/sequential/fallback logic) across all three compression
|
|
188
|
+
types. `tests/test_validate.py` covers index integrity (size/mtime/hash
|
|
189
|
+
header fields, `mafutils validate`'s three-way verdict, and `--verify-hash`
|
|
190
|
+
on `fetch`/`stats`/`gc`).
|
|
191
|
+
|
|
158
192
|
## Releasing to PyPI
|
|
159
193
|
|
|
160
194
|
- Publishing is triggered by pushing a Git tag that matches `v*` (for example
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: mafutils
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.1
|
|
4
4
|
Summary: Utilities for indexing, fetching, and summarizing MAFs.
|
|
5
5
|
Author: Gregg Thomas
|
|
6
6
|
License: MIT License
|
|
@@ -50,7 +50,7 @@ It currently provides five commands:
|
|
|
50
50
|
|
|
51
51
|
## Disclaimer
|
|
52
52
|
|
|
53
|
-
This project was developed with significant assistance from
|
|
53
|
+
This project was developed with significant assistance from large language models (GPT-5 / Codex, Claude Sonnet).
|
|
54
54
|
|
|
55
55
|
## Third-Party Code
|
|
56
56
|
|
|
@@ -61,69 +61,62 @@ for full attribution and license text.
|
|
|
61
61
|
|
|
62
62
|
## Installation
|
|
63
63
|
|
|
64
|
-
From this `mafutils/` directory:
|
|
65
|
-
|
|
66
64
|
```bash
|
|
67
|
-
pip install
|
|
65
|
+
pip install mafutils
|
|
68
66
|
```
|
|
69
67
|
|
|
70
|
-
|
|
71
|
-
package environment, you can also run it directly with:
|
|
72
|
-
|
|
73
|
-
```bash
|
|
74
|
-
python -m mafutils --help
|
|
75
|
-
```
|
|
68
|
+
This installs the `mafutils` command.
|
|
76
69
|
|
|
77
70
|
## Quick Start
|
|
78
71
|
|
|
79
72
|
Show top-level help:
|
|
80
73
|
|
|
81
74
|
```bash
|
|
82
|
-
|
|
75
|
+
mafutils --help
|
|
83
76
|
```
|
|
84
77
|
|
|
85
78
|
Create block and scaffold indexes for a MAF (defaults to
|
|
86
79
|
`input.maf.block.idx` / `input.maf.scaffold.idx` if output paths are omitted):
|
|
87
80
|
|
|
88
81
|
```bash
|
|
89
|
-
|
|
82
|
+
mafutils index input.maf
|
|
90
83
|
```
|
|
91
84
|
|
|
92
85
|
Fetch trimmed MAF regions from a BED file (index defaults to
|
|
93
86
|
`input.maf.block.idx`):
|
|
94
87
|
|
|
95
88
|
```bash
|
|
96
|
-
|
|
89
|
+
mafutils fetch input.maf regions.bed -o outdir
|
|
97
90
|
```
|
|
98
91
|
|
|
99
92
|
Fetch FASTA output instead of MAF:
|
|
100
93
|
|
|
101
94
|
```bash
|
|
102
|
-
|
|
95
|
+
mafutils fetch input.maf regions.bed -o outdir -f -fh species-coords-id
|
|
103
96
|
```
|
|
104
97
|
|
|
105
98
|
Extract full scaffolds using a scaffold index:
|
|
106
99
|
|
|
107
100
|
```bash
|
|
108
|
-
|
|
101
|
+
mafutils fetch input.maf scaffolds.bed -m scaffold -o outdir
|
|
109
102
|
```
|
|
110
103
|
|
|
111
104
|
Summarize an indexed MAF:
|
|
112
105
|
|
|
113
106
|
```bash
|
|
114
|
-
|
|
107
|
+
mafutils stats input.maf -o summary/example
|
|
115
108
|
```
|
|
116
109
|
|
|
117
110
|
Calculate per-species GC content:
|
|
118
111
|
|
|
119
112
|
```bash
|
|
120
|
-
|
|
113
|
+
mafutils gc input.maf -o summary/example
|
|
121
114
|
```
|
|
122
115
|
|
|
123
116
|
Check whether an index is still trustworthy (see Index Integrity below):
|
|
124
117
|
|
|
125
118
|
```bash
|
|
126
|
-
|
|
119
|
+
mafutils validate input.maf
|
|
127
120
|
```
|
|
128
121
|
|
|
129
122
|
## Compression
|
|
@@ -139,13 +132,10 @@ python -m mafutils validate input.maf
|
|
|
139
132
|
If you plan to use `--processes > 1` against compressed input, compress with
|
|
140
133
|
`bgzip` rather than plain `gzip` to get real parallel speedup.
|
|
141
134
|
|
|
142
|
-
**No extra dependency for bgzip:** BGZF support is
|
|
143
|
-
`mafutils/lib/bgzf.py
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
`numpy` for no other reason. This does mean upstream bug fixes to that file
|
|
147
|
-
aren't picked up automatically — see `DEVELOPMENT.md` if you're debugging a
|
|
148
|
-
bgzip-related issue.
|
|
135
|
+
**No extra dependency for bgzip:** BGZF support is vendored in
|
|
136
|
+
`mafutils/lib/bgzf.py` rather than depending on the `biopython` package —
|
|
137
|
+
see `DEVELOPMENT.md` for why, and [`THIRD_PARTY_LICENSES.md`](THIRD_PARTY_LICENSES.md)
|
|
138
|
+
for attribution.
|
|
149
139
|
|
|
150
140
|
## Index Integrity
|
|
151
141
|
|
|
@@ -184,7 +174,7 @@ For a "check once, trust thereafter" workflow instead of passing
|
|
|
184
174
|
Create block and scaffold indexes for a MAF file.
|
|
185
175
|
|
|
186
176
|
```bash
|
|
187
|
-
|
|
177
|
+
mafutils index MAF_FILE [BLOCK_INDEX] [SCAFFOLD_INDEX]
|
|
188
178
|
```
|
|
189
179
|
|
|
190
180
|
Arguments:
|
|
@@ -200,7 +190,7 @@ Arguments:
|
|
|
200
190
|
Fetch regions or scaffolds from a MAF using an existing index.
|
|
201
191
|
|
|
202
192
|
```bash
|
|
203
|
-
|
|
193
|
+
mafutils fetch [OPTIONS] MAF_FILE BED_FILE
|
|
204
194
|
```
|
|
205
195
|
|
|
206
196
|
Arguments:
|
|
@@ -239,7 +229,7 @@ file) order, and decodes shared blocks only once even when regions overlap.
|
|
|
239
229
|
Summarize an indexed MAF at overall, species, and block levels.
|
|
240
230
|
|
|
241
231
|
```bash
|
|
242
|
-
|
|
232
|
+
mafutils stats [OPTIONS] MAF_FILE [INDEX_FILE]
|
|
243
233
|
```
|
|
244
234
|
|
|
245
235
|
Arguments:
|
|
@@ -276,7 +266,7 @@ GC is computed as `(G+C) / (A+C+G+T)`, case-insensitive; gaps, `N`, and other
|
|
|
276
266
|
ambiguity codes are excluded from both the numerator and denominator.
|
|
277
267
|
|
|
278
268
|
```bash
|
|
279
|
-
|
|
269
|
+
mafutils gc MAF_FILE [INDEX_FILE] [OPTIONS]
|
|
280
270
|
```
|
|
281
271
|
|
|
282
272
|
Arguments:
|
|
@@ -329,7 +319,7 @@ two indexes weren't built together (e.g. only one was rebuilt) and
|
|
|
329
319
|
shouldn't be trusted as a matched pair.
|
|
330
320
|
|
|
331
321
|
```bash
|
|
332
|
-
|
|
322
|
+
mafutils validate MAF_FILE [INDEX_FILE]
|
|
333
323
|
```
|
|
334
324
|
|
|
335
325
|
Arguments:
|
|
@@ -349,23 +339,7 @@ verdict. Exit codes:
|
|
|
349
339
|
| `1` | MISMATCH | A conclusive difference was found (compression, size, or hash against the MAF file; or the block and scaffold index headers disagree) — rebuild the index(es). |
|
|
350
340
|
| `2` | UNVERIFIABLE | Nothing contradicts, but there's no stored hash to be fully sure (the index predates this feature), or the scaffold index is missing/headerless so the pair couldn't be cross-checked. |
|
|
351
341
|
|
|
352
|
-
## Testing
|
|
353
|
-
|
|
354
|
-
From inside this `mafutils/` directory:
|
|
355
|
-
|
|
356
|
-
```bash
|
|
357
|
-
pytest tests/
|
|
358
|
-
```
|
|
359
|
-
|
|
360
|
-
`tests/test_compression.py` covers cross-cutting compression behavior
|
|
361
|
-
(bgzip/gzip detection, index headers, index auto-derivation, and the
|
|
362
|
-
per-command parallel/sequential/fallback logic described above) across all
|
|
363
|
-
three compression types. `tests/test_validate.py` covers index integrity
|
|
364
|
-
(size/mtime/hash header fields, `mafutils validate`'s three-way verdict, and
|
|
365
|
-
`--verify-hash` on `fetch`/`stats`/`gc`).
|
|
366
|
-
|
|
367
342
|
## Notes
|
|
368
343
|
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
installed.
|
|
344
|
+
For development setup (installing from a source checkout), running the test
|
|
345
|
+
suite, and internal implementation notes, see [`DEVELOPMENT.md`](DEVELOPMENT.md).
|
|
@@ -13,7 +13,7 @@ It currently provides five commands:
|
|
|
13
13
|
|
|
14
14
|
## Disclaimer
|
|
15
15
|
|
|
16
|
-
This project was developed with significant assistance from
|
|
16
|
+
This project was developed with significant assistance from large language models (GPT-5 / Codex, Claude Sonnet).
|
|
17
17
|
|
|
18
18
|
## Third-Party Code
|
|
19
19
|
|
|
@@ -24,69 +24,62 @@ for full attribution and license text.
|
|
|
24
24
|
|
|
25
25
|
## Installation
|
|
26
26
|
|
|
27
|
-
From this `mafutils/` directory:
|
|
28
|
-
|
|
29
27
|
```bash
|
|
30
|
-
pip install
|
|
28
|
+
pip install mafutils
|
|
31
29
|
```
|
|
32
30
|
|
|
33
|
-
|
|
34
|
-
package environment, you can also run it directly with:
|
|
35
|
-
|
|
36
|
-
```bash
|
|
37
|
-
python -m mafutils --help
|
|
38
|
-
```
|
|
31
|
+
This installs the `mafutils` command.
|
|
39
32
|
|
|
40
33
|
## Quick Start
|
|
41
34
|
|
|
42
35
|
Show top-level help:
|
|
43
36
|
|
|
44
37
|
```bash
|
|
45
|
-
|
|
38
|
+
mafutils --help
|
|
46
39
|
```
|
|
47
40
|
|
|
48
41
|
Create block and scaffold indexes for a MAF (defaults to
|
|
49
42
|
`input.maf.block.idx` / `input.maf.scaffold.idx` if output paths are omitted):
|
|
50
43
|
|
|
51
44
|
```bash
|
|
52
|
-
|
|
45
|
+
mafutils index input.maf
|
|
53
46
|
```
|
|
54
47
|
|
|
55
48
|
Fetch trimmed MAF regions from a BED file (index defaults to
|
|
56
49
|
`input.maf.block.idx`):
|
|
57
50
|
|
|
58
51
|
```bash
|
|
59
|
-
|
|
52
|
+
mafutils fetch input.maf regions.bed -o outdir
|
|
60
53
|
```
|
|
61
54
|
|
|
62
55
|
Fetch FASTA output instead of MAF:
|
|
63
56
|
|
|
64
57
|
```bash
|
|
65
|
-
|
|
58
|
+
mafutils fetch input.maf regions.bed -o outdir -f -fh species-coords-id
|
|
66
59
|
```
|
|
67
60
|
|
|
68
61
|
Extract full scaffolds using a scaffold index:
|
|
69
62
|
|
|
70
63
|
```bash
|
|
71
|
-
|
|
64
|
+
mafutils fetch input.maf scaffolds.bed -m scaffold -o outdir
|
|
72
65
|
```
|
|
73
66
|
|
|
74
67
|
Summarize an indexed MAF:
|
|
75
68
|
|
|
76
69
|
```bash
|
|
77
|
-
|
|
70
|
+
mafutils stats input.maf -o summary/example
|
|
78
71
|
```
|
|
79
72
|
|
|
80
73
|
Calculate per-species GC content:
|
|
81
74
|
|
|
82
75
|
```bash
|
|
83
|
-
|
|
76
|
+
mafutils gc input.maf -o summary/example
|
|
84
77
|
```
|
|
85
78
|
|
|
86
79
|
Check whether an index is still trustworthy (see Index Integrity below):
|
|
87
80
|
|
|
88
81
|
```bash
|
|
89
|
-
|
|
82
|
+
mafutils validate input.maf
|
|
90
83
|
```
|
|
91
84
|
|
|
92
85
|
## Compression
|
|
@@ -102,13 +95,10 @@ python -m mafutils validate input.maf
|
|
|
102
95
|
If you plan to use `--processes > 1` against compressed input, compress with
|
|
103
96
|
`bgzip` rather than plain `gzip` to get real parallel speedup.
|
|
104
97
|
|
|
105
|
-
**No extra dependency for bgzip:** BGZF support is
|
|
106
|
-
`mafutils/lib/bgzf.py
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
`numpy` for no other reason. This does mean upstream bug fixes to that file
|
|
110
|
-
aren't picked up automatically — see `DEVELOPMENT.md` if you're debugging a
|
|
111
|
-
bgzip-related issue.
|
|
98
|
+
**No extra dependency for bgzip:** BGZF support is vendored in
|
|
99
|
+
`mafutils/lib/bgzf.py` rather than depending on the `biopython` package —
|
|
100
|
+
see `DEVELOPMENT.md` for why, and [`THIRD_PARTY_LICENSES.md`](THIRD_PARTY_LICENSES.md)
|
|
101
|
+
for attribution.
|
|
112
102
|
|
|
113
103
|
## Index Integrity
|
|
114
104
|
|
|
@@ -147,7 +137,7 @@ For a "check once, trust thereafter" workflow instead of passing
|
|
|
147
137
|
Create block and scaffold indexes for a MAF file.
|
|
148
138
|
|
|
149
139
|
```bash
|
|
150
|
-
|
|
140
|
+
mafutils index MAF_FILE [BLOCK_INDEX] [SCAFFOLD_INDEX]
|
|
151
141
|
```
|
|
152
142
|
|
|
153
143
|
Arguments:
|
|
@@ -163,7 +153,7 @@ Arguments:
|
|
|
163
153
|
Fetch regions or scaffolds from a MAF using an existing index.
|
|
164
154
|
|
|
165
155
|
```bash
|
|
166
|
-
|
|
156
|
+
mafutils fetch [OPTIONS] MAF_FILE BED_FILE
|
|
167
157
|
```
|
|
168
158
|
|
|
169
159
|
Arguments:
|
|
@@ -202,7 +192,7 @@ file) order, and decodes shared blocks only once even when regions overlap.
|
|
|
202
192
|
Summarize an indexed MAF at overall, species, and block levels.
|
|
203
193
|
|
|
204
194
|
```bash
|
|
205
|
-
|
|
195
|
+
mafutils stats [OPTIONS] MAF_FILE [INDEX_FILE]
|
|
206
196
|
```
|
|
207
197
|
|
|
208
198
|
Arguments:
|
|
@@ -239,7 +229,7 @@ GC is computed as `(G+C) / (A+C+G+T)`, case-insensitive; gaps, `N`, and other
|
|
|
239
229
|
ambiguity codes are excluded from both the numerator and denominator.
|
|
240
230
|
|
|
241
231
|
```bash
|
|
242
|
-
|
|
232
|
+
mafutils gc MAF_FILE [INDEX_FILE] [OPTIONS]
|
|
243
233
|
```
|
|
244
234
|
|
|
245
235
|
Arguments:
|
|
@@ -292,7 +282,7 @@ two indexes weren't built together (e.g. only one was rebuilt) and
|
|
|
292
282
|
shouldn't be trusted as a matched pair.
|
|
293
283
|
|
|
294
284
|
```bash
|
|
295
|
-
|
|
285
|
+
mafutils validate MAF_FILE [INDEX_FILE]
|
|
296
286
|
```
|
|
297
287
|
|
|
298
288
|
Arguments:
|
|
@@ -312,23 +302,7 @@ verdict. Exit codes:
|
|
|
312
302
|
| `1` | MISMATCH | A conclusive difference was found (compression, size, or hash against the MAF file; or the block and scaffold index headers disagree) — rebuild the index(es). |
|
|
313
303
|
| `2` | UNVERIFIABLE | Nothing contradicts, but there's no stored hash to be fully sure (the index predates this feature), or the scaffold index is missing/headerless so the pair couldn't be cross-checked. |
|
|
314
304
|
|
|
315
|
-
## Testing
|
|
316
|
-
|
|
317
|
-
From inside this `mafutils/` directory:
|
|
318
|
-
|
|
319
|
-
```bash
|
|
320
|
-
pytest tests/
|
|
321
|
-
```
|
|
322
|
-
|
|
323
|
-
`tests/test_compression.py` covers cross-cutting compression behavior
|
|
324
|
-
(bgzip/gzip detection, index headers, index auto-derivation, and the
|
|
325
|
-
per-command parallel/sequential/fallback logic described above) across all
|
|
326
|
-
three compression types. `tests/test_validate.py` covers index integrity
|
|
327
|
-
(size/mtime/hash header fields, `mafutils validate`'s three-way verdict, and
|
|
328
|
-
`--verify-hash` on `fetch`/`stats`/`gc`).
|
|
329
|
-
|
|
330
305
|
## Notes
|
|
331
306
|
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
installed.
|
|
307
|
+
For development setup (installing from a source checkout), running the test
|
|
308
|
+
suite, and internal implementation notes, see [`DEVELOPMENT.md`](DEVELOPMENT.md).
|
|
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
|
|
|
18
18
|
commit_id: str | None
|
|
19
19
|
__commit_id__: str | None
|
|
20
20
|
|
|
21
|
-
__version__ = version = '0.2.
|
|
22
|
-
__version_tuple__ = version_tuple = (0, 2,
|
|
21
|
+
__version__ = version = '0.2.1'
|
|
22
|
+
__version_tuple__ = version_tuple = (0, 2, 1)
|
|
23
23
|
|
|
24
|
-
__commit_id__ = commit_id = '
|
|
24
|
+
__commit_id__ = commit_id = 'g5c611ce9e'
|
|
@@ -1075,8 +1075,15 @@ def fetchScaffoldsSequential(ordered_scaffolds, index, maf_file, maf_header, maf
|
|
|
1075
1075
|
|
|
1076
1076
|
def run_fetch(args, cmdline="mafutils fetch"):
|
|
1077
1077
|
|
|
1078
|
+
if not args.single_output:
|
|
1079
|
+
os.makedirs(args.output, exist_ok=True);
|
|
1080
|
+
# Create the output directory before deciding where the log goes -- the
|
|
1081
|
+
# log path check below needs args.output to already exist, since that's
|
|
1082
|
+
# the whole point of checking os.path.isdir() rather than just always
|
|
1083
|
+
# joining onto it.
|
|
1084
|
+
|
|
1078
1085
|
logfilename = os.path.join(args.output, "maf_fetch.log") if os.path.isdir(args.output) else "maf_fetch.log"
|
|
1079
|
-
maf_fetch_logger = LOGINIT.configureLogging(log_level="INFO", log_verbosity="BOTH", log_filename=logfilename, logger_name="maf_fetch_logger")
|
|
1086
|
+
maf_fetch_logger = LOGINIT.configureLogging(log_level="INFO", log_verbosity="BOTH", log_filename=logfilename, logger_name="maf_fetch_logger", overwrite_log_file=True)
|
|
1080
1087
|
LOG = logging.getLogger(maf_fetch_logger)
|
|
1081
1088
|
# Set up logging
|
|
1082
1089
|
|
|
@@ -1105,9 +1112,6 @@ def run_fetch(args, cmdline="mafutils fetch"):
|
|
|
1105
1112
|
if maf_compression not in ("none", "gz", "bgzip"):
|
|
1106
1113
|
LOG.error(f"Unsupported MAF compression: {maf_compression}")
|
|
1107
1114
|
sys.exit(1)
|
|
1108
|
-
if not args.single_output:
|
|
1109
|
-
os.makedirs(args.output, exist_ok=True);
|
|
1110
|
-
# Create output directory if it doesn't exist
|
|
1111
1115
|
|
|
1112
1116
|
index_file = args.index_file
|
|
1113
1117
|
if index_file is None:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: mafutils
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.1
|
|
4
4
|
Summary: Utilities for indexing, fetching, and summarizing MAFs.
|
|
5
5
|
Author: Gregg Thomas
|
|
6
6
|
License: MIT License
|
|
@@ -50,7 +50,7 @@ It currently provides five commands:
|
|
|
50
50
|
|
|
51
51
|
## Disclaimer
|
|
52
52
|
|
|
53
|
-
This project was developed with significant assistance from
|
|
53
|
+
This project was developed with significant assistance from large language models (GPT-5 / Codex, Claude Sonnet).
|
|
54
54
|
|
|
55
55
|
## Third-Party Code
|
|
56
56
|
|
|
@@ -61,69 +61,62 @@ for full attribution and license text.
|
|
|
61
61
|
|
|
62
62
|
## Installation
|
|
63
63
|
|
|
64
|
-
From this `mafutils/` directory:
|
|
65
|
-
|
|
66
64
|
```bash
|
|
67
|
-
pip install
|
|
65
|
+
pip install mafutils
|
|
68
66
|
```
|
|
69
67
|
|
|
70
|
-
|
|
71
|
-
package environment, you can also run it directly with:
|
|
72
|
-
|
|
73
|
-
```bash
|
|
74
|
-
python -m mafutils --help
|
|
75
|
-
```
|
|
68
|
+
This installs the `mafutils` command.
|
|
76
69
|
|
|
77
70
|
## Quick Start
|
|
78
71
|
|
|
79
72
|
Show top-level help:
|
|
80
73
|
|
|
81
74
|
```bash
|
|
82
|
-
|
|
75
|
+
mafutils --help
|
|
83
76
|
```
|
|
84
77
|
|
|
85
78
|
Create block and scaffold indexes for a MAF (defaults to
|
|
86
79
|
`input.maf.block.idx` / `input.maf.scaffold.idx` if output paths are omitted):
|
|
87
80
|
|
|
88
81
|
```bash
|
|
89
|
-
|
|
82
|
+
mafutils index input.maf
|
|
90
83
|
```
|
|
91
84
|
|
|
92
85
|
Fetch trimmed MAF regions from a BED file (index defaults to
|
|
93
86
|
`input.maf.block.idx`):
|
|
94
87
|
|
|
95
88
|
```bash
|
|
96
|
-
|
|
89
|
+
mafutils fetch input.maf regions.bed -o outdir
|
|
97
90
|
```
|
|
98
91
|
|
|
99
92
|
Fetch FASTA output instead of MAF:
|
|
100
93
|
|
|
101
94
|
```bash
|
|
102
|
-
|
|
95
|
+
mafutils fetch input.maf regions.bed -o outdir -f -fh species-coords-id
|
|
103
96
|
```
|
|
104
97
|
|
|
105
98
|
Extract full scaffolds using a scaffold index:
|
|
106
99
|
|
|
107
100
|
```bash
|
|
108
|
-
|
|
101
|
+
mafutils fetch input.maf scaffolds.bed -m scaffold -o outdir
|
|
109
102
|
```
|
|
110
103
|
|
|
111
104
|
Summarize an indexed MAF:
|
|
112
105
|
|
|
113
106
|
```bash
|
|
114
|
-
|
|
107
|
+
mafutils stats input.maf -o summary/example
|
|
115
108
|
```
|
|
116
109
|
|
|
117
110
|
Calculate per-species GC content:
|
|
118
111
|
|
|
119
112
|
```bash
|
|
120
|
-
|
|
113
|
+
mafutils gc input.maf -o summary/example
|
|
121
114
|
```
|
|
122
115
|
|
|
123
116
|
Check whether an index is still trustworthy (see Index Integrity below):
|
|
124
117
|
|
|
125
118
|
```bash
|
|
126
|
-
|
|
119
|
+
mafutils validate input.maf
|
|
127
120
|
```
|
|
128
121
|
|
|
129
122
|
## Compression
|
|
@@ -139,13 +132,10 @@ python -m mafutils validate input.maf
|
|
|
139
132
|
If you plan to use `--processes > 1` against compressed input, compress with
|
|
140
133
|
`bgzip` rather than plain `gzip` to get real parallel speedup.
|
|
141
134
|
|
|
142
|
-
**No extra dependency for bgzip:** BGZF support is
|
|
143
|
-
`mafutils/lib/bgzf.py
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
`numpy` for no other reason. This does mean upstream bug fixes to that file
|
|
147
|
-
aren't picked up automatically — see `DEVELOPMENT.md` if you're debugging a
|
|
148
|
-
bgzip-related issue.
|
|
135
|
+
**No extra dependency for bgzip:** BGZF support is vendored in
|
|
136
|
+
`mafutils/lib/bgzf.py` rather than depending on the `biopython` package —
|
|
137
|
+
see `DEVELOPMENT.md` for why, and [`THIRD_PARTY_LICENSES.md`](THIRD_PARTY_LICENSES.md)
|
|
138
|
+
for attribution.
|
|
149
139
|
|
|
150
140
|
## Index Integrity
|
|
151
141
|
|
|
@@ -184,7 +174,7 @@ For a "check once, trust thereafter" workflow instead of passing
|
|
|
184
174
|
Create block and scaffold indexes for a MAF file.
|
|
185
175
|
|
|
186
176
|
```bash
|
|
187
|
-
|
|
177
|
+
mafutils index MAF_FILE [BLOCK_INDEX] [SCAFFOLD_INDEX]
|
|
188
178
|
```
|
|
189
179
|
|
|
190
180
|
Arguments:
|
|
@@ -200,7 +190,7 @@ Arguments:
|
|
|
200
190
|
Fetch regions or scaffolds from a MAF using an existing index.
|
|
201
191
|
|
|
202
192
|
```bash
|
|
203
|
-
|
|
193
|
+
mafutils fetch [OPTIONS] MAF_FILE BED_FILE
|
|
204
194
|
```
|
|
205
195
|
|
|
206
196
|
Arguments:
|
|
@@ -239,7 +229,7 @@ file) order, and decodes shared blocks only once even when regions overlap.
|
|
|
239
229
|
Summarize an indexed MAF at overall, species, and block levels.
|
|
240
230
|
|
|
241
231
|
```bash
|
|
242
|
-
|
|
232
|
+
mafutils stats [OPTIONS] MAF_FILE [INDEX_FILE]
|
|
243
233
|
```
|
|
244
234
|
|
|
245
235
|
Arguments:
|
|
@@ -276,7 +266,7 @@ GC is computed as `(G+C) / (A+C+G+T)`, case-insensitive; gaps, `N`, and other
|
|
|
276
266
|
ambiguity codes are excluded from both the numerator and denominator.
|
|
277
267
|
|
|
278
268
|
```bash
|
|
279
|
-
|
|
269
|
+
mafutils gc MAF_FILE [INDEX_FILE] [OPTIONS]
|
|
280
270
|
```
|
|
281
271
|
|
|
282
272
|
Arguments:
|
|
@@ -329,7 +319,7 @@ two indexes weren't built together (e.g. only one was rebuilt) and
|
|
|
329
319
|
shouldn't be trusted as a matched pair.
|
|
330
320
|
|
|
331
321
|
```bash
|
|
332
|
-
|
|
322
|
+
mafutils validate MAF_FILE [INDEX_FILE]
|
|
333
323
|
```
|
|
334
324
|
|
|
335
325
|
Arguments:
|
|
@@ -349,23 +339,7 @@ verdict. Exit codes:
|
|
|
349
339
|
| `1` | MISMATCH | A conclusive difference was found (compression, size, or hash against the MAF file; or the block and scaffold index headers disagree) — rebuild the index(es). |
|
|
350
340
|
| `2` | UNVERIFIABLE | Nothing contradicts, but there's no stored hash to be fully sure (the index predates this feature), or the scaffold index is missing/headerless so the pair couldn't be cross-checked. |
|
|
351
341
|
|
|
352
|
-
## Testing
|
|
353
|
-
|
|
354
|
-
From inside this `mafutils/` directory:
|
|
355
|
-
|
|
356
|
-
```bash
|
|
357
|
-
pytest tests/
|
|
358
|
-
```
|
|
359
|
-
|
|
360
|
-
`tests/test_compression.py` covers cross-cutting compression behavior
|
|
361
|
-
(bgzip/gzip detection, index headers, index auto-derivation, and the
|
|
362
|
-
per-command parallel/sequential/fallback logic described above) across all
|
|
363
|
-
three compression types. `tests/test_validate.py` covers index integrity
|
|
364
|
-
(size/mtime/hash header fields, `mafutils validate`'s three-way verdict, and
|
|
365
|
-
`--verify-hash` on `fetch`/`stats`/`gc`).
|
|
366
|
-
|
|
367
342
|
## Notes
|
|
368
343
|
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
installed.
|
|
344
|
+
For development setup (installing from a source checkout), running the test
|
|
345
|
+
suite, and internal implementation notes, see [`DEVELOPMENT.md`](DEVELOPMENT.md).
|
|
@@ -5,7 +5,6 @@ DEVELOPMENT.md
|
|
|
5
5
|
LICENSE
|
|
6
6
|
README.md
|
|
7
7
|
THIRD_PARTY_LICENSES.md
|
|
8
|
-
maf_fetch.log
|
|
9
8
|
pyproject.toml
|
|
10
9
|
.github/workflows/publish-pypi.yml
|
|
11
10
|
benchmarks/README.md
|
|
@@ -13,7 +12,6 @@ benchmarks/Snakefile
|
|
|
13
12
|
benchmarks/baseline-manual-timings.txt
|
|
14
13
|
benchmarks/config.yaml
|
|
15
14
|
benchmarks/history.csv
|
|
16
|
-
benchmarks/maf_fetch.log
|
|
17
15
|
benchmarks/notebook.ipynb
|
|
18
16
|
benchmarks/parse_benchmarks.py
|
|
19
17
|
benchmarks/rulegraph.png
|
|
@@ -4,7 +4,6 @@
|
|
|
4
4
|
"README.md",
|
|
5
5
|
".codex",
|
|
6
6
|
"DEVELOPMENT.md",
|
|
7
|
-
"maf_fetch.log",
|
|
8
7
|
"pyproject.toml",
|
|
9
8
|
"THIRD_PARTY_LICENSES.md",
|
|
10
9
|
"LICENSE",
|
|
@@ -15,7 +14,6 @@
|
|
|
15
14
|
"benchmarks/baseline-manual-timings.txt",
|
|
16
15
|
"benchmarks/rulegraph.png",
|
|
17
16
|
"benchmarks/parse_benchmarks.py",
|
|
18
|
-
"benchmarks/maf_fetch.log",
|
|
19
17
|
"benchmarks/notebook.ipynb",
|
|
20
18
|
"benchmarks/config.yaml",
|
|
21
19
|
"benchmarks/profiles/slurm/config.yaml",
|