hicstuff 3.2.4__tar.gz → 3.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- hicstuff-3.3.0/PKG-INFO +512 -0
- hicstuff-3.3.0/README.md +461 -0
- hicstuff-3.3.0/pyproject.toml +132 -0
- {hicstuff-3.2.4 → hicstuff-3.3.0}/setup.cfg +0 -3
- {hicstuff-3.2.4 → hicstuff-3.3.0/src}/hicstuff/__init__.py +11 -2
- hicstuff-3.3.0/src/hicstuff/_misc.py +1162 -0
- hicstuff-3.3.0/src/hicstuff/cli.py +1607 -0
- {hicstuff-3.2.4 → hicstuff-3.3.0/src}/hicstuff/cutsite.py +25 -38
- {hicstuff-3.2.4 → hicstuff-3.3.0/src}/hicstuff/digest.py +111 -115
- {hicstuff-3.2.4 → hicstuff-3.3.0/src}/hicstuff/distance_law.py +281 -146
- {hicstuff-3.2.4 → hicstuff-3.3.0/src}/hicstuff/filter.py +85 -70
- hicstuff-3.3.0/src/hicstuff/hicstuff.py +897 -0
- {hicstuff-3.2.4 → hicstuff-3.3.0/src}/hicstuff/io.py +363 -303
- {hicstuff-3.2.4 → hicstuff-3.3.0/src}/hicstuff/iteralign.py +50 -56
- hicstuff-3.3.0/src/hicstuff/log.py +62 -0
- {hicstuff-3.2.4 → hicstuff-3.3.0/src}/hicstuff/pipeline.py +544 -279
- hicstuff-3.3.0/src/hicstuff/stats.py +240 -0
- {hicstuff-3.2.4 → hicstuff-3.3.0/src}/hicstuff/view.py +21 -29
- hicstuff-3.3.0/src/hicstuff.egg-info/PKG-INFO +512 -0
- hicstuff-3.3.0/src/hicstuff.egg-info/SOURCES.txt +38 -0
- hicstuff-3.3.0/src/hicstuff.egg-info/entry_points.txt +2 -0
- hicstuff-3.3.0/src/hicstuff.egg-info/requires.txt +21 -0
- {hicstuff-3.2.4 → hicstuff-3.3.0/src}/hicstuff.egg-info/top_level.txt +0 -1
- hicstuff-3.3.0/tests/test_commands.py +297 -0
- hicstuff-3.3.0/tests/test_digest.py +177 -0
- hicstuff-3.3.0/tests/test_distance_law.py +299 -0
- hicstuff-3.3.0/tests/test_doctests.py +23 -0
- hicstuff-3.3.0/tests/test_filter.py +194 -0
- hicstuff-3.3.0/tests/test_hicstuff.py +221 -0
- hicstuff-3.3.0/tests/test_io.py +574 -0
- hicstuff-3.3.0/tests/test_iteralign.py +258 -0
- hicstuff-3.3.0/tests/test_pipeline.py +763 -0
- hicstuff-3.3.0/tests/test_synthetic_cli.py +683 -0
- hicstuff-3.3.0/tests/test_synthetic_generator.py +158 -0
- hicstuff-3.3.0/tests/test_synthetic_io.py +430 -0
- hicstuff-3.3.0/tests/test_synthetic_matrices.py +180 -0
- hicstuff-3.3.0/tests/test_synthetic_pipeline.py +951 -0
- hicstuff-3.3.0/tests/test_synthetic_stats.py +174 -0
- hicstuff-3.2.4/MANIFEST.in +0 -3
- hicstuff-3.2.4/PKG-INFO +0 -431
- hicstuff-3.2.4/README.md +0 -403
- hicstuff-3.2.4/hicstuff/commands.py +0 -1756
- hicstuff-3.2.4/hicstuff/hicstuff.py +0 -2208
- hicstuff-3.2.4/hicstuff/log.py +0 -92
- hicstuff-3.2.4/hicstuff/main.py +0 -61
- hicstuff-3.2.4/hicstuff/stats.py +0 -185
- hicstuff-3.2.4/hicstuff/version.py +0 -1
- hicstuff-3.2.4/hicstuff.egg-info/PKG-INFO +0 -431
- hicstuff-3.2.4/hicstuff.egg-info/SOURCES.txt +0 -27
- hicstuff-3.2.4/hicstuff.egg-info/entry_points.txt +0 -3
- hicstuff-3.2.4/hicstuff.egg-info/requires.txt +0 -15
- hicstuff-3.2.4/requirements.txt +0 -12
- hicstuff-3.2.4/setup.py +0 -68
- {hicstuff-3.2.4 → hicstuff-3.3.0}/LICENSE +0 -0
- {hicstuff-3.2.4 → hicstuff-3.3.0/src}/hicstuff.egg-info/dependency_links.txt +0 -0
hicstuff-3.3.0/PKG-INFO
ADDED
|
@@ -0,0 +1,512 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: hicstuff
|
|
3
|
+
Version: 3.3.0
|
|
4
|
+
Summary: General purpose stuff to generate and handle Hi-C data in its simplest form.
|
|
5
|
+
Author-email: Jacques Serizay <jacques.serizay@pasteur.fr>, Cyril Matthey-Doret <cyril.matthey-doret@pasteur.fr>, Lyam Baudry <lyam.baudry@pasteur.fr>
|
|
6
|
+
License: BSD-3-Clause
|
|
7
|
+
Project-URL: Homepage, https://github.com/koszullab/hicstuff
|
|
8
|
+
Project-URL: Documentation, https://hicstuff.readthedocs.io/en/latest
|
|
9
|
+
Project-URL: Repository, https://github.com/koszullab/hicstuff
|
|
10
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
11
|
+
Classifier: Intended Audience :: Science/Research
|
|
12
|
+
Classifier: License :: OSI Approved :: BSD License
|
|
13
|
+
Classifier: Programming Language :: Python
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
23
|
+
Classifier: Topic :: Scientific/Engineering
|
|
24
|
+
Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
|
|
25
|
+
Classifier: Topic :: Scientific/Engineering :: Visualization
|
|
26
|
+
Classifier: Operating System :: POSIX
|
|
27
|
+
Classifier: Operating System :: Unix
|
|
28
|
+
Classifier: Operating System :: MacOS
|
|
29
|
+
Requires-Python: >=3.8
|
|
30
|
+
Description-Content-Type: text/markdown
|
|
31
|
+
License-File: LICENSE
|
|
32
|
+
Requires-Dist: numpy
|
|
33
|
+
Requires-Dist: scipy
|
|
34
|
+
Requires-Dist: pandas>=1.5.0
|
|
35
|
+
Requires-Dist: matplotlib>=3.4.0
|
|
36
|
+
Requires-Dist: rich-click>=1.7
|
|
37
|
+
Requires-Dist: biopython
|
|
38
|
+
Requires-Dist: pysam
|
|
39
|
+
Requires-Dist: pyfastx
|
|
40
|
+
Requires-Dist: cooler
|
|
41
|
+
Provides-Extra: dev
|
|
42
|
+
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
43
|
+
Requires-Dist: pytest-cov>=4.0; extra == "dev"
|
|
44
|
+
Requires-Dist: ruff>=0.4; extra == "dev"
|
|
45
|
+
Requires-Dist: pre-commit>=3.0; extra == "dev"
|
|
46
|
+
Requires-Dist: sphinx; extra == "dev"
|
|
47
|
+
Requires-Dist: sphinx-rtd-theme; extra == "dev"
|
|
48
|
+
Requires-Dist: ipython>=8.12.3; extra == "dev"
|
|
49
|
+
Requires-Dist: bump-my-version>=1.2; python_version >= "3.10" and extra == "dev"
|
|
50
|
+
Dynamic: license-file
|
|
51
|
+
|
|
52
|
+
# hicstuff
|
|
53
|
+
|
|
54
|
+
[](https://badge.fury.io/py/hicstuff)
|
|
55
|
+

|
|
56
|
+
[](http://bioconda.github.io/recipes/hicstuff/README.html)
|
|
57
|
+
[](https://github.com/koszullab/hicstuff/actions/workflows/ci.yml)
|
|
58
|
+
[](https://doi.org/10.5281/zenodo.2620608)
|
|
59
|
+
[](https://codecov.io/gh/koszullab/hicstuff)
|
|
60
|
+
[](https://hicstuff.readthedocs.io)
|
|
61
|
+
[](https://github.com/astral-sh/ruff)
|
|
62
|
+
|
|
63
|
+
A lightweight library that generates and handles Hi-C contact maps in either cooler-compatible 2Dbedgraph or [instaGRAAL](https://github.com/koszullab/instaGRAAL) format. It is essentially a merge of the [yahcp](https://github.com/baudrly/yahcp) pipeline, the [hicstuff](https://github.com/baudrly/hicstuff) library and extra features illustrated in the [3C tutorial](https://github.com/axelcournac/3C_tutorial) and the [DADE pipeline](https://github.com/scovit/dade), all packaged together for extra convenience.
|
|
64
|
+
|
|
65
|
+
The goal is to make generation and manipulation of Hi-C matrices as simple as possible and work for any organism.
|
|
66
|
+
|
|
67
|
+
## Table of contents
|
|
68
|
+
|
|
69
|
+
1. [Table of contents](#table-of-contents)
|
|
70
|
+
2. [Installation](#installation)
|
|
71
|
+
1. [pip](#pip)
|
|
72
|
+
2. [Conda](#conda)
|
|
73
|
+
3. [Docker](#docker)
|
|
74
|
+
3. [Usage](#usage)
|
|
75
|
+
1. [Full pipeline](#full-pipeline)
|
|
76
|
+
2. [Individual components](#individual-components)
|
|
77
|
+
1. [Digestion of fastq reads](#digestion-of-fastq-reads)
|
|
78
|
+
2. [Iterative alignment](#iterative-alignment)
|
|
79
|
+
3. [Digestion of the genome](#digestion-of-the-genome)
|
|
80
|
+
4. [Filtering of 3C events](#filtering-of-3c-events)
|
|
81
|
+
5. [Viewing the contact map](#viewing-the-contact-map)
|
|
82
|
+
3. [File formats](#file-formats)
|
|
83
|
+
4. [Contributing](#contributing)
|
|
84
|
+
5. [Releasing](#releasing)
|
|
85
|
+
6. [Citation](#citation)
|
|
86
|
+
|
|
87
|
+
## Installation
|
|
88
|
+
|
|
89
|
+
### pip
|
|
90
|
+
|
|
91
|
+
To install a stable version:
|
|
92
|
+
```sh
|
|
93
|
+
pip install hicstuff
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
or, for the latest development version:
|
|
97
|
+
|
|
98
|
+
```sh
|
|
99
|
+
git clone https://github.com/koszullab/hicstuff.git
|
|
100
|
+
cd hicstuff
|
|
101
|
+
uv sync --extra dev
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
If `hicstuff` is installed via pip, additional system dependencies such as
|
|
105
|
+
`bowtie2`, `bwa`, `minimap2` and `samtools` will be needed. You can install
|
|
106
|
+
them as follows on Debian/Ubuntu:
|
|
107
|
+
|
|
108
|
+
```bash
|
|
109
|
+
apt-get install bowtie2 bwa minimap2 samtools libbz2-dev liblzma-dev
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
### Conda
|
|
113
|
+
|
|
114
|
+
hicstuff is also available on bioconda, which will automatically handle all
|
|
115
|
+
dependencies:
|
|
116
|
+
|
|
117
|
+
```bash
|
|
118
|
+
conda install bioconda::hicstuff
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
### Docker
|
|
122
|
+
|
|
123
|
+
Docker images are automatically built and published to GitHub Container
|
|
124
|
+
Registry (GHCR) when releases are tagged:
|
|
125
|
+
|
|
126
|
+
```bash
|
|
127
|
+
docker pull ghcr.io/baudrly/hicstuff:latest
|
|
128
|
+
docker pull ghcr.io/baudrly/hicstuff:<version>
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
## Usage
|
|
132
|
+
|
|
133
|
+
The hicstuff command line interface is composed of multiple subcommands.
|
|
134
|
+
You can always get a summary of all available commands by running:
|
|
135
|
+
|
|
136
|
+
```txt
|
|
137
|
+
hicstuff --help
|
|
138
|
+
|
|
139
|
+
Usage: hicstuff [OPTIONS] COMMAND [ARGS]...
|
|
140
|
+
|
|
141
|
+
Simple Hi-C pipeline for generating and manipulating contact matrices.
|
|
142
|
+
|
|
143
|
+
╭─ Options ────────────────────────────────────────────────────────────────────────────────────────╮
|
|
144
|
+
│ --version Show the version and exit. │
|
|
145
|
+
│ --help -h Show this message and exit. │
|
|
146
|
+
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
147
|
+
╭─ Alignment ──────────────────────────────────────────────────────────────────────────────────────╮
|
|
148
|
+
│ iteralign Iteratively align reads to a reference genome. │
|
|
149
|
+
│ cutsite Preprocess FASTQ files by cutting reads at religation sites. │
|
|
150
|
+
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
151
|
+
╭─ Processing ─────────────────────────────────────────────────────────────────────────────────────╮
|
|
152
|
+
│ digest Digest a genome FASTA into restriction fragments. │
|
|
153
|
+
│ filter Filter spurious Hi-C events (loops and uncuts) from a pairs file. │
|
|
154
|
+
│ pipeline Run the full Hi-C pipeline from FASTQ to contact matrix. │
|
|
155
|
+
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
156
|
+
╭─ Matrix operations ──────────────────────────────────────────────────────────────────────────────╮
|
|
157
|
+
│ rebin Rebin a Hi-C matrix to a coarser resolution. │
|
|
158
|
+
│ subsample Subsample contacts from a Hi-C matrix. │
|
|
159
|
+
│ convert Convert between Hi-C matrix formats (graal, bg2, cool). │
|
|
160
|
+
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
161
|
+
╭─ Analysis & Visualization ───────────────────────────────────────────────────────────────────────╮
|
|
162
|
+
│ view Visualize a Hi-C matrix as a heatmap. │
|
|
163
|
+
│ scalogram Generate a scalogram from a Hi-C contact matrix. │
|
|
164
|
+
│ distancelaw Analyse and plot the Hi-C distance law (P(s) curve). │
|
|
165
|
+
│ missview Preview unmappable Hi-C bins for a given read length. │
|
|
166
|
+
│ stats Extract mapping statistics from a hicstuff pipeline log file. │
|
|
167
|
+
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
### Full pipeline
|
|
171
|
+
|
|
172
|
+
All components of the pipeline can be run at once using the `hicstuff pipeline` command.
|
|
173
|
+
This allows to generate a contact matrix from reads in a single command.
|
|
174
|
+
By default, the output is a `cool` file.
|
|
175
|
+
|
|
176
|
+
```txt
|
|
177
|
+
hicstuff pipeline --help
|
|
178
|
+
|
|
179
|
+
Usage: hicstuff pipeline [OPTIONS] INPUT1 [INPUT2]
|
|
180
|
+
|
|
181
|
+
Run the full Hi-C pipeline from FASTQ to contact matrix.
|
|
182
|
+
Example — generate a multi-resolution .mcool for Arima Hi-C:
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
hicstuff pipeline --enzyme "DpnII,HinfI" --binning 1000 --threads 8 \
|
|
186
|
+
--genome ref.fa R1.fq.gz R2.fq.gz
|
|
187
|
+
|
|
188
|
+
╭─ Arguments ──────────────────────────────────────────────────────────────────────────────────────╮
|
|
189
|
+
│ * INPUT1 TEXT [required] │
|
|
190
|
+
│ INPUT2 TEXT │
|
|
191
|
+
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
192
|
+
╭─ Options ────────────────────────────────────────────────────────────────────────────────────────╮
|
|
193
|
+
│ * --genome -g FILE Reference genome or aligner index. [required] │
|
|
194
|
+
│ --aligner -a STR Aligner: bowtie2, minimap2 or bwa. [default: bowtie2] │
|
|
195
|
+
│ --balancing-args -B STR Extra arguments passed to `cooler balance`. │
|
|
196
|
+
│ --binning -b INT Bin the cool matrix to this resolution (bp). 0 means no │
|
|
197
|
+
│ binning. [default: 0] │
|
|
198
|
+
│ --centromeres -c FILE Centromere positions file. │
|
|
199
|
+
│ --circular -C Genome is circular. │
|
|
200
|
+
│ --distance-law -d Generate a distance law output file. │
|
|
201
|
+
│ --duplicates -D Filter PCR duplicates. │
|
|
202
|
+
│ --enzyme -e {STR|INT} Restriction enzyme name, 'mnase'/'dnase', or chunk size │
|
|
203
|
+
│ in bp. [default: 5000] │
|
|
204
|
+
│ --exclude -E STR Comma-separated chromosomes to exclude (e.g. chrM,2u). │
|
|
205
|
+
│ --filter -f Filter spurious 3C events (loops and uncuts). │
|
|
206
|
+
│ --force -F Overwrite existing output files. │
|
|
207
|
+
│ --mapping -m STR Mapping mode. [default: normal] │
|
|
208
|
+
│ --matfmt -M STR Output matrix format. [default: cool] │
|
|
209
|
+
│ --no-cleanup -n Keep intermediary files. │
|
|
210
|
+
│ --outdir -o DIR Output directory (default: current directory). │
|
|
211
|
+
│ --plot -p Generate plots at pipeline steps. │
|
|
212
|
+
│ --prefix -P STR Prefix for all output files. │
|
|
213
|
+
│ --quality-min -q INT Minimum mapping quality. [default: 30] │
|
|
214
|
+
│ --remove-centromeres -r INT kb to remove around centromere positions. [default: 0] │
|
|
215
|
+
│ --read-len -R INT Maximum read length (estimated from first read if │
|
|
216
|
+
│ omitted). │
|
|
217
|
+
│ --size -s INT Minimum contig size (bp): shorter contigs, and the pairs │
|
|
218
|
+
│ on them, are left out. [default: 0] │
|
|
219
|
+
│ --start-stage -S STR Pipeline start stage. [default: fastq] │
|
|
220
|
+
│ --threads -t INT Number of parallel threads. [default: 1] │
|
|
221
|
+
│ --tmpdir -T DIR Temporary directory. │
|
|
222
|
+
│ --zoomify -z BOOL Generate multi-resolution .mcool from the binned cool │
|
|
223
|
+
│ matrix. [default: True] │
|
|
224
|
+
│ --help -h Show this message and exit. │
|
|
225
|
+
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
226
|
+
```
|
|
227
|
+
|
|
228
|
+
For example, to run the pipeline with `minimap2` using 8 threads and save output
|
|
229
|
+
files to the directory `out`:
|
|
230
|
+
|
|
231
|
+
```sh
|
|
232
|
+
hicstuff pipeline -t 8 -a minimap2 -e DpnII -o out/ -g genome.fa reads_for.fq reads_rev.fq
|
|
233
|
+
```
|
|
234
|
+
|
|
235
|
+
If you have already aligned your reads, hicstuff pipeline can also take bam files as input. For example,
|
|
236
|
+
to generate a matrix in mcool file with multiple resolutions starting from 5kb, you can run:
|
|
237
|
+
|
|
238
|
+
```sh
|
|
239
|
+
# Note the bam files have to be name-sorted, this can be done using samtools
|
|
240
|
+
samtools sort -n aligned_for.bam -o namesorted_for.bam
|
|
241
|
+
samtools sort -n aligned_rev.bam -o namesorted_rev.bam
|
|
242
|
+
hicstuff pipeline -S bam -b 5000 -o out/ -g genome.fa namesorted_for.bam namesorted_rev.bam
|
|
243
|
+
```
|
|
244
|
+
|
|
245
|
+
The general steps of the pipeline are as follows:
|
|
246
|
+
|
|
247
|
+

|
|
248
|
+
|
|
249
|
+
### Individual components
|
|
250
|
+
|
|
251
|
+
For more advanced usage, different scripts can be used independently on the command line to perform individual parts of the pipeline. This readme contains quick descriptions and example usages. To obtain detailed instructions on any subcommand, one can use `hicstuff <subcommand> --help`.
|
|
252
|
+
|
|
253
|
+
#### Digestion of fastq reads
|
|
254
|
+
|
|
255
|
+
Generates new gzipped fastq files from original fastq. The function will cut the reads at their religation sites and creates new pairs of reads with the different fragments obtained after cutting at the digestion sites.
|
|
256
|
+
|
|
257
|
+
```
|
|
258
|
+
hicstuff cutsite --help
|
|
259
|
+
|
|
260
|
+
Usage: hicstuff cutsite [OPTIONS]
|
|
261
|
+
|
|
262
|
+
Preprocess FASTQ files by cutting reads at religation sites.
|
|
263
|
+
Generates gzipped FASTQ files with reads cut at ligation junctions, creating new fragment pairs
|
|
264
|
+
for mapping.
|
|
265
|
+
|
|
266
|
+
╭─ Options ────────────────────────────────────────────────────────────────────────────────────────╮
|
|
267
|
+
│ * --forward -F FILE Forward reads FASTQ file. [required] │
|
|
268
|
+
│ * --reverse -R FILE Reverse reads FASTQ file. [required] │
|
|
269
|
+
│ * --prefix -p STR Output prefix (suffixed with _R1.fq.gz / _R2.fq.gz). [required] │
|
|
270
|
+
│ * --enzyme -e STR Comma-separated restriction enzyme(s). [required] │
|
|
271
|
+
│ --mode -m STR Fragment pairing mode. [default: for_vs_rev] │
|
|
272
|
+
│ --seed-size -s INT Minimum read size after cutting. [default: 20] │
|
|
273
|
+
│ --threads -t INT Number of parallel threads. [default: 1] │
|
|
274
|
+
│ --help -h Show this message and exit. │
|
|
275
|
+
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
276
|
+
```
|
|
277
|
+
|
|
278
|
+
#### Iterative alignment
|
|
279
|
+
|
|
280
|
+
Truncate reads from a fastq file to 20 basepairs and iteratively extend and re-align the unmapped reads to optimize the proportion of uniquely aligned reads in a 3C library.
|
|
281
|
+
|
|
282
|
+
```
|
|
283
|
+
hicstuff iteralign --help
|
|
284
|
+
|
|
285
|
+
Usage: hicstuff iteralign [OPTIONS] READS_FQ
|
|
286
|
+
|
|
287
|
+
Iteratively align reads to a reference genome.
|
|
288
|
+
Truncates reads to 20 bp then iteratively extends and re-aligns unmapped reads to maximise the
|
|
289
|
+
proportion of uniquely aligned reads in a 3C library.
|
|
290
|
+
|
|
291
|
+
╭─ Arguments ──────────────────────────────────────────────────────────────────────────────────────╮
|
|
292
|
+
│ * READS_FQ TEXT [required] │
|
|
293
|
+
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
294
|
+
╭─ Options ────────────────────────────────────────────────────────────────────────────────────────╮
|
|
295
|
+
│ * --genome -g FILE Reference genome or aligner index path. [required] │
|
|
296
|
+
│ * --out-bam -o FILE Output BAM file path. [required] │
|
|
297
|
+
│ --threads -t INT Number of parallel threads. [default: 1] │
|
|
298
|
+
│ --tempdir -T DIR Temporary directory (default: current directory). │
|
|
299
|
+
│ --aligner -a STR Aligner: bowtie2, minimap2 or bwa. [default: bowtie2] │
|
|
300
|
+
│ --min-len -l INT Minimum truncated read length. [default: 20] │
|
|
301
|
+
│ --read-len -R INT Maximum read length (estimated from first read if omitted). │
|
|
302
|
+
│ --help -h Show this message and exit. │
|
|
303
|
+
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
304
|
+
```
|
|
305
|
+
|
|
306
|
+
#### Digestion of the genome
|
|
307
|
+
|
|
308
|
+
Digests a fasta file into fragments based on a restriction enzyme or a
|
|
309
|
+
fixed chunk size. Generates two output files into the target directory
|
|
310
|
+
named "info_contigs.txt" and "fragments_list.txt"
|
|
311
|
+
|
|
312
|
+
```
|
|
313
|
+
hicstuff digest --help
|
|
314
|
+
|
|
315
|
+
Usage: hicstuff digest [OPTIONS] FASTA
|
|
316
|
+
|
|
317
|
+
Digest a genome FASTA into restriction fragments.
|
|
318
|
+
Writes ``fragments_list.txt`` and ``info_contigs.txt`` to the output directory.
|
|
319
|
+
|
|
320
|
+
╭─ Arguments ──────────────────────────────────────────────────────────────────────────────────────╮
|
|
321
|
+
│ * FASTA TEXT [required] │
|
|
322
|
+
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
323
|
+
╭─ Options ────────────────────────────────────────────────────────────────────────────────────────╮
|
|
324
|
+
│ * --enzyme -e ENZ[,ENZ2,...] Restriction enzyme name(s) or fixed chunk size in bp. │
|
|
325
|
+
│ [required] │
|
|
326
|
+
│ --outdir -o DIR Output directory (default: current directory). │
|
|
327
|
+
│ --size -s INT Minimum contig size (bp): shorter contigs are left out. │
|
|
328
|
+
│ [default: 0] │
|
|
329
|
+
│ --circular -c Genome is circular. │
|
|
330
|
+
│ --plot -p Show fragment length distribution histogram. │
|
|
331
|
+
│ --figdir -f DIR Directory to save the distribution figure. │
|
|
332
|
+
│ --force -F Overwrite existing output directory. │
|
|
333
|
+
│ --help -h Show this message and exit. │
|
|
334
|
+
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
335
|
+
```
|
|
336
|
+
|
|
337
|
+
For example, to digest the yeast genome with MaeII and HinfI and show histogram of fragment lengths:
|
|
338
|
+
|
|
339
|
+
```sh
|
|
340
|
+
hicstuff digest --plot --outdir output_dir --enzyme MaeII,HinfI Sc_ref.fa
|
|
341
|
+
```
|
|
342
|
+
|
|
343
|
+
#### Filtering of 3C events
|
|
344
|
+
|
|
345
|
+
Filters spurious 3C events such as loops and uncuts from the library based on a minimum distance threshold automatically estimated from the library by default. Can also plot 3C library statistics. This module takes a pairs file with 9 columns as input (readID, chr1, pos1, chr2, pos2, strand1, strand2, frag1, frag2) and filters it.
|
|
346
|
+
|
|
347
|
+
```
|
|
348
|
+
hicstuff filter --help
|
|
349
|
+
|
|
350
|
+
Usage: hicstuff filter [OPTIONS] INPUT_PAIRS OUTPUT_PAIRS
|
|
351
|
+
|
|
352
|
+
Filter spurious Hi-C events (loops and uncuts) from a pairs file.
|
|
353
|
+
|
|
354
|
+
╭─ Arguments ──────────────────────────────────────────────────────────────────────────────────────╮
|
|
355
|
+
│ * INPUT_PAIRS TEXT [required] │
|
|
356
|
+
│ * OUTPUT_PAIRS TEXT [required] │
|
|
357
|
+
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
358
|
+
╭─ Options ────────────────────────────────────────────────────────────────────────────────────────╮
|
|
359
|
+
│ --figdir -f DIR Directory for output figures. │
|
|
360
|
+
│ --interactive -i Ask for thresholds interactively after showing plots. │
|
|
361
|
+
│ --plot -p Show library composition and 3C event abundance plots. │
|
|
362
|
+
│ --prefix -P STR Library name displayed on figures. │
|
|
363
|
+
│ --thresholds -t INT-INT Manual thresholds as UNCUT-LOOP (e.g. 4-5). │
|
|
364
|
+
│ --help -h Show this message and exit. │
|
|
365
|
+
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
366
|
+
```
|
|
367
|
+
|
|
368
|
+
#### Viewing the contact map
|
|
369
|
+
|
|
370
|
+
Visualize a Hi-C matrix file as a heatmap of contact frequencies. Allows to
|
|
371
|
+
tune visualisation by binning and normalizing the matrix, and to save the
|
|
372
|
+
output image to disk. If no output is specified, the output is displayed.
|
|
373
|
+
|
|
374
|
+
```
|
|
375
|
+
hicstuff view --help
|
|
376
|
+
|
|
377
|
+
Usage: hicstuff view [OPTIONS] CONTACT_MAP [CONTACT_MAP2]
|
|
378
|
+
|
|
379
|
+
Visualize a Hi-C matrix as a heatmap.
|
|
380
|
+
|
|
381
|
+
╭─ Arguments ──────────────────────────────────────────────────────────────────────────────────────╮
|
|
382
|
+
│ * CONTACT_MAP TEXT [required] │
|
|
383
|
+
│ CONTACT_MAP2 TEXT │
|
|
384
|
+
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
385
|
+
╭─ Options ────────────────────────────────────────────────────────────────────────────────────────╮
|
|
386
|
+
│ --binning -b INT[bp|kb|Mb] Merge bins by factor or generate fixed-size bins. [default: 1] │
|
|
387
|
+
│ --cmap -c STR Matplotlib colormap name. [default: Reds] │
|
|
388
|
+
│ --circular -C Genome is circular. │
|
|
389
|
+
│ --despeckle -d Remove speckle artefacts. │
|
|
390
|
+
│ --dpi -D INT Output image DPI. [default: 300] │
|
|
391
|
+
│ --frags -f FILE fragments_list.txt (required for bp binning and --lines). │
|
|
392
|
+
│ --transform -T STR Pixel transform: log2, log10, ln, sqrt, exp<val>. │
|
|
393
|
+
│ --lines -l Draw chromosome separator lines (requires --frags). │
|
|
394
|
+
│ --max -M NUM[%] Colorscale maximum (percentile with %). [default: 99%] │
|
|
395
|
+
│ --min -m NUM[%] Colorscale minimum. [default: 0] │
|
|
396
|
+
│ --n-mad -N FLOAT MAD threshold for ICE normalization bin filtering. [default: │
|
|
397
|
+
│ 3.0] │
|
|
398
|
+
│ --normalize -n Perform ICE normalization before rendering. │
|
|
399
|
+
│ --output -o FILE Output image path (display interactively if omitted). │
|
|
400
|
+
│ --region -r STR[;STR] UCSC region to zoom into (e.g. chr1:1000-12000). │
|
|
401
|
+
│ --trim -t FLOAT Trim bins deviating by more than this many MADs. │
|
|
402
|
+
│ --help -h Show this message and exit. │
|
|
403
|
+
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
404
|
+
```
|
|
405
|
+
|
|
406
|
+
For example, to view a 1Mb region of chromosome 1 from a full genome Hi-C matrix rebinned at 10kb:
|
|
407
|
+
|
|
408
|
+
```sh
|
|
409
|
+
hicstuff view --normalize --binning 10kb --region chr1:10,000,000-11,000,000 --frags fragments_list.txt contact_map.tsv
|
|
410
|
+
```
|
|
411
|
+
|
|
412
|
+
### File formats
|
|
413
|
+
|
|
414
|
+
* `pairs` files: This format is used for all intermediate files in the pipeline and is also used by `hicstuff filter`. It is a tab-separated format holding informations about Hi-C pairs. It has an [official specification](https://github.com/4dn-dcic/pairix/blob/master/pairs_format_specification.md) defined by the 4D Nucleome data coordination and integration center.
|
|
415
|
+
* `bedgraph2` bedgraph: This is an optional output format of `hicstuff pipeline` for the sparse matrix. It has two fragment per line, and the number of times they are found together. It has the following fields: **chr1, start1, end1, chr2, start2, end2, occurences**
|
|
416
|
+
- Those files can be [loaded by cooler](https://cooler.readthedocs.io/en/latest/cli.html?highlight=load#cooler-load) using `cooler load -f bg2 <chrom.sizes>:<binsize> in.bg2.gz out.cool` where `chrom.sizes` is a tab delimited file with chromosome names and length on each line, and binsize is the size of bins in the matrix.
|
|
417
|
+
* `GRAAL` sparse matrix: This is a simple tab-separated file with 3 columns: **frag1, frag2, contacts**. The id columns correspond to the absolute id of the restriction fragments (0-indexed). The first row is a header containing the number of rows, number of columns and number of nonzero entries in the matrix. Example:
|
|
418
|
+
|
|
419
|
+
```
|
|
420
|
+
564 564 6978
|
|
421
|
+
0 0 3
|
|
422
|
+
1 2 4
|
|
423
|
+
1 3 3
|
|
424
|
+
```
|
|
425
|
+
|
|
426
|
+
* `fragments_list.txt`: This tab separated file provides information about restriction fragments positions, size and GC content. Note the coordinates are 0 point basepairs, unlike the pairs format, which has 1 point basepairs. Example:
|
|
427
|
+
- id: 1 based restriction fragment index within chromosome.
|
|
428
|
+
- chrom: Chromosome identifier. Order should be the same as in info_contigs.txt or pairs files.
|
|
429
|
+
- start_pos: 0-based start of fragment, in base pairs.
|
|
430
|
+
- end_pos: 0-based end of fragment, in base pairs.
|
|
431
|
+
- size: Size of fragment, in base pairs.
|
|
432
|
+
- gc_content: Proportion of G and C nucleotide in the fragment.
|
|
433
|
+
|
|
434
|
+
```
|
|
435
|
+
id chrom start_pos end_pos size gc_content
|
|
436
|
+
1 seq1 0 21 21 0.5238095238095238
|
|
437
|
+
2 seq1 21 80 59 0.576271186440678
|
|
438
|
+
3 seq1 80 328 248 0.5201612903225806
|
|
439
|
+
```
|
|
440
|
+
|
|
441
|
+
* `info_contigs.txt`: This tab separated file gives information on contigs, such as number of restriction fragments and size. Example:
|
|
442
|
+
- contig: Chromosome identified. Order should be the same in pairs files or fragments_list.txt.
|
|
443
|
+
- length: Chromosome length, in base pairs.
|
|
444
|
+
- n_frags: Number of restriction fragments in chromosome.
|
|
445
|
+
- cumul_length: Cumulative length of previous chromosome, in base pairs.
|
|
446
|
+
|
|
447
|
+
```
|
|
448
|
+
contig length n_frags cumul_length
|
|
449
|
+
seq1 60000 409 0
|
|
450
|
+
seq2 20000 155 409
|
|
451
|
+
```
|
|
452
|
+
|
|
453
|
+
### Contributing
|
|
454
|
+
|
|
455
|
+
All contributions are welcome, in the form of bug reports, suggestions, documentation or pull requests.
|
|
456
|
+
We use the [numpy standard](https://numpydoc.readthedocs.io/en/latest/format.html) for docstrings when documenting functions.
|
|
457
|
+
|
|
458
|
+
The code formatting standard we use is [ruff](https://github.com/astral-sh/ruff) (line-length 100). We use `pytest` as our testing framework. Ideally, new functions should have associated unit tests, placed in the `tests` folder.
|
|
459
|
+
To test the code, you can run:
|
|
460
|
+
|
|
461
|
+
```bash
|
|
462
|
+
uv run pytest tests/
|
|
463
|
+
```
|
|
464
|
+
|
|
465
|
+
To lint and format:
|
|
466
|
+
|
|
467
|
+
```bash
|
|
468
|
+
uv run ruff check src/
|
|
469
|
+
uv run ruff format src/
|
|
470
|
+
```
|
|
471
|
+
|
|
472
|
+
### Releasing
|
|
473
|
+
|
|
474
|
+
Releases are cut with [bump-my-version](https://github.com/callowayproject/bump-my-version) (configured in `pyproject.toml`, needs Python >= 3.10). From a clean, up-to-date `master`:
|
|
475
|
+
|
|
476
|
+
```bash
|
|
477
|
+
uv sync --extra dev # once, to install bump-my-version
|
|
478
|
+
uv run bump-my-version bump minor # or patch / major
|
|
479
|
+
git push --follow-tags
|
|
480
|
+
```
|
|
481
|
+
|
|
482
|
+
The bump updates the version in `pyproject.toml`, `uv.lock`, `doc/conf.py` and the `Dockerfile`, commits as `chore(release): vX.Y.Z` and creates the `vX.Y.Z` tag. Preview the changes first with `uv run bump-my-version bump patch --dry-run -vv`.
|
|
483
|
+
|
|
484
|
+
Pushing the tag triggers the [publish workflow](.github/workflows/publish.yml): it checks that the tag matches the `pyproject.toml` version, publishes the package to PyPI and pushes the Docker image to GHCR (`ghcr.io/koszullab/hicstuff`).
|
|
485
|
+
|
|
486
|
+
### Citation
|
|
487
|
+
|
|
488
|
+
Please cite hicstuff using the official DOI as follows:
|
|
489
|
+
|
|
490
|
+
Cyril Matthey-Doret, Lyam Baudry, Amaury Bignaud, Axel Cournac, Remi-Montagne, Nadège Guiglielmoni, Théo Foutel Rodier and Vittore F. Scolari. 2020. hicstuff: Simple library/pipeline to generate and handle Hi-C data . Zenodo. http://doi.org/10.5281/zenodo.4066363
|
|
491
|
+
|
|
492
|
+
Bibtex entry:
|
|
493
|
+
|
|
494
|
+
```
|
|
495
|
+
@software{cyril_matthey_doret_2020_4066351,
|
|
496
|
+
author = {Cyril Matthey-Doret and
|
|
497
|
+
Lyam Baudry and
|
|
498
|
+
Amaury Bignaud and
|
|
499
|
+
Axel Cournac and
|
|
500
|
+
Remi-Montagne and
|
|
501
|
+
Nadège Guiglielmoni and
|
|
502
|
+
Théo Foutel-Rodier and
|
|
503
|
+
Vittore F. Scolari},
|
|
504
|
+
title = {hicstuff: Simple library/pipeline to generate and handle Hi-C data },
|
|
505
|
+
month = oct,
|
|
506
|
+
year = 2020,
|
|
507
|
+
publisher = {Zenodo},
|
|
508
|
+
version = {v2.3.1},
|
|
509
|
+
doi = {10.5281/zenodo.4066351},
|
|
510
|
+
url = {http://doi.org/10.5281/zenodo.4066363}
|
|
511
|
+
}
|
|
512
|
+
```
|