omamer 2.1.2__tar.gz → 2.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- omamer-2.1.2/README.md → omamer-2.2.0/PKG-INFO +54 -69
- omamer-2.2.0/README.md +150 -0
- {omamer-2.1.2 → omamer-2.2.0}/omamer/__init__.py +1 -1
- {omamer-2.1.2 → omamer-2.2.0}/omamer/_runners.py +31 -22
- {omamer-2.1.2 → omamer-2.2.0}/omamer/database.py +43 -36
- {omamer-2.1.2 → omamer-2.2.0}/omamer/hierarchy.py +2 -0
- {omamer-2.1.2 → omamer-2.2.0}/omamer/index.py +21 -7
- {omamer-2.1.2 → omamer-2.2.0}/omamer/main.py +1 -1
- {omamer-2.1.2 → omamer-2.2.0}/omamer/merge_search.py +8 -6
- {omamer-2.1.2 → omamer-2.2.0}/omamer/orthoxml_parser.py +105 -0
- omamer-2.2.0/omamer/species_tree.py +235 -0
- omamer-2.2.0/omamer.egg-info/PKG-INFO +188 -0
- {omamer-2.1.2 → omamer-2.2.0}/omamer.egg-info/SOURCES.txt +6 -0
- {omamer-2.1.2 → omamer-2.2.0}/omamer.egg-info/requires.txt +3 -4
- omamer-2.2.0/setup.cfg +16 -0
- {omamer-2.1.2 → omamer-2.2.0}/setup.py +10 -6
- omamer-2.2.0/tests/test_mkdb.py +397 -0
- omamer-2.2.0/tests/test_mkdb_core.py +466 -0
- omamer-2.2.0/tests/test_mkdb_sources.py +604 -0
- omamer-2.2.0/tests/test_search.py +268 -0
- omamer-2.2.0/tests/test_species_tree.py +184 -0
- omamer-2.1.2/PKG-INFO +0 -34
- omamer-2.1.2/omamer.egg-info/PKG-INFO +0 -34
- omamer-2.1.2/setup.cfg +0 -15
- {omamer-2.1.2 → omamer-2.2.0}/COPYING +0 -0
- {omamer-2.1.2 → omamer-2.2.0}/COPYING.LESSER +0 -0
- {omamer-2.1.2 → omamer-2.2.0}/LICENSE +0 -0
- {omamer-2.1.2 → omamer-2.2.0}/omamer/_clock.py +0 -0
- {omamer-2.1.2 → omamer-2.2.0}/omamer/_utils.py +0 -0
- {omamer-2.1.2 → omamer-2.2.0}/omamer/alphabets.py +0 -0
- {omamer-2.1.2 → omamer-2.2.0}/omamer/results_reader.py +0 -0
- {omamer-2.1.2 → omamer-2.2.0}/omamer/sequence_buffer.py +0 -0
- {omamer-2.1.2 → omamer-2.2.0}/omamer/sequence_reader.py +0 -0
- {omamer-2.1.2 → omamer-2.2.0}/omamer.egg-info/dependency_links.txt +0 -0
- {omamer-2.1.2 → omamer-2.2.0}/omamer.egg-info/entry_points.txt +0 -0
- {omamer-2.1.2 → omamer-2.2.0}/omamer.egg-info/top_level.txt +0 -0
- {omamer-2.1.2 → omamer-2.2.0}/tests/tests.py +0 -0
|
@@ -1,14 +1,50 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: omamer
|
|
3
|
+
Version: 2.2.0
|
|
4
|
+
Summary: OMAmer - tree-driven and alignment-free protein assignment to sub-families
|
|
5
|
+
Home-page: https://github.com/DessimozLab/omamer
|
|
6
|
+
Author: Victor Rossier and Alex Warwick Vesztrocy
|
|
7
|
+
Author-email: alex@warwickvesztrocy.co.uk
|
|
8
|
+
License: LGPLv3
|
|
9
|
+
Requires-Python: >=3.10
|
|
10
|
+
Description-Content-Type: text/markdown
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
License-File: COPYING
|
|
13
|
+
License-File: COPYING.LESSER
|
|
14
|
+
Requires-Dist: alive_progress
|
|
15
|
+
Requires-Dist: biopython
|
|
16
|
+
Requires-Dist: lxml
|
|
17
|
+
Requires-Dist: numba
|
|
18
|
+
Requires-Dist: numpy>=1.26
|
|
19
|
+
Requires-Dist: pandas<3,>2.0.0
|
|
20
|
+
Requires-Dist: property_manager
|
|
21
|
+
Requires-Dist: Rmath4
|
|
22
|
+
Requires-Dist: scipy
|
|
23
|
+
Requires-Dist: tables>=3.10.1
|
|
24
|
+
Requires-Dist: tqdm
|
|
25
|
+
Provides-Extra: build
|
|
26
|
+
Requires-Dist: pysais; extra == "build"
|
|
27
|
+
Dynamic: author
|
|
28
|
+
Dynamic: author-email
|
|
29
|
+
Dynamic: description
|
|
30
|
+
Dynamic: description-content-type
|
|
31
|
+
Dynamic: home-page
|
|
32
|
+
Dynamic: license
|
|
33
|
+
Dynamic: license-file
|
|
34
|
+
Dynamic: provides-extra
|
|
35
|
+
Dynamic: requires-dist
|
|
36
|
+
Dynamic: requires-python
|
|
37
|
+
Dynamic: summary
|
|
38
|
+
|
|
1
39
|
# OMAmer - tree-driven and alignment-free protein assignment to subfamilies
|
|
2
40
|
|
|
3
41
|
OMAmer is a novel alignment-free protein family assignment method, which limits over-specific subfamily assignments and is suited to phylogenomic databases with thousands of genomes. It is based on an innovative method using evolutionary-informed _k_-mers for alignment-free mapping to ancestral protein subfamilies. Whilst able to reject non-homologous family-level assignments, it has provided better and quicker subfamily-level assignments than a method based on closest sequences (using DIAMOND).
|
|
4
42
|
|
|
5
43
|
# Installation
|
|
6
|
-
Requires Python >= 3.
|
|
44
|
+
Requires Python >= 3.10. Download the package from the PyPI, resolving the dependencies by using ``pip install omamer``.
|
|
7
45
|
|
|
8
46
|
Alternatively, clone this repository and install manually.
|
|
9
47
|
|
|
10
|
-
Note: Python 3.12 is currently not supported, until the `numba` package is updated ([issue](https://github.com/numba/numba/issues/9197)).
|
|
11
|
-
|
|
12
48
|
# Pre-Built Databases
|
|
13
49
|
|
|
14
50
|
Pre-built databases are available for the latest OMA release from the [download section on the OMA Browser website](https://omabrowser.org/oma/current).
|
|
@@ -101,14 +137,21 @@ The taxon from the predicted HOG that is closest from the reference taxon (given
|
|
|
101
137
|
|
|
102
138
|
|
|
103
139
|
# omamer mkdb - Building a Database
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
-
|
|
140
|
+
A database can be built from one of:
|
|
141
|
+
|
|
142
|
+
- the OMA browser's database file and species tree (``--oma_path``), e.g.
|
|
143
|
+
- https://omabrowser.org/All/OmaServer.h5
|
|
144
|
+
- https://omabrowser.org/All/speciestree.nwk
|
|
145
|
+
- HOGs in an OrthoXML file (e.g. from FastOMA) and the proteomes as FASTA files (``--orthoxml``, ``--sequences``). The species tree is read from the ``<taxonomy>`` of the OrthoXML file (OrthoXML >= 0.5), or given as newick file with ``--species_tree``.
|
|
146
|
+
|
|
147
|
+
The taxon names in the species tree below the root taxon must be unique.
|
|
148
|
+
|
|
107
149
|
## Usage
|
|
108
|
-
Required arguments: ``--db``, ``--oma_path``
|
|
150
|
+
Required arguments: ``--db``, and ``--oma_path`` or ``--orthoxml`` and ``--sequences``
|
|
109
151
|
|
|
110
152
|
usage: omamer mkdb [-h] --db DB [--nthreads NTHREADS] [--min_fam_size MIN_FAM_SIZE] [--min_fam_completeness MIN_FAM_COMPLETENESS] [--logic {AND,OR}]
|
|
111
|
-
[--root_taxon ROOT_TAXON] [--hidden_taxa HIDDEN_TAXA] [--
|
|
153
|
+
[--root_taxon ROOT_TAXON] [--hidden_taxa HIDDEN_TAXA] [--reduced_alphabet] [--k K] [--oma_path OMA_PATH]
|
|
154
|
+
[--orthoxml ORTHOXML] [--species_tree SPECIES_TREE] [--sequences [SEQUENCES ...]]
|
|
112
155
|
[--log_level {debug,info,warning}]
|
|
113
156
|
|
|
114
157
|
## Arguments
|
|
@@ -125,70 +168,12 @@ Required arguments: ``--db``, ``--oma_path``
|
|
|
125
168
|
| [``--reduced_alphabet``](#markdown-header--reduced_alphabet)||Use reduced alphabet from Linclust paper
|
|
126
169
|
| [``--k``](#markdown-header--k)|6|_k_-mer length
|
|
127
170
|
| [``--oma_path``](#markdown-header--oma_path)||Path to a directory with both OmaServer.h5 and speciestree.nwk
|
|
171
|
+
| [``--orthoxml``](#markdown-header--orthoxml)||Path to an OrthoXML file with the HOGs
|
|
172
|
+
| [``--species_tree``](#markdown-header--species_tree)|``<taxonomy>`` of the OrthoXML file|Path to a newick species tree for ``--orthoxml``. Required if the OrthoXML file has no taxonomy (OrthoXML < 0.5).
|
|
173
|
+
| [``--sequences``](#markdown-header--sequences)||FASTA files with the proteins of the species in the OrthoXML file
|
|
128
174
|
| [``--log_level``](#markdown-header--log_level)|info|Logging level
|
|
129
175
|
|
|
130
176
|
|
|
131
|
-
# Change log
|
|
132
|
-
|
|
133
|
-
#### Version 2.1.1
|
|
134
|
-
|
|
135
|
-
- Performance improvements to the mkdb command with orthoxml input
|
|
136
|
-
- Added a check for non-unique protein IDs in the input fasta files. Now it gives a more informative error message
|
|
137
|
-
- fixed #49
|
|
138
|
-
|
|
139
|
-
#### Version 2.1.0
|
|
140
|
-
- significant improvements to classification speed
|
|
141
|
-
|
|
142
|
-
#### Version 2.0.4
|
|
143
|
-
- fixes issue #34 (numpy2 incompatibility)
|
|
144
|
-
- experimental support to build omamer databases from orthoxml/fasta files
|
|
145
|
-
- update github action to latest versions
|
|
146
|
-
|
|
147
|
-
#### Version 2.0.3
|
|
148
|
-
- fixes issue #30
|
|
149
|
-
- update github action to latest versions
|
|
150
|
-
|
|
151
|
-
#### Version 2.0.2
|
|
152
|
-
- changed method for hiding taxa in build process. Now takes a file containing taxa to hide on separate lines.
|
|
153
|
-
- checks and improved feedback for root taxon and requested taxa to hide.
|
|
154
|
-
- root taxon set by default to the root level in speciestree.nwk (previously hard-coded to default to LUCA)
|
|
155
|
-
|
|
156
|
-
#### Version 2.0.1
|
|
157
|
-
- remove dependency for filehash library
|
|
158
|
-
- return better error message if build dependencies are not met, but trying to building an omamer database
|
|
159
|
-
- minor fixes
|
|
160
|
-
|
|
161
|
-
#### Version 2.0.0
|
|
162
|
-
- Major update of database format and search code to improve overall memory useage. Most standard runs with LUCA-level database will run on a machine with 16GB RAM.
|
|
163
|
-
- Update to the scoring algorithm for root-level HOG / family assignments, to allow for significance testing. This estimates a binomial distribution for each family, so that we can compute the probability of matching at least as many k-mers as we have observed by chance, for each family that has a match to a given query.
|
|
164
|
-
- UX improvements - more feedback during interactive search runs, whilst maintaining small log files.
|
|
165
|
-
|
|
166
|
-
#### Version 0.2.5
|
|
167
|
-
- Fixes an issue when storing the pre-conputed statistics
|
|
168
|
-
|
|
169
|
-
#### Version 0.2.4
|
|
170
|
-
- Improved loading time for standard search by pre-computing statistics
|
|
171
|
-
- Adding new command line option "info" to show the metadata of the
|
|
172
|
-
dataset used to build the omamer database.
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
#### Version 0.2.2
|
|
176
|
-
- Automated deployment to PyPI
|
|
177
|
-
- Removed PyHAM dependency
|
|
178
|
-
|
|
179
|
-
#### Version 0.2.0
|
|
180
|
-
- Added ``--min_fam_completeness``, ``--logic``, ``--score`` and ``--reference_taxon`` options
|
|
181
|
-
- New output format
|
|
182
|
-
- Debugging
|
|
183
|
-
|
|
184
|
-
#### Version 0.1.2 - 0.1.3
|
|
185
|
-
- Debugging
|
|
186
|
-
|
|
187
|
-
#### Version 0.1.0
|
|
188
|
-
- Added hidden_taxa and threshold arguments
|
|
189
|
-
|
|
190
|
-
#### Version 0.0.1
|
|
191
|
-
- Initial release
|
|
192
177
|
|
|
193
178
|
# License
|
|
194
179
|
OMAmer is a free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
|
omamer-2.2.0/README.md
ADDED
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
# OMAmer - tree-driven and alignment-free protein assignment to subfamilies
|
|
2
|
+
|
|
3
|
+
OMAmer is a novel alignment-free protein family assignment method, which limits over-specific subfamily assignments and is suited to phylogenomic databases with thousands of genomes. It is based on an innovative method using evolutionary-informed _k_-mers for alignment-free mapping to ancestral protein subfamilies. Whilst able to reject non-homologous family-level assignments, it has provided better and quicker subfamily-level assignments than a method based on closest sequences (using DIAMOND).
|
|
4
|
+
|
|
5
|
+
# Installation
|
|
6
|
+
Requires Python >= 3.10. Download the package from the PyPI, resolving the dependencies by using ``pip install omamer``.
|
|
7
|
+
|
|
8
|
+
Alternatively, clone this repository and install manually.
|
|
9
|
+
|
|
10
|
+
# Pre-Built Databases
|
|
11
|
+
|
|
12
|
+
Pre-built databases are available for the latest OMA release from the [download section on the OMA Browser website](https://omabrowser.org/oma/current).
|
|
13
|
+
|
|
14
|
+
- LUCA: https://omabrowser.org/All/LUCA.h5
|
|
15
|
+
- _Metazoa_: https://omabrowser.org/All/Metazoa.h5
|
|
16
|
+
- _Viridiplantae_: https://omabrowser.org/All/Viridiplantae.h5
|
|
17
|
+
- _Saccharomyceta_: https://omabrowser.org/All/Saccharomyceta.h5
|
|
18
|
+
- _Primates_: https://omabrowser.org/All/Primates.h5
|
|
19
|
+
|
|
20
|
+
Their names indicate the root-taxon parameter used. Other non-required parameters were left to default.
|
|
21
|
+
|
|
22
|
+
Note: databases included in the [Zenodo upload](https://zenodo.org/record/4593702) from the manuscript are not supported by the most recent version of OMAmer. We recommend using the most recent release with databases built on the most recent OMA browser release.
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
# omamer search - Searching a Database
|
|
27
|
+
Assign proteins to families and subfamilies in a pre-existing database.
|
|
28
|
+
## Usage
|
|
29
|
+
Required arguments: ``--db``, ``--query``
|
|
30
|
+
|
|
31
|
+
usage: omamer search [-h] -d DB -q QUERY [--threshold THRESHOLD] [--family_alpha FAMILY_ALPHA] [-fo] [-n TOP_N_FAMS] [--reference_taxon REFERENCE_TAXON]
|
|
32
|
+
[-o OUT] [--include_extant_genes] [-c CHUNKSIZE] [-t {0,1,2,3,4,5,6,7,8}] [--log_level {debug,info,warning}] [--silent]
|
|
33
|
+
|
|
34
|
+
## Arguments
|
|
35
|
+
### Quick reference table
|
|
36
|
+
|
|
37
|
+
| Short Flag | Flag | Default | Description |
|
|
38
|
+
|:-----------|:---------------------|:-----------------------|:------------|
|
|
39
|
+
| [``-d``](#markdown-header-d) | [``--db``](#markdown-header--db) || Path to existing database (including filename)
|
|
40
|
+
| [``-q``](#markdown-header-q) | [``--query``](#markdown-header--query) || Path to FASTA formatted sequences
|
|
41
|
+
| | [``--threshold``](#markdown-header--threshold) | 0.1 | Threshold applied on the OMAmer-score that is used to vary the specificity of predicted HOGs. The lower the theshold the more (over-)specific predicted HOGs will be.
|
|
42
|
+
| | [``--family_alpha``](#markdown-header--family_alpha) | 1e-6 | Significance threshold used when filtering families.
|
|
43
|
+
| [``-fo``](#markdown-header-fo) | [``--family_only``](#markdown-header--family_only) | False | If set, only place at the family level. Useful for certain analysis. Note: `subfamily_medianseqlen` in the results is for the family level.
|
|
44
|
+
| [``-n``](#markdown-header-n) | [``--top_n_fams``](#markdown-header--top_n_fams) | 1 | Number of top level families to place into. By default, placed into only the best scoring family.
|
|
45
|
+
| [``-o``](#markdown-header-o) | [``--out``](#markdown-header--db) | stdout | Path to output. If not set, defaults to stdout.
|
|
46
|
+
| | [``--include_extant_genes``](#markdown-header--include_extant_genes)||Include extant gene IDs as comma separated entry in results
|
|
47
|
+
| [``-c``](#markdown-header-c) | [``--chunksize``](#markdown-header--chunksize) |10000| Number of queries to process at once.
|
|
48
|
+
| [``-t``](#markdown-header-t) | [``--nthreads``](#markdown-header--db) |1|Number of threads to use
|
|
49
|
+
| | [``--log_level``](#markdown-header--db) |info| Logging level (options debug, info, warning)
|
|
50
|
+
| | [``--silent``](#markdown-header--silent) || Set to silence the output.
|
|
51
|
+
<!--| | [``--reference_taxon``](#markdown-header--reference_taxon) || The placement is stopped when reaching a HOG with the reference taxon (must exist in the OMA database). This is a complementary option to vary the specificity of predicted HOGs.-->
|
|
52
|
+
# Output
|
|
53
|
+
|
|
54
|
+
Output is in the form of a tab-seperated value file (TSV), with metadata added to the header using ``!<tag>: <value>``. A parser can be imported for further analysis in python as ``from omamer.results_reader import results_reader``.
|
|
55
|
+
|
|
56
|
+
## Output Columns
|
|
57
|
+
|
|
58
|
+
#### Query sequence identifier (`qseqid`)
|
|
59
|
+
The sequence identifier from the input FASTA-formatted sequences.
|
|
60
|
+
|
|
61
|
+
#### Predicted HOG identifier (`hogid`)
|
|
62
|
+
The identifier of the hierarchical orthologous group (HOG) in OMA, which you can access through the OMA browser search bar or its REST API (https://omabrowser.org/api/docs).
|
|
63
|
+
|
|
64
|
+
A HOG identifier is composed of the root-HOG identifier (following “HOG:” and before the first dot), which is followed by its sub-HOGs (before each subsequent dot). For example, for subfamily HOG:0487954.3l.27l, HOG:0487954 is the root-HOG (HOG without-parent), HOG:0487954.3l is its child and HOG:0487954.3l.27l its grandchild.
|
|
65
|
+
|
|
66
|
+
#### Predicted HOG taxonomic level (`hoglevel`)
|
|
67
|
+
The taxonomic level that the predicted HOG is defined at.
|
|
68
|
+
|
|
69
|
+
#### Family p-value (`family_p`)
|
|
70
|
+
p-value of having as many or more of k-mers in common under a binomial distribution. Reported in negative natural log units.
|
|
71
|
+
|
|
72
|
+
#### Family count (`family_count`)
|
|
73
|
+
Count of _k_-mers in common with the family / root level HOG.
|
|
74
|
+
|
|
75
|
+
#### Family normalised count (`family_normcount`)
|
|
76
|
+
Family count, normalised by the expected number of hits for the query's sequence length, with the family's _k_-mer content.
|
|
77
|
+
|
|
78
|
+
#### Sub-Family score (`subfamily_score`)
|
|
79
|
+
The OMAmer-score of the predicted HOG. At the subfamily level, this score captures the excess of similarity that is shared between the query and a given HOG, thus excluding the similarity with regions conserved in more ancestral HOGs.
|
|
80
|
+
|
|
81
|
+
#### Sub-Family count (`subfamily_count`)
|
|
82
|
+
Count of _k_-mers in common with the sub-family / HOG.
|
|
83
|
+
|
|
84
|
+
#### Query sequence length (`qseqlen`)
|
|
85
|
+
Count of _k_-mers in common with the sub-family / HOG.
|
|
86
|
+
|
|
87
|
+
#### Sub-Family median sequence length (`subfamily_medianseqlen`)
|
|
88
|
+
Median length of the sequences that are present in the predicted HOG. In the case of family-only placement, this is instead reported at the root-HOG level.
|
|
89
|
+
|
|
90
|
+
#### Query sequence overlap (`qseq_overlap`)
|
|
91
|
+
The proportion of the query sequence overlapping with _k_-mers of reference root-HOGs. This may be helpful to reject partially homologous matches that are problematic in some applications.
|
|
92
|
+
|
|
93
|
+
#### Sub-Family gene set (`subfamily_geneset`)
|
|
94
|
+
Optionally printed (see ``--include_extant_genes``). Comma-seperated list of extant gene IDs of predicted HOG. The [OMA browser](https://omabrowser.org) can be used to find out more information. In particular, using the [REST API](https://omabrowser.org/api/docs), or via the [Python API Client](https://github.com/DessimozLab/pyomadb).
|
|
95
|
+
|
|
96
|
+
<!-- #### Closest taxon from reference taxon (`closest_taxa`)
|
|
97
|
+
The taxon from the predicted HOG that is closest from the reference taxon (given one was provided). This option provides a mean to evaluate the performance of OMAmer placement given some knowledge of the query taxonomy is available.
|
|
98
|
+
-->
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
# omamer mkdb - Building a Database
|
|
102
|
+
A database can be built from one of:
|
|
103
|
+
|
|
104
|
+
- the OMA browser's database file and species tree (``--oma_path``), e.g.
|
|
105
|
+
- https://omabrowser.org/All/OmaServer.h5
|
|
106
|
+
- https://omabrowser.org/All/speciestree.nwk
|
|
107
|
+
- HOGs in an OrthoXML file (e.g. from FastOMA) and the proteomes as FASTA files (``--orthoxml``, ``--sequences``). The species tree is read from the ``<taxonomy>`` of the OrthoXML file (OrthoXML >= 0.5), or given as newick file with ``--species_tree``.
|
|
108
|
+
|
|
109
|
+
The taxon names in the species tree below the root taxon must be unique.
|
|
110
|
+
|
|
111
|
+
## Usage
|
|
112
|
+
Required arguments: ``--db``, and ``--oma_path`` or ``--orthoxml`` and ``--sequences``
|
|
113
|
+
|
|
114
|
+
usage: omamer mkdb [-h] --db DB [--nthreads NTHREADS] [--min_fam_size MIN_FAM_SIZE] [--min_fam_completeness MIN_FAM_COMPLETENESS] [--logic {AND,OR}]
|
|
115
|
+
[--root_taxon ROOT_TAXON] [--hidden_taxa HIDDEN_TAXA] [--reduced_alphabet] [--k K] [--oma_path OMA_PATH]
|
|
116
|
+
[--orthoxml ORTHOXML] [--species_tree SPECIES_TREE] [--sequences [SEQUENCES ...]]
|
|
117
|
+
[--log_level {debug,info,warning}]
|
|
118
|
+
|
|
119
|
+
## Arguments
|
|
120
|
+
<!-- todo: add short flags -->
|
|
121
|
+
| Flag | Default | Description |
|
|
122
|
+
|:--------------------|:----------------------|:-----------|
|
|
123
|
+
| [``--db``](#markdown-header--db)||Path to new database (including filename)
|
|
124
|
+
| [``--nthreads``](#markdown-header--nthreads)|1|Number of threads to use
|
|
125
|
+
| [``--min_fam_size``](#markdown-header--min_fam_size)|6|Only root-HOGs with a protein count passing this threshold are used.
|
|
126
|
+
| [``--min_fam_completeness``](#markdown-header--min_fam_completeness)|0.5|Only root-HOGs passing this threshold are used. The completeness of a HOG is defined as the number of observed species divided by the expected number of species at the HOG taxonomic level.
|
|
127
|
+
| [``--logic``](#markdown-header--logic)|OR|Logic used between the two above arguments to filter root-HOGs. Options are "AND" or "OR".
|
|
128
|
+
| [``--root_taxon``](#markdown-header--root_taxon)|root of speciestree.nwk|HOGs defined at, or descending from, this taxon are uses as root-HOGs.
|
|
129
|
+
| [``--hidden_taxa``](#markdown-header--hidden_taxa)||The proteins from these taxa are removed before the database computation. Usage: a file containing taxa on seperate lines (scientific name). These must match EXACTLY with the node name in the tree given.
|
|
130
|
+
| [``--reduced_alphabet``](#markdown-header--reduced_alphabet)||Use reduced alphabet from Linclust paper
|
|
131
|
+
| [``--k``](#markdown-header--k)|6|_k_-mer length
|
|
132
|
+
| [``--oma_path``](#markdown-header--oma_path)||Path to a directory with both OmaServer.h5 and speciestree.nwk
|
|
133
|
+
| [``--orthoxml``](#markdown-header--orthoxml)||Path to an OrthoXML file with the HOGs
|
|
134
|
+
| [``--species_tree``](#markdown-header--species_tree)|``<taxonomy>`` of the OrthoXML file|Path to a newick species tree for ``--orthoxml``. Required if the OrthoXML file has no taxonomy (OrthoXML < 0.5).
|
|
135
|
+
| [``--sequences``](#markdown-header--sequences)||FASTA files with the proteins of the species in the OrthoXML file
|
|
136
|
+
| [``--log_level``](#markdown-header--log_level)|info|Logging level
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
# License
|
|
141
|
+
OMAmer is a free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
|
|
142
|
+
|
|
143
|
+
OMAmer is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details.
|
|
144
|
+
|
|
145
|
+
You should have received a copy of the GNU Lesser General Public License along with OMAmer. If not, see <http://www.gnu.org/licenses/>.
|
|
146
|
+
|
|
147
|
+
# Citation
|
|
148
|
+
Victor Rossier, Alex Warwick Vesztrocy, Marc Robinson-Rechavi, Christophe Dessimoz, OMAmer: tree-driven and alignment-free protein assignment to subfamilies outperforms closest sequence approaches, Bioinformatics, 2021;, btab219, https://doi.org/10.1093/bioinformatics/btab219
|
|
149
|
+
|
|
150
|
+
Code used for that paper is available here: [](https://doi.org/10.5281/zenodo.4593702)
|
|
@@ -24,7 +24,7 @@
|
|
|
24
24
|
from datetime import date
|
|
25
25
|
|
|
26
26
|
__packagename__ = "omamer"
|
|
27
|
-
__version__ = "2.
|
|
27
|
+
__version__ = "2.2.0"
|
|
28
28
|
__copyright__ = "(C) 2019-{:d} Victor Rossier <victor.rossier@unil.ch> and Alex Warwick Vesztrocy <alex@warwickvesztrocy.co.uk> and Nikolai Romashchenko <nikolai.romashchenko@unil.ch>".format(
|
|
29
29
|
date.today().year
|
|
30
30
|
)
|
|
@@ -28,8 +28,8 @@ from ._utils import LOG, check_file_exists
|
|
|
28
28
|
def mkdb_oma(args):
|
|
29
29
|
from .database import DatabaseFromOMABrowser, DatabaseFromOrthoXML
|
|
30
30
|
from .index import Index
|
|
31
|
+
from .species_tree import SpeciesTree, SpeciesTreeError
|
|
31
32
|
|
|
32
|
-
from ete3 import Tree
|
|
33
33
|
import os
|
|
34
34
|
|
|
35
35
|
_ensure_db_build_dependencies_available()
|
|
@@ -59,8 +59,8 @@ def mkdb_oma(args):
|
|
|
59
59
|
check_file_exists(nwk)
|
|
60
60
|
|
|
61
61
|
else:
|
|
62
|
-
if args.orthoxml is None or
|
|
63
|
-
raise ValueError('Must pass either browser database (--oma_path) or all files for OrthoXML build (--orthoxml --species_tree
|
|
62
|
+
if args.orthoxml is None or not args.sequences:
|
|
63
|
+
raise ValueError('Must pass either browser database (--oma_path) or all files for OrthoXML build (--orthoxml --sequences, and --species_tree unless the OrthoXML file contains the taxonomy)')
|
|
64
64
|
|
|
65
65
|
# orthoxml mode
|
|
66
66
|
Database = DatabaseFromOrthoXML
|
|
@@ -68,29 +68,36 @@ def mkdb_oma(args):
|
|
|
68
68
|
# find files
|
|
69
69
|
oxml_fn = args.orthoxml.name
|
|
70
70
|
fasta_fns = list(map(lambda x: x.name, args.sequences))
|
|
71
|
-
nwk = args.species_tree.name
|
|
72
|
-
|
|
73
|
-
#
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
raise
|
|
81
|
-
|
|
82
|
-
|
|
71
|
+
nwk = args.species_tree.name if args.species_tree is not None else None
|
|
72
|
+
|
|
73
|
+
# read the species tree, from the OrthoXML file if none is given
|
|
74
|
+
if nwk is not None:
|
|
75
|
+
full_stree = SpeciesTree.read(nwk)
|
|
76
|
+
else:
|
|
77
|
+
try:
|
|
78
|
+
full_stree = SpeciesTree.from_orthoxml(oxml_fn)
|
|
79
|
+
except SpeciesTreeError:
|
|
80
|
+
raise SpeciesTreeError(
|
|
81
|
+
"{} contains no taxonomy (OrthoXML >= 0.5), pass the species "
|
|
82
|
+
"tree with --species_tree".format(oxml_fn)
|
|
83
|
+
)
|
|
84
|
+
LOG.info("Using the taxonomy of the OrthoXML file as species tree")
|
|
85
|
+
|
|
86
|
+
# check the root taxon, and that the taxa below it have unique names
|
|
87
|
+
stree = full_stree
|
|
88
|
+
root_taxon = args.root_taxon if args.root_taxon is not None else stree.root
|
|
89
|
+
if root_taxon is None:
|
|
90
|
+
raise SpeciesTreeError(
|
|
91
|
+
"The root of the species tree has no name, name it or set --root_taxon"
|
|
92
|
+
)
|
|
93
|
+
stree = stree.subtree(root_taxon)
|
|
83
94
|
|
|
84
95
|
if args.hidden_taxa:
|
|
85
96
|
# load and check for hidden taxa
|
|
86
97
|
hidden_taxa = list(map(lambda x: x.rstrip(), args.hidden_taxa))
|
|
87
98
|
|
|
88
99
|
for ht in hidden_taxa:
|
|
89
|
-
|
|
90
|
-
if len(r) == 0:
|
|
91
|
-
raise ValueError("Unable to find {} as hidden taxon".format(ht))
|
|
92
|
-
elif len(r) > 1:
|
|
93
|
-
raise ValueError("Ambiguous hidden taxon {} ({})".format(ht, ", ".join(map(lambda x: x.name, r))))
|
|
100
|
+
stree.check_taxon(ht, "hidden taxon")
|
|
94
101
|
else:
|
|
95
102
|
hidden_taxa = []
|
|
96
103
|
|
|
@@ -107,9 +114,9 @@ def mkdb_oma(args):
|
|
|
107
114
|
# add sequences from database
|
|
108
115
|
LOG.info("Loading sequences")
|
|
109
116
|
if browser_db_mode:
|
|
110
|
-
seq_buff = db.build_database(oma_db_fn,
|
|
117
|
+
seq_buff = db.build_database(oma_db_fn, full_stree)
|
|
111
118
|
else:
|
|
112
|
-
seq_buff = db.build_database(oxml_fn, fasta_fns,
|
|
119
|
+
seq_buff = db.build_database(oxml_fn, fasta_fns, full_stree)
|
|
113
120
|
|
|
114
121
|
LOG.info("Building index")
|
|
115
122
|
db.ki = Index(
|
|
@@ -254,6 +261,8 @@ def search(args):
|
|
|
254
261
|
search_times.append((len(ids), t_search1 - t_search0))
|
|
255
262
|
|
|
256
263
|
db.close()
|
|
264
|
+
if args.out is not sys.stdout:
|
|
265
|
+
args.out.close()
|
|
257
266
|
|
|
258
267
|
search_rate = sum(map(lambda x: x[0], search_times)) / sum(
|
|
259
268
|
map(lambda x: x[1], search_times)
|
|
@@ -191,11 +191,10 @@ class Database(object):
|
|
|
191
191
|
self._check_open_writeable()
|
|
192
192
|
|
|
193
193
|
def _parse_stree(stree_path, roottax):
|
|
194
|
-
from
|
|
194
|
+
from .species_tree import load_species_tree
|
|
195
195
|
|
|
196
196
|
LOG.debug("parsing species tree {}".format(stree_path))
|
|
197
|
-
stree =
|
|
198
|
-
pruned_stree = [x for x in stree.traverse() if x.name == roottax][0]
|
|
197
|
+
stree = load_species_tree(stree_path).subtree(roottax)
|
|
199
198
|
|
|
200
199
|
tax2parent = {}
|
|
201
200
|
tax2children = {}
|
|
@@ -203,19 +202,15 @@ class Database(object):
|
|
|
203
202
|
species = set()
|
|
204
203
|
nspecies_below = {}
|
|
205
204
|
|
|
206
|
-
for
|
|
207
|
-
tax =
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
)
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
species.add(sp_name)
|
|
216
|
-
nspecies_below[sp_name] = 1
|
|
217
|
-
else:
|
|
218
|
-
nspecies_below[tl.name.encode("ascii")] = sum(1 for _ in tl.iter_leaves())
|
|
205
|
+
for name, parent, children, level, is_species, nspecies in stree.taxa():
|
|
206
|
+
tax = name.encode("ascii")
|
|
207
|
+
# the parent of the root taxon is not used, it gets ParentOff -1
|
|
208
|
+
tax2parent[tax] = (parent if parent is not None else "root").encode("ascii")
|
|
209
|
+
tax2children[tax] = [x.encode("ascii") for x in children]
|
|
210
|
+
tax2level[tax] = level
|
|
211
|
+
if is_species:
|
|
212
|
+
species.add(tax)
|
|
213
|
+
nspecies_below[tax] = nspecies
|
|
219
214
|
|
|
220
215
|
return (tax2parent, tax2children, tax2level, species, nspecies_below)
|
|
221
216
|
|
|
@@ -398,7 +393,7 @@ class Database(object):
|
|
|
398
393
|
for fam_id, hogs in sorted(fam2hogs.items(), key=lambda x: int(x[0])):
|
|
399
394
|
### hogs
|
|
400
395
|
# sort by level
|
|
401
|
-
hogs = sorted(hogs, key=lambda x: len(x.split(b".")))
|
|
396
|
+
hogs = sorted(hogs, key=lambda x: (len(x.split(b".")), x))
|
|
402
397
|
hog_num = len(hogs)
|
|
403
398
|
|
|
404
399
|
# parents
|
|
@@ -464,8 +459,8 @@ class Database(object):
|
|
|
464
459
|
child_hogs_numbers[i],
|
|
465
460
|
child_prots_offsets[i],
|
|
466
461
|
child_prots_numbers[i],
|
|
467
|
-
|
|
468
|
-
|
|
462
|
+
0,
|
|
463
|
+
0,
|
|
469
464
|
0,
|
|
470
465
|
oma_hog_size,
|
|
471
466
|
oma_hog_completeness,
|
|
@@ -609,18 +604,20 @@ class Database(object):
|
|
|
609
604
|
|
|
610
605
|
@staticmethod
|
|
611
606
|
def find_parent_hog(hog_id, hogs):
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
607
|
+
"""
|
|
608
|
+
Offset in the sorted array hogs of the closest ancestor of hog_id, or -1.
|
|
609
|
+
|
|
610
|
+
Intermediate HOGs can be missing, e.g. for a duplication nested directly
|
|
611
|
+
in another duplication HOG:1.1a.1a exists but HOG:1.1a does not.
|
|
612
|
+
"""
|
|
613
|
+
sep = "." if isinstance(hog_id, str) else b"."
|
|
614
|
+
split_hog = hog_id.split(sep)
|
|
615
|
+
for i in range(len(split_hog) - 1, 0, -1):
|
|
616
|
+
parent_id = sep.join(split_hog[:i])
|
|
617
|
+
parent_hog = np.searchsorted(hogs, parent_id)
|
|
618
|
+
if parent_hog < len(hogs) and hogs[parent_hog] == parent_id:
|
|
619
|
+
return parent_hog
|
|
620
|
+
return -1
|
|
624
621
|
|
|
625
622
|
def add_median_seqlen_col(self):
|
|
626
623
|
"""
|
|
@@ -1069,8 +1066,15 @@ class DatabaseFromOrthoXML(DatabaseFromOMA):
|
|
|
1069
1066
|
self.oxml_path = oxml_path
|
|
1070
1067
|
self.hog_parser = OrthoxmlParser(oxml_path)
|
|
1071
1068
|
|
|
1072
|
-
def parse_oxml(self, nspecies_below):
|
|
1073
|
-
from .orthoxml_parser import is_orthologGroup_node
|
|
1069
|
+
def parse_oxml(self, nspecies_below, stree_path):
|
|
1070
|
+
from .orthoxml_parser import TaxonomyLevels, is_orthologGroup_node
|
|
1071
|
+
from .species_tree import load_species_tree
|
|
1072
|
+
|
|
1073
|
+
# the whole tree, HOGs above the root taxon are only filtered later
|
|
1074
|
+
taxonomy = TaxonomyLevels(load_species_tree(stree_path).parents())
|
|
1075
|
+
|
|
1076
|
+
def gene_species(gene_id):
|
|
1077
|
+
return self.hog_parser.species.get_species(gene_id).name
|
|
1074
1078
|
|
|
1075
1079
|
def generate_hog_tab(hog):
|
|
1076
1080
|
# this is more memory intensive (we cache the results)
|
|
@@ -1143,6 +1147,8 @@ class DatabaseFromOrthoXML(DatabaseFromOMA):
|
|
|
1143
1147
|
entries = []
|
|
1144
1148
|
LOG.debug("loading hog structure and membership from orthoxml")
|
|
1145
1149
|
for hog in tqdm(self.hog_parser.iter_hogs(auto_clean=True), desc="Parsing HOGs"):
|
|
1150
|
+
# same HOG levels as in the OMA browser
|
|
1151
|
+
hog.insert_missing_levels(taxonomy, gene_species)
|
|
1146
1152
|
hog_data = list(filter(lambda x: x is not None, generate_hog_tab(hog)))
|
|
1147
1153
|
gene_data = list(generate_prot_tab(hog))
|
|
1148
1154
|
hog_data += list(generate_hogs_for_prot(hog, gene_data))
|
|
@@ -1182,7 +1188,7 @@ class DatabaseFromOrthoXML(DatabaseFromOMA):
|
|
|
1182
1188
|
self.add_taxid_col()
|
|
1183
1189
|
|
|
1184
1190
|
LOG.debug("build hog_table from orthoxml")
|
|
1185
|
-
(hog_tab, ent_tab) = self.parse_oxml(nspecies_below)
|
|
1191
|
+
(hog_tab, ent_tab) = self.parse_oxml(nspecies_below, stree_path)
|
|
1186
1192
|
|
|
1187
1193
|
LOG.debug("select and strip OMA HOGs")
|
|
1188
1194
|
(
|
|
@@ -1381,8 +1387,9 @@ class DatabaseFromOrthoXML(DatabaseFromOMA):
|
|
|
1381
1387
|
taxid_column = []
|
|
1382
1388
|
i = -100
|
|
1383
1389
|
for tax_name in self._db_Taxonomy.col("ID"):
|
|
1384
|
-
sp = self.hog_parser.species[tax_name]
|
|
1385
|
-
|
|
1390
|
+
sp = self.hog_parser.species[tax_name.decode("ascii")]
|
|
1391
|
+
# NCBITaxId="0" means unknown, e.g. in FastOMA output
|
|
1392
|
+
if sp is not None and sp.NCBITaxID > 0:
|
|
1386
1393
|
taxid_column.append(sp.NCBITaxID)
|
|
1387
1394
|
else:
|
|
1388
1395
|
taxid_column.append(i)
|
|
@@ -35,6 +35,8 @@ def get_root_leaf_offsets(off, parent_arr):
|
|
|
35
35
|
parent = parent_arr[off]
|
|
36
36
|
while parent != -1:
|
|
37
37
|
leaf_root.append(parent)
|
|
38
|
+
if len(leaf_root) > len(parent_arr):
|
|
39
|
+
raise ValueError("cycle in parent pointers")
|
|
38
40
|
parent = parent_arr[parent]
|
|
39
41
|
return np.array(leaf_root[::-1], dtype=np.uint32)
|
|
40
42
|
|
|
@@ -93,10 +93,9 @@ class Index(object):
|
|
|
93
93
|
assert "/Index" not in self.db.db, "Index has already been computed"
|
|
94
94
|
|
|
95
95
|
# build suffix array with option to translate the sequence buffer first
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
)
|
|
99
|
-
self._build_kmer_table(seq_buff, sa)
|
|
96
|
+
translated = self.alphabet.translate(seq_buff)
|
|
97
|
+
sa = self._build_suffixarray(translated, len(self.db._db_Protein))
|
|
98
|
+
self._build_kmer_table(seq_buff, sa, translated)
|
|
100
99
|
|
|
101
100
|
@staticmethod
|
|
102
101
|
def _build_suffixarray(seqs, n):
|
|
@@ -110,7 +109,17 @@ class Index(object):
|
|
|
110
109
|
sa[:n].sort() # Sort delimiters by position
|
|
111
110
|
return sa
|
|
112
111
|
|
|
113
|
-
def _build_kmer_table(self, seq_buff, sa):
|
|
112
|
+
def _build_kmer_table(self, seq_buff, sa, translated=None):
|
|
113
|
+
"""
|
|
114
|
+
Build the k-mer table from the suffix array of the translated sequence
|
|
115
|
+
buffer. The k-mer codes are computed from the original sequences, the
|
|
116
|
+
suffixes of a k-mer are found by comparing the translated ones, as with
|
|
117
|
+
a reduced alphabet different k-mers of the original sequences belong
|
|
118
|
+
to the same k-mer.
|
|
119
|
+
"""
|
|
120
|
+
if translated is None:
|
|
121
|
+
translated = self.alphabet.translate(seq_buff)
|
|
122
|
+
|
|
114
123
|
@numba.njit(parallel=True, nogil=True)
|
|
115
124
|
def _compute_mask_and_filter(
|
|
116
125
|
sa, sa_mask, sa_filter, k, n, prot2spoff, prot2hogoff, sp_filter
|
|
@@ -168,6 +177,7 @@ class Index(object):
|
|
|
168
177
|
def _compute_kmer_table(
|
|
169
178
|
sa,
|
|
170
179
|
seq_buff,
|
|
180
|
+
translated,
|
|
171
181
|
sa_mask,
|
|
172
182
|
hog_fams,
|
|
173
183
|
hog_parents,
|
|
@@ -190,10 +200,13 @@ class Index(object):
|
|
|
190
200
|
kk1 += int(DIGITS_AA_LOOKUP[kmer[i]] * trans[i])
|
|
191
201
|
|
|
192
202
|
## find offset of new k-mer in sa (jj)
|
|
203
|
+
# the suffix array is sorted by the translated sequences, so
|
|
204
|
+
# the suffixes of the k-mer are compared in translated form
|
|
205
|
+
translated_kmer = translated[sa[ii] : (sa[ii] + k)].view(np.uint8)
|
|
193
206
|
# first in windows of 50s
|
|
194
207
|
# THIS MAY NEED TO BE INCREASED OVER TIME AND IS OPTIMISED FOR THE LUCA DB
|
|
195
208
|
jj = min(ii + 50, len(sa))
|
|
196
|
-
while (jj < len(sa)) and _same_kmer(
|
|
209
|
+
while (jj < len(sa)) and _same_kmer(translated, sa, translated_kmer, jj, k):
|
|
197
210
|
jj = min(jj + 50, len(sa))
|
|
198
211
|
|
|
199
212
|
# then, refine with binary search
|
|
@@ -201,7 +214,7 @@ class Index(object):
|
|
|
201
214
|
hi = jj
|
|
202
215
|
while lo < hi:
|
|
203
216
|
m = int(np.floor((lo + hi) / 2))
|
|
204
|
-
if _same_kmer(
|
|
217
|
+
if _same_kmer(translated, sa, translated_kmer, m, k):
|
|
205
218
|
lo = m + 1
|
|
206
219
|
else:
|
|
207
220
|
hi = m
|
|
@@ -314,6 +327,7 @@ class Index(object):
|
|
|
314
327
|
ii_table_buff = _compute_kmer_table(
|
|
315
328
|
sa,
|
|
316
329
|
seq_buff,
|
|
330
|
+
translated,
|
|
317
331
|
sa_mask,
|
|
318
332
|
h2f,
|
|
319
333
|
self.db._db_HOG.col("ParentOff"),
|
|
@@ -159,7 +159,7 @@ def main():
|
|
|
159
159
|
)
|
|
160
160
|
mkdb_parser.add_argument(
|
|
161
161
|
"--species_tree",
|
|
162
|
-
help="Path to newick file containing species tree. [OXMLBUILD]",
|
|
162
|
+
help="Path to newick file containing species tree. Optional if the OrthoXML file contains the taxonomy (OrthoXML >= 0.5). [OXMLBUILD]",
|
|
163
163
|
type=FileType("r")
|
|
164
164
|
)
|
|
165
165
|
mkdb_parser.add_argument(
|