omamer 2.1.2__tar.gz → 2.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. omamer-2.1.2/README.md → omamer-2.2.0/PKG-INFO +54 -69
  2. omamer-2.2.0/README.md +150 -0
  3. {omamer-2.1.2 → omamer-2.2.0}/omamer/__init__.py +1 -1
  4. {omamer-2.1.2 → omamer-2.2.0}/omamer/_runners.py +31 -22
  5. {omamer-2.1.2 → omamer-2.2.0}/omamer/database.py +43 -36
  6. {omamer-2.1.2 → omamer-2.2.0}/omamer/hierarchy.py +2 -0
  7. {omamer-2.1.2 → omamer-2.2.0}/omamer/index.py +21 -7
  8. {omamer-2.1.2 → omamer-2.2.0}/omamer/main.py +1 -1
  9. {omamer-2.1.2 → omamer-2.2.0}/omamer/merge_search.py +8 -6
  10. {omamer-2.1.2 → omamer-2.2.0}/omamer/orthoxml_parser.py +105 -0
  11. omamer-2.2.0/omamer/species_tree.py +235 -0
  12. omamer-2.2.0/omamer.egg-info/PKG-INFO +188 -0
  13. {omamer-2.1.2 → omamer-2.2.0}/omamer.egg-info/SOURCES.txt +6 -0
  14. {omamer-2.1.2 → omamer-2.2.0}/omamer.egg-info/requires.txt +3 -4
  15. omamer-2.2.0/setup.cfg +16 -0
  16. {omamer-2.1.2 → omamer-2.2.0}/setup.py +10 -6
  17. omamer-2.2.0/tests/test_mkdb.py +397 -0
  18. omamer-2.2.0/tests/test_mkdb_core.py +466 -0
  19. omamer-2.2.0/tests/test_mkdb_sources.py +604 -0
  20. omamer-2.2.0/tests/test_search.py +268 -0
  21. omamer-2.2.0/tests/test_species_tree.py +184 -0
  22. omamer-2.1.2/PKG-INFO +0 -34
  23. omamer-2.1.2/omamer.egg-info/PKG-INFO +0 -34
  24. omamer-2.1.2/setup.cfg +0 -15
  25. {omamer-2.1.2 → omamer-2.2.0}/COPYING +0 -0
  26. {omamer-2.1.2 → omamer-2.2.0}/COPYING.LESSER +0 -0
  27. {omamer-2.1.2 → omamer-2.2.0}/LICENSE +0 -0
  28. {omamer-2.1.2 → omamer-2.2.0}/omamer/_clock.py +0 -0
  29. {omamer-2.1.2 → omamer-2.2.0}/omamer/_utils.py +0 -0
  30. {omamer-2.1.2 → omamer-2.2.0}/omamer/alphabets.py +0 -0
  31. {omamer-2.1.2 → omamer-2.2.0}/omamer/results_reader.py +0 -0
  32. {omamer-2.1.2 → omamer-2.2.0}/omamer/sequence_buffer.py +0 -0
  33. {omamer-2.1.2 → omamer-2.2.0}/omamer/sequence_reader.py +0 -0
  34. {omamer-2.1.2 → omamer-2.2.0}/omamer.egg-info/dependency_links.txt +0 -0
  35. {omamer-2.1.2 → omamer-2.2.0}/omamer.egg-info/entry_points.txt +0 -0
  36. {omamer-2.1.2 → omamer-2.2.0}/omamer.egg-info/top_level.txt +0 -0
  37. {omamer-2.1.2 → omamer-2.2.0}/tests/tests.py +0 -0
@@ -1,14 +1,50 @@
1
+ Metadata-Version: 2.4
2
+ Name: omamer
3
+ Version: 2.2.0
4
+ Summary: OMAmer - tree-driven and alignment-free protein assignment to sub-families
5
+ Home-page: https://github.com/DessimozLab/omamer
6
+ Author: Victor Rossier and Alex Warwick Vesztrocy
7
+ Author-email: alex@warwickvesztrocy.co.uk
8
+ License: LGPLv3
9
+ Requires-Python: >=3.10
10
+ Description-Content-Type: text/markdown
11
+ License-File: LICENSE
12
+ License-File: COPYING
13
+ License-File: COPYING.LESSER
14
+ Requires-Dist: alive_progress
15
+ Requires-Dist: biopython
16
+ Requires-Dist: lxml
17
+ Requires-Dist: numba
18
+ Requires-Dist: numpy>=1.26
19
+ Requires-Dist: pandas<3,>2.0.0
20
+ Requires-Dist: property_manager
21
+ Requires-Dist: Rmath4
22
+ Requires-Dist: scipy
23
+ Requires-Dist: tables>=3.10.1
24
+ Requires-Dist: tqdm
25
+ Provides-Extra: build
26
+ Requires-Dist: pysais; extra == "build"
27
+ Dynamic: author
28
+ Dynamic: author-email
29
+ Dynamic: description
30
+ Dynamic: description-content-type
31
+ Dynamic: home-page
32
+ Dynamic: license
33
+ Dynamic: license-file
34
+ Dynamic: provides-extra
35
+ Dynamic: requires-dist
36
+ Dynamic: requires-python
37
+ Dynamic: summary
38
+
1
39
  # OMAmer - tree-driven and alignment-free protein assignment to subfamilies
2
40
 
3
41
  OMAmer is a novel alignment-free protein family assignment method, which limits over-specific subfamily assignments and is suited to phylogenomic databases with thousands of genomes. It is based on an innovative method using evolutionary-informed _k_-mers for alignment-free mapping to ancestral protein subfamilies. Whilst able to reject non-homologous family-level assignments, it has provided better and quicker subfamily-level assignments than a method based on closest sequences (using DIAMOND).
4
42
 
5
43
  # Installation
6
- Requires Python >= 3.8. Download the package from the PyPI, resolving the dependencies by using ``pip install omamer``.
44
+ Requires Python >= 3.10. Download the package from the PyPI, resolving the dependencies by using ``pip install omamer``.
7
45
 
8
46
  Alternatively, clone this repository and install manually.
9
47
 
10
- Note: Python 3.12 is currently not supported, until the `numba` package is updated ([issue](https://github.com/numba/numba/issues/9197)).
11
-
12
48
  # Pre-Built Databases
13
49
 
14
50
  Pre-built databases are available for the latest OMA release from the [download section on the OMA Browser website](https://omabrowser.org/oma/current).
@@ -101,14 +137,21 @@ The taxon from the predicted HOG that is closest from the reference taxon (given
101
137
 
102
138
 
103
139
  # omamer mkdb - Building a Database
104
- This is currently reliant on the OMA browser's database file and the species phylogeny of HOGs. Building using OrthoXML files available shortly.
105
- - https://omabrowser.org/All/OmaServer.h5
106
- - https://omabrowser.org/All/speciestree.nwk
140
+ A database can be built from one of:
141
+
142
+ - the OMA browser's database file and species tree (``--oma_path``), e.g.
143
+ - https://omabrowser.org/All/OmaServer.h5
144
+ - https://omabrowser.org/All/speciestree.nwk
145
+ - HOGs in an OrthoXML file (e.g. from FastOMA) and the proteomes as FASTA files (``--orthoxml``, ``--sequences``). The species tree is read from the ``<taxonomy>`` of the OrthoXML file (OrthoXML >= 0.5), or given as newick file with ``--species_tree``.
146
+
147
+ The taxon names in the species tree below the root taxon must be unique.
148
+
107
149
  ## Usage
108
- Required arguments: ``--db``, ``--oma_path``
150
+ Required arguments: ``--db``, and ``--oma_path`` or ``--orthoxml`` and ``--sequences``
109
151
 
110
152
  usage: omamer mkdb [-h] --db DB [--nthreads NTHREADS] [--min_fam_size MIN_FAM_SIZE] [--min_fam_completeness MIN_FAM_COMPLETENESS] [--logic {AND,OR}]
111
- [--root_taxon ROOT_TAXON] [--hidden_taxa HIDDEN_TAXA] [--species SPECIES] [--reduced_alphabet] [--k K] --oma_path OMA_PATH
153
+ [--root_taxon ROOT_TAXON] [--hidden_taxa HIDDEN_TAXA] [--reduced_alphabet] [--k K] [--oma_path OMA_PATH]
154
+ [--orthoxml ORTHOXML] [--species_tree SPECIES_TREE] [--sequences [SEQUENCES ...]]
112
155
  [--log_level {debug,info,warning}]
113
156
 
114
157
  ## Arguments
@@ -125,70 +168,12 @@ Required arguments: ``--db``, ``--oma_path``
125
168
  | [``--reduced_alphabet``](#markdown-header--reduced_alphabet)||Use reduced alphabet from Linclust paper
126
169
  | [``--k``](#markdown-header--k)|6|_k_-mer length
127
170
  | [``--oma_path``](#markdown-header--oma_path)||Path to a directory with both OmaServer.h5 and speciestree.nwk
171
+ | [``--orthoxml``](#markdown-header--orthoxml)||Path to an OrthoXML file with the HOGs
172
+ | [``--species_tree``](#markdown-header--species_tree)|``<taxonomy>`` of the OrthoXML file|Path to a newick species tree for ``--orthoxml``. Required if the OrthoXML file has no taxonomy (OrthoXML < 0.5).
173
+ | [``--sequences``](#markdown-header--sequences)||FASTA files with the proteins of the species in the OrthoXML file
128
174
  | [``--log_level``](#markdown-header--log_level)|info|Logging level
129
175
 
130
176
 
131
- # Change log
132
-
133
- #### Version 2.1.1
134
-
135
- - Performance improvements to the mkdb command with orthoxml input
136
- - Added a check for non-unique protein IDs in the input fasta files. Now it gives a more informative error message
137
- - fixed #49
138
-
139
- #### Version 2.1.0
140
- - significant improvements to classification speed
141
-
142
- #### Version 2.0.4
143
- - fixes issue #34 (numpy2 incompatibility)
144
- - experimental support to build omamer databases from orthoxml/fasta files
145
- - update github action to latest versions
146
-
147
- #### Version 2.0.3
148
- - fixes issue #30
149
- - update github action to latest versions
150
-
151
- #### Version 2.0.2
152
- - changed method for hiding taxa in build process. Now takes a file containing taxa to hide on separate lines.
153
- - checks and improved feedback for root taxon and requested taxa to hide.
154
- - root taxon set by default to the root level in speciestree.nwk (previously hard-coded to default to LUCA)
155
-
156
- #### Version 2.0.1
157
- - remove dependency for filehash library
158
- - return better error message if build dependencies are not met, but trying to building an omamer database
159
- - minor fixes
160
-
161
- #### Version 2.0.0
162
- - Major update of database format and search code to improve overall memory useage. Most standard runs with LUCA-level database will run on a machine with 16GB RAM.
163
- - Update to the scoring algorithm for root-level HOG / family assignments, to allow for significance testing. This estimates a binomial distribution for each family, so that we can compute the probability of matching at least as many k-mers as we have observed by chance, for each family that has a match to a given query.
164
- - UX improvements - more feedback during interactive search runs, whilst maintaining small log files.
165
-
166
- #### Version 0.2.5
167
- - Fixes an issue when storing the pre-conputed statistics
168
-
169
- #### Version 0.2.4
170
- - Improved loading time for standard search by pre-computing statistics
171
- - Adding new command line option "info" to show the metadata of the
172
- dataset used to build the omamer database.
173
-
174
-
175
- #### Version 0.2.2
176
- - Automated deployment to PyPI
177
- - Removed PyHAM dependency
178
-
179
- #### Version 0.2.0
180
- - Added ``--min_fam_completeness``, ``--logic``, ``--score`` and ``--reference_taxon`` options
181
- - New output format
182
- - Debugging
183
-
184
- #### Version 0.1.2 - 0.1.3
185
- - Debugging
186
-
187
- #### Version 0.1.0
188
- - Added hidden_taxa and threshold arguments
189
-
190
- #### Version 0.0.1
191
- - Initial release
192
177
 
193
178
  # License
194
179
  OMAmer is a free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
omamer-2.2.0/README.md ADDED
@@ -0,0 +1,150 @@
1
+ # OMAmer - tree-driven and alignment-free protein assignment to subfamilies
2
+
3
+ OMAmer is a novel alignment-free protein family assignment method, which limits over-specific subfamily assignments and is suited to phylogenomic databases with thousands of genomes. It is based on an innovative method using evolutionary-informed _k_-mers for alignment-free mapping to ancestral protein subfamilies. Whilst able to reject non-homologous family-level assignments, it has provided better and quicker subfamily-level assignments than a method based on closest sequences (using DIAMOND).
4
+
5
+ # Installation
6
+ Requires Python >= 3.10. Download the package from the PyPI, resolving the dependencies by using ``pip install omamer``.
7
+
8
+ Alternatively, clone this repository and install manually.
9
+
10
+ # Pre-Built Databases
11
+
12
+ Pre-built databases are available for the latest OMA release from the [download section on the OMA Browser website](https://omabrowser.org/oma/current).
13
+
14
+ - LUCA: https://omabrowser.org/All/LUCA.h5
15
+ - _Metazoa_: https://omabrowser.org/All/Metazoa.h5
16
+ - _Viridiplantae_: https://omabrowser.org/All/Viridiplantae.h5
17
+ - _Saccharomyceta_: https://omabrowser.org/All/Saccharomyceta.h5
18
+ - _Primates_: https://omabrowser.org/All/Primates.h5
19
+
20
+ Their names indicate the root-taxon parameter used. Other non-required parameters were left to default.
21
+
22
+ Note: databases included in the [Zenodo upload](https://zenodo.org/record/4593702) from the manuscript are not supported by the most recent version of OMAmer. We recommend using the most recent release with databases built on the most recent OMA browser release.
23
+
24
+
25
+
26
+ # omamer search - Searching a Database
27
+ Assign proteins to families and subfamilies in a pre-existing database.
28
+ ## Usage
29
+ Required arguments: ``--db``, ``--query``
30
+
31
+ usage: omamer search [-h] -d DB -q QUERY [--threshold THRESHOLD] [--family_alpha FAMILY_ALPHA] [-fo] [-n TOP_N_FAMS] [--reference_taxon REFERENCE_TAXON]
32
+ [-o OUT] [--include_extant_genes] [-c CHUNKSIZE] [-t {0,1,2,3,4,5,6,7,8}] [--log_level {debug,info,warning}] [--silent]
33
+
34
+ ## Arguments
35
+ ### Quick reference table
36
+
37
+ | Short Flag | Flag | Default | Description |
38
+ |:-----------|:---------------------|:-----------------------|:------------|
39
+ | [``-d``](#markdown-header-d) | [``--db``](#markdown-header--db) || Path to existing database (including filename)
40
+ | [``-q``](#markdown-header-q) | [``--query``](#markdown-header--query) || Path to FASTA formatted sequences
41
+ | | [``--threshold``](#markdown-header--threshold) | 0.1 | Threshold applied on the OMAmer-score that is used to vary the specificity of predicted HOGs. The lower the theshold the more (over-)specific predicted HOGs will be.
42
+ | | [``--family_alpha``](#markdown-header--family_alpha) | 1e-6 | Significance threshold used when filtering families.
43
+ | [``-fo``](#markdown-header-fo) | [``--family_only``](#markdown-header--family_only) | False | If set, only place at the family level. Useful for certain analysis. Note: `subfamily_medianseqlen` in the results is for the family level.
44
+ | [``-n``](#markdown-header-n) | [``--top_n_fams``](#markdown-header--top_n_fams) | 1 | Number of top level families to place into. By default, placed into only the best scoring family.
45
+ | [``-o``](#markdown-header-o) | [``--out``](#markdown-header--db) | stdout | Path to output. If not set, defaults to stdout.
46
+ | | [``--include_extant_genes``](#markdown-header--include_extant_genes)||Include extant gene IDs as comma separated entry in results
47
+ | [``-c``](#markdown-header-c) | [``--chunksize``](#markdown-header--chunksize) |10000| Number of queries to process at once.
48
+ | [``-t``](#markdown-header-t) | [``--nthreads``](#markdown-header--db) |1|Number of threads to use
49
+ | | [``--log_level``](#markdown-header--db) |info| Logging level (options debug, info, warning)
50
+ | | [``--silent``](#markdown-header--silent) || Set to silence the output.
51
+ <!--| | [``--reference_taxon``](#markdown-header--reference_taxon) || The placement is stopped when reaching a HOG with the reference taxon (must exist in the OMA database). This is a complementary option to vary the specificity of predicted HOGs.-->
52
+ # Output
53
+
54
+ Output is in the form of a tab-seperated value file (TSV), with metadata added to the header using ``!<tag>: <value>``. A parser can be imported for further analysis in python as ``from omamer.results_reader import results_reader``.
55
+
56
+ ## Output Columns
57
+
58
+ #### Query sequence identifier (`qseqid`)
59
+ The sequence identifier from the input FASTA-formatted sequences.
60
+
61
+ #### Predicted HOG identifier (`hogid`)
62
+ The identifier of the hierarchical orthologous group (HOG) in OMA, which you can access through the OMA browser search bar or its REST API (https://omabrowser.org/api/docs).
63
+
64
+ A HOG identifier is composed of the root-HOG identifier (following “HOG:” and before the first dot), which is followed by its sub-HOGs (before each subsequent dot). For example, for subfamily HOG:0487954.3l.27l, HOG:0487954 is the root-HOG (HOG without-parent), HOG:0487954.3l is its child and HOG:0487954.3l.27l its grandchild.
65
+
66
+ #### Predicted HOG taxonomic level (`hoglevel`)
67
+ The taxonomic level that the predicted HOG is defined at.
68
+
69
+ #### Family p-value (`family_p`)
70
+ p-value of having as many or more of k-mers in common under a binomial distribution. Reported in negative natural log units.
71
+
72
+ #### Family count (`family_count`)
73
+ Count of _k_-mers in common with the family / root level HOG.
74
+
75
+ #### Family normalised count (`family_normcount`)
76
+ Family count, normalised by the expected number of hits for the query's sequence length, with the family's _k_-mer content.
77
+
78
+ #### Sub-Family score (`subfamily_score`)
79
+ The OMAmer-score of the predicted HOG. At the subfamily level, this score captures the excess of similarity that is shared between the query and a given HOG, thus excluding the similarity with regions conserved in more ancestral HOGs.
80
+
81
+ #### Sub-Family count (`subfamily_count`)
82
+ Count of _k_-mers in common with the sub-family / HOG.
83
+
84
+ #### Query sequence length (`qseqlen`)
85
+ Count of _k_-mers in common with the sub-family / HOG.
86
+
87
+ #### Sub-Family median sequence length (`subfamily_medianseqlen`)
88
+ Median length of the sequences that are present in the predicted HOG. In the case of family-only placement, this is instead reported at the root-HOG level.
89
+
90
+ #### Query sequence overlap (`qseq_overlap`)
91
+ The proportion of the query sequence overlapping with _k_-mers of reference root-HOGs. This may be helpful to reject partially homologous matches that are problematic in some applications.
92
+
93
+ #### Sub-Family gene set (`subfamily_geneset`)
94
+ Optionally printed (see ``--include_extant_genes``). Comma-seperated list of extant gene IDs of predicted HOG. The [OMA browser](https://omabrowser.org) can be used to find out more information. In particular, using the [REST API](https://omabrowser.org/api/docs), or via the [Python API Client](https://github.com/DessimozLab/pyomadb).
95
+
96
+ <!-- #### Closest taxon from reference taxon (`closest_taxa`)
97
+ The taxon from the predicted HOG that is closest from the reference taxon (given one was provided). This option provides a mean to evaluate the performance of OMAmer placement given some knowledge of the query taxonomy is available.
98
+ -->
99
+
100
+
101
+ # omamer mkdb - Building a Database
102
+ A database can be built from one of:
103
+
104
+ - the OMA browser's database file and species tree (``--oma_path``), e.g.
105
+ - https://omabrowser.org/All/OmaServer.h5
106
+ - https://omabrowser.org/All/speciestree.nwk
107
+ - HOGs in an OrthoXML file (e.g. from FastOMA) and the proteomes as FASTA files (``--orthoxml``, ``--sequences``). The species tree is read from the ``<taxonomy>`` of the OrthoXML file (OrthoXML >= 0.5), or given as newick file with ``--species_tree``.
108
+
109
+ The taxon names in the species tree below the root taxon must be unique.
110
+
111
+ ## Usage
112
+ Required arguments: ``--db``, and ``--oma_path`` or ``--orthoxml`` and ``--sequences``
113
+
114
+ usage: omamer mkdb [-h] --db DB [--nthreads NTHREADS] [--min_fam_size MIN_FAM_SIZE] [--min_fam_completeness MIN_FAM_COMPLETENESS] [--logic {AND,OR}]
115
+ [--root_taxon ROOT_TAXON] [--hidden_taxa HIDDEN_TAXA] [--reduced_alphabet] [--k K] [--oma_path OMA_PATH]
116
+ [--orthoxml ORTHOXML] [--species_tree SPECIES_TREE] [--sequences [SEQUENCES ...]]
117
+ [--log_level {debug,info,warning}]
118
+
119
+ ## Arguments
120
+ <!-- todo: add short flags -->
121
+ | Flag | Default | Description |
122
+ |:--------------------|:----------------------|:-----------|
123
+ | [``--db``](#markdown-header--db)||Path to new database (including filename)
124
+ | [``--nthreads``](#markdown-header--nthreads)|1|Number of threads to use
125
+ | [``--min_fam_size``](#markdown-header--min_fam_size)|6|Only root-HOGs with a protein count passing this threshold are used.
126
+ | [``--min_fam_completeness``](#markdown-header--min_fam_completeness)|0.5|Only root-HOGs passing this threshold are used. The completeness of a HOG is defined as the number of observed species divided by the expected number of species at the HOG taxonomic level.
127
+ | [``--logic``](#markdown-header--logic)|OR|Logic used between the two above arguments to filter root-HOGs. Options are "AND" or "OR".
128
+ | [``--root_taxon``](#markdown-header--root_taxon)|root of speciestree.nwk|HOGs defined at, or descending from, this taxon are uses as root-HOGs.
129
+ | [``--hidden_taxa``](#markdown-header--hidden_taxa)||The proteins from these taxa are removed before the database computation. Usage: a file containing taxa on seperate lines (scientific name). These must match EXACTLY with the node name in the tree given.
130
+ | [``--reduced_alphabet``](#markdown-header--reduced_alphabet)||Use reduced alphabet from Linclust paper
131
+ | [``--k``](#markdown-header--k)|6|_k_-mer length
132
+ | [``--oma_path``](#markdown-header--oma_path)||Path to a directory with both OmaServer.h5 and speciestree.nwk
133
+ | [``--orthoxml``](#markdown-header--orthoxml)||Path to an OrthoXML file with the HOGs
134
+ | [``--species_tree``](#markdown-header--species_tree)|``<taxonomy>`` of the OrthoXML file|Path to a newick species tree for ``--orthoxml``. Required if the OrthoXML file has no taxonomy (OrthoXML < 0.5).
135
+ | [``--sequences``](#markdown-header--sequences)||FASTA files with the proteins of the species in the OrthoXML file
136
+ | [``--log_level``](#markdown-header--log_level)|info|Logging level
137
+
138
+
139
+
140
+ # License
141
+ OMAmer is a free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
142
+
143
+ OMAmer is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details.
144
+
145
+ You should have received a copy of the GNU Lesser General Public License along with OMAmer. If not, see <http://www.gnu.org/licenses/>.
146
+
147
+ # Citation
148
+ Victor Rossier, Alex Warwick Vesztrocy, Marc Robinson-Rechavi, Christophe Dessimoz, OMAmer: tree-driven and alignment-free protein assignment to subfamilies outperforms closest sequence approaches, Bioinformatics, 2021;, btab219, https://doi.org/10.1093/bioinformatics/btab219
149
+
150
+ Code used for that paper is available here: [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.4593702.svg)](https://doi.org/10.5281/zenodo.4593702)
@@ -24,7 +24,7 @@
24
24
  from datetime import date
25
25
 
26
26
  __packagename__ = "omamer"
27
- __version__ = "2.1.2"
27
+ __version__ = "2.2.0"
28
28
  __copyright__ = "(C) 2019-{:d} Victor Rossier <victor.rossier@unil.ch> and Alex Warwick Vesztrocy <alex@warwickvesztrocy.co.uk> and Nikolai Romashchenko <nikolai.romashchenko@unil.ch>".format(
29
29
  date.today().year
30
30
  )
@@ -28,8 +28,8 @@ from ._utils import LOG, check_file_exists
28
28
  def mkdb_oma(args):
29
29
  from .database import DatabaseFromOMABrowser, DatabaseFromOrthoXML
30
30
  from .index import Index
31
+ from .species_tree import SpeciesTree, SpeciesTreeError
31
32
 
32
- from ete3 import Tree
33
33
  import os
34
34
 
35
35
  _ensure_db_build_dependencies_available()
@@ -59,8 +59,8 @@ def mkdb_oma(args):
59
59
  check_file_exists(nwk)
60
60
 
61
61
  else:
62
- if args.orthoxml is None or args.species_tree is None or len(args.sequences) == 0:
63
- raise ValueError('Must pass either browser database (--oma_path) or all files for OrthoXML build (--orthoxml --species_tree --sequences)')
62
+ if args.orthoxml is None or not args.sequences:
63
+ raise ValueError('Must pass either browser database (--oma_path) or all files for OrthoXML build (--orthoxml --sequences, and --species_tree unless the OrthoXML file contains the taxonomy)')
64
64
 
65
65
  # orthoxml mode
66
66
  Database = DatabaseFromOrthoXML
@@ -68,29 +68,36 @@ def mkdb_oma(args):
68
68
  # find files
69
69
  oxml_fn = args.orthoxml.name
70
70
  fasta_fns = list(map(lambda x: x.name, args.sequences))
71
- nwk = args.species_tree.name
72
-
73
- # check if root_taxon in tree
74
- t = Tree(nwk, format=1, quoted_node_names=True)
75
- if args.root_taxon is not None:
76
- pruned_t = t.search_nodes(name=args.root_taxon)
77
- if len(pruned_t) == 0:
78
- raise ValueError("Unable to find {} as root taxon".format(args.root_taxon))
79
- elif len(pruned_t) > 1:
80
- raise ValueError("Ambiguous root taxon {} ({})".format(args.root_taxon, ", ".join(map(lambda x: x.name, pruned_t))))
81
- t = pruned_t[0]
82
- root_taxon = t.name
71
+ nwk = args.species_tree.name if args.species_tree is not None else None
72
+
73
+ # read the species tree, from the OrthoXML file if none is given
74
+ if nwk is not None:
75
+ full_stree = SpeciesTree.read(nwk)
76
+ else:
77
+ try:
78
+ full_stree = SpeciesTree.from_orthoxml(oxml_fn)
79
+ except SpeciesTreeError:
80
+ raise SpeciesTreeError(
81
+ "{} contains no taxonomy (OrthoXML >= 0.5), pass the species "
82
+ "tree with --species_tree".format(oxml_fn)
83
+ )
84
+ LOG.info("Using the taxonomy of the OrthoXML file as species tree")
85
+
86
+ # check the root taxon, and that the taxa below it have unique names
87
+ stree = full_stree
88
+ root_taxon = args.root_taxon if args.root_taxon is not None else stree.root
89
+ if root_taxon is None:
90
+ raise SpeciesTreeError(
91
+ "The root of the species tree has no name, name it or set --root_taxon"
92
+ )
93
+ stree = stree.subtree(root_taxon)
83
94
 
84
95
  if args.hidden_taxa:
85
96
  # load and check for hidden taxa
86
97
  hidden_taxa = list(map(lambda x: x.rstrip(), args.hidden_taxa))
87
98
 
88
99
  for ht in hidden_taxa:
89
- r = t.search_nodes(name=ht)
90
- if len(r) == 0:
91
- raise ValueError("Unable to find {} as hidden taxon".format(ht))
92
- elif len(r) > 1:
93
- raise ValueError("Ambiguous hidden taxon {} ({})".format(ht, ", ".join(map(lambda x: x.name, r))))
100
+ stree.check_taxon(ht, "hidden taxon")
94
101
  else:
95
102
  hidden_taxa = []
96
103
 
@@ -107,9 +114,9 @@ def mkdb_oma(args):
107
114
  # add sequences from database
108
115
  LOG.info("Loading sequences")
109
116
  if browser_db_mode:
110
- seq_buff = db.build_database(oma_db_fn, nwk)
117
+ seq_buff = db.build_database(oma_db_fn, full_stree)
111
118
  else:
112
- seq_buff = db.build_database(oxml_fn, fasta_fns, nwk)
119
+ seq_buff = db.build_database(oxml_fn, fasta_fns, full_stree)
113
120
 
114
121
  LOG.info("Building index")
115
122
  db.ki = Index(
@@ -254,6 +261,8 @@ def search(args):
254
261
  search_times.append((len(ids), t_search1 - t_search0))
255
262
 
256
263
  db.close()
264
+ if args.out is not sys.stdout:
265
+ args.out.close()
257
266
 
258
267
  search_rate = sum(map(lambda x: x[0], search_times)) / sum(
259
268
  map(lambda x: x[1], search_times)
@@ -191,11 +191,10 @@ class Database(object):
191
191
  self._check_open_writeable()
192
192
 
193
193
  def _parse_stree(stree_path, roottax):
194
- from ete3 import Tree
194
+ from .species_tree import load_species_tree
195
195
 
196
196
  LOG.debug("parsing species tree {}".format(stree_path))
197
- stree = Tree(stree_path, format=1, quoted_node_names=True)
198
- pruned_stree = [x for x in stree.traverse() if x.name == roottax][0]
197
+ stree = load_species_tree(stree_path).subtree(roottax)
199
198
 
200
199
  tax2parent = {}
201
200
  tax2children = {}
@@ -203,19 +202,15 @@ class Database(object):
203
202
  species = set()
204
203
  nspecies_below = {}
205
204
 
206
- for tl in pruned_stree.traverse():
207
- tax = tl.name.encode("ascii")
208
- tax2parent[tax] = (
209
- tl.up.name.encode("ascii") if tl.up else "root".encode("ascii")
210
- )
211
- tax2children[tax] = [x.name.encode("ascii") for x in tl.children]
212
- tax2level[tax] = tl.get_distance(stree)
213
- if tl.is_leaf():
214
- sp_name = tl.name.encode("ascii")
215
- species.add(sp_name)
216
- nspecies_below[sp_name] = 1
217
- else:
218
- nspecies_below[tl.name.encode("ascii")] = sum(1 for _ in tl.iter_leaves())
205
+ for name, parent, children, level, is_species, nspecies in stree.taxa():
206
+ tax = name.encode("ascii")
207
+ # the parent of the root taxon is not used, it gets ParentOff -1
208
+ tax2parent[tax] = (parent if parent is not None else "root").encode("ascii")
209
+ tax2children[tax] = [x.encode("ascii") for x in children]
210
+ tax2level[tax] = level
211
+ if is_species:
212
+ species.add(tax)
213
+ nspecies_below[tax] = nspecies
219
214
 
220
215
  return (tax2parent, tax2children, tax2level, species, nspecies_below)
221
216
 
@@ -398,7 +393,7 @@ class Database(object):
398
393
  for fam_id, hogs in sorted(fam2hogs.items(), key=lambda x: int(x[0])):
399
394
  ### hogs
400
395
  # sort by level
401
- hogs = sorted(hogs, key=lambda x: len(x.split(b".")))
396
+ hogs = sorted(hogs, key=lambda x: (len(x.split(b".")), x))
402
397
  hog_num = len(hogs)
403
398
 
404
399
  # parents
@@ -464,8 +459,8 @@ class Database(object):
464
459
  child_hogs_numbers[i],
465
460
  child_prots_offsets[i],
466
461
  child_prots_numbers[i],
467
- -1,
468
- -1,
462
+ 0,
463
+ 0,
469
464
  0,
470
465
  oma_hog_size,
471
466
  oma_hog_completeness,
@@ -609,18 +604,20 @@ class Database(object):
609
604
 
610
605
  @staticmethod
611
606
  def find_parent_hog(hog_id, hogs):
612
- if isinstance(hog_id, str):
613
- split_hog = hog_id.split(".")
614
- parent_hog = -1
615
- if len(split_hog) > 1:
616
- parent_hog = np.searchsorted(hogs, ".".join(split_hog[:-1]))
617
- return parent_hog
618
- elif isinstance(hog_id, bytes):
619
- split_hog = hog_id.split(b".")
620
- parent_hog = -1
621
- if len(split_hog) > 1:
622
- parent_hog = np.searchsorted(hogs, b".".join(split_hog[:-1]))
623
- return parent_hog
607
+ """
608
+ Offset in the sorted array hogs of the closest ancestor of hog_id, or -1.
609
+
610
+ Intermediate HOGs can be missing, e.g. for a duplication nested directly
611
+ in another duplication HOG:1.1a.1a exists but HOG:1.1a does not.
612
+ """
613
+ sep = "." if isinstance(hog_id, str) else b"."
614
+ split_hog = hog_id.split(sep)
615
+ for i in range(len(split_hog) - 1, 0, -1):
616
+ parent_id = sep.join(split_hog[:i])
617
+ parent_hog = np.searchsorted(hogs, parent_id)
618
+ if parent_hog < len(hogs) and hogs[parent_hog] == parent_id:
619
+ return parent_hog
620
+ return -1
624
621
 
625
622
  def add_median_seqlen_col(self):
626
623
  """
@@ -1069,8 +1066,15 @@ class DatabaseFromOrthoXML(DatabaseFromOMA):
1069
1066
  self.oxml_path = oxml_path
1070
1067
  self.hog_parser = OrthoxmlParser(oxml_path)
1071
1068
 
1072
- def parse_oxml(self, nspecies_below):
1073
- from .orthoxml_parser import is_orthologGroup_node
1069
+ def parse_oxml(self, nspecies_below, stree_path):
1070
+ from .orthoxml_parser import TaxonomyLevels, is_orthologGroup_node
1071
+ from .species_tree import load_species_tree
1072
+
1073
+ # the whole tree, HOGs above the root taxon are only filtered later
1074
+ taxonomy = TaxonomyLevels(load_species_tree(stree_path).parents())
1075
+
1076
+ def gene_species(gene_id):
1077
+ return self.hog_parser.species.get_species(gene_id).name
1074
1078
 
1075
1079
  def generate_hog_tab(hog):
1076
1080
  # this is more memory intensive (we cache the results)
@@ -1143,6 +1147,8 @@ class DatabaseFromOrthoXML(DatabaseFromOMA):
1143
1147
  entries = []
1144
1148
  LOG.debug("loading hog structure and membership from orthoxml")
1145
1149
  for hog in tqdm(self.hog_parser.iter_hogs(auto_clean=True), desc="Parsing HOGs"):
1150
+ # same HOG levels as in the OMA browser
1151
+ hog.insert_missing_levels(taxonomy, gene_species)
1146
1152
  hog_data = list(filter(lambda x: x is not None, generate_hog_tab(hog)))
1147
1153
  gene_data = list(generate_prot_tab(hog))
1148
1154
  hog_data += list(generate_hogs_for_prot(hog, gene_data))
@@ -1182,7 +1188,7 @@ class DatabaseFromOrthoXML(DatabaseFromOMA):
1182
1188
  self.add_taxid_col()
1183
1189
 
1184
1190
  LOG.debug("build hog_table from orthoxml")
1185
- (hog_tab, ent_tab) = self.parse_oxml(nspecies_below)
1191
+ (hog_tab, ent_tab) = self.parse_oxml(nspecies_below, stree_path)
1186
1192
 
1187
1193
  LOG.debug("select and strip OMA HOGs")
1188
1194
  (
@@ -1381,8 +1387,9 @@ class DatabaseFromOrthoXML(DatabaseFromOMA):
1381
1387
  taxid_column = []
1382
1388
  i = -100
1383
1389
  for tax_name in self._db_Taxonomy.col("ID"):
1384
- sp = self.hog_parser.species[tax_name]
1385
- if sp is not None and hasattr(sp, "NCBITaxID"):
1390
+ sp = self.hog_parser.species[tax_name.decode("ascii")]
1391
+ # NCBITaxId="0" means unknown, e.g. in FastOMA output
1392
+ if sp is not None and sp.NCBITaxID > 0:
1386
1393
  taxid_column.append(sp.NCBITaxID)
1387
1394
  else:
1388
1395
  taxid_column.append(i)
@@ -35,6 +35,8 @@ def get_root_leaf_offsets(off, parent_arr):
35
35
  parent = parent_arr[off]
36
36
  while parent != -1:
37
37
  leaf_root.append(parent)
38
+ if len(leaf_root) > len(parent_arr):
39
+ raise ValueError("cycle in parent pointers")
38
40
  parent = parent_arr[parent]
39
41
  return np.array(leaf_root[::-1], dtype=np.uint32)
40
42
 
@@ -93,10 +93,9 @@ class Index(object):
93
93
  assert "/Index" not in self.db.db, "Index has already been computed"
94
94
 
95
95
  # build suffix array with option to translate the sequence buffer first
96
- sa = self._build_suffixarray(
97
- self.alphabet.translate(seq_buff), len(self.db._db_Protein)
98
- )
99
- self._build_kmer_table(seq_buff, sa)
96
+ translated = self.alphabet.translate(seq_buff)
97
+ sa = self._build_suffixarray(translated, len(self.db._db_Protein))
98
+ self._build_kmer_table(seq_buff, sa, translated)
100
99
 
101
100
  @staticmethod
102
101
  def _build_suffixarray(seqs, n):
@@ -110,7 +109,17 @@ class Index(object):
110
109
  sa[:n].sort() # Sort delimiters by position
111
110
  return sa
112
111
 
113
- def _build_kmer_table(self, seq_buff, sa):
112
+ def _build_kmer_table(self, seq_buff, sa, translated=None):
113
+ """
114
+ Build the k-mer table from the suffix array of the translated sequence
115
+ buffer. The k-mer codes are computed from the original sequences, the
116
+ suffixes of a k-mer are found by comparing the translated ones, as with
117
+ a reduced alphabet different k-mers of the original sequences belong
118
+ to the same k-mer.
119
+ """
120
+ if translated is None:
121
+ translated = self.alphabet.translate(seq_buff)
122
+
114
123
  @numba.njit(parallel=True, nogil=True)
115
124
  def _compute_mask_and_filter(
116
125
  sa, sa_mask, sa_filter, k, n, prot2spoff, prot2hogoff, sp_filter
@@ -168,6 +177,7 @@ class Index(object):
168
177
  def _compute_kmer_table(
169
178
  sa,
170
179
  seq_buff,
180
+ translated,
171
181
  sa_mask,
172
182
  hog_fams,
173
183
  hog_parents,
@@ -190,10 +200,13 @@ class Index(object):
190
200
  kk1 += int(DIGITS_AA_LOOKUP[kmer[i]] * trans[i])
191
201
 
192
202
  ## find offset of new k-mer in sa (jj)
203
+ # the suffix array is sorted by the translated sequences, so
204
+ # the suffixes of the k-mer are compared in translated form
205
+ translated_kmer = translated[sa[ii] : (sa[ii] + k)].view(np.uint8)
193
206
  # first in windows of 50s
194
207
  # THIS MAY NEED TO BE INCREASED OVER TIME AND IS OPTIMISED FOR THE LUCA DB
195
208
  jj = min(ii + 50, len(sa))
196
- while (jj < len(sa)) and _same_kmer(seq_buff, sa, kmer, jj, k):
209
+ while (jj < len(sa)) and _same_kmer(translated, sa, translated_kmer, jj, k):
197
210
  jj = min(jj + 50, len(sa))
198
211
 
199
212
  # then, refine with binary search
@@ -201,7 +214,7 @@ class Index(object):
201
214
  hi = jj
202
215
  while lo < hi:
203
216
  m = int(np.floor((lo + hi) / 2))
204
- if _same_kmer(seq_buff, sa, kmer, m, k):
217
+ if _same_kmer(translated, sa, translated_kmer, m, k):
205
218
  lo = m + 1
206
219
  else:
207
220
  hi = m
@@ -314,6 +327,7 @@ class Index(object):
314
327
  ii_table_buff = _compute_kmer_table(
315
328
  sa,
316
329
  seq_buff,
330
+ translated,
317
331
  sa_mask,
318
332
  h2f,
319
333
  self.db._db_HOG.col("ParentOff"),
@@ -159,7 +159,7 @@ def main():
159
159
  )
160
160
  mkdb_parser.add_argument(
161
161
  "--species_tree",
162
- help="Path to newick file containing species tree. [OXMLBUILD]",
162
+ help="Path to newick file containing species tree. Optional if the OrthoXML file contains the taxonomy (OrthoXML >= 0.5). [OXMLBUILD]",
163
163
  type=FileType("r")
164
164
  )
165
165
  mkdb_parser.add_argument(