eukcc 2.1.0__tar.gz → 2.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. eukcc-2.2.0/PKG-INFO +146 -0
  2. eukcc-2.2.0/README.md +108 -0
  3. {eukcc-2.1.0 → eukcc-2.2.0}/eukcc/__main__.py +96 -6
  4. {eukcc-2.1.0 → eukcc-2.2.0}/eukcc/base.py +6 -3
  5. {eukcc-2.1.0 → eukcc-2.2.0}/eukcc/bin.py +1 -0
  6. {eukcc-2.1.0 → eukcc-2.2.0}/eukcc/eukcc.py +201 -48
  7. {eukcc-2.1.0 → eukcc-2.2.0}/eukcc/exec.py +1 -1
  8. {eukcc-2.1.0 → eukcc-2.2.0}/eukcc/fasta.py +4 -3
  9. eukcc-2.2.0/eukcc/helper.py +44 -0
  10. {eukcc-2.1.0 → eukcc-2.2.0}/eukcc/refine.py +43 -15
  11. eukcc-2.2.0/eukcc/set_define.py +35 -0
  12. {eukcc-2.1.0 → eukcc-2.2.0}/eukcc/treehandler.py +70 -27
  13. eukcc-2.2.0/eukcc/version.py +1 -0
  14. eukcc-2.2.0/eukcc.egg-info/PKG-INFO +146 -0
  15. {eukcc-2.1.0 → eukcc-2.2.0}/eukcc.egg-info/SOURCES.txt +9 -1
  16. {eukcc-2.1.0 → eukcc-2.2.0}/eukcc.egg-info/entry_points.txt +2 -1
  17. eukcc-2.2.0/eukcc.egg-info/requires.txt +23 -0
  18. eukcc-2.2.0/eukcc.egg-info/top_level.txt +2 -0
  19. eukcc-2.2.0/pyproject.toml +71 -0
  20. {eukcc-2.1.0 → eukcc-2.2.0}/scripts/binlinks.py +38 -17
  21. {eukcc-2.1.0 → eukcc-2.2.0}/setup.py +2 -2
  22. eukcc-2.2.0/tests/test_basic.py +92 -0
  23. eukcc-2.2.0/tests/test_eukcc.py +7 -0
  24. eukcc-2.2.0/tests/test_fasta.py +73 -0
  25. eukcc-2.2.0/tests/test_file.py +14 -0
  26. eukcc-2.2.0/tests/test_tree.py +42 -0
  27. eukcc-2.1.0/PKG-INFO +0 -101
  28. eukcc-2.1.0/README.md +0 -84
  29. eukcc-2.1.0/eukcc/version.py +0 -1
  30. eukcc-2.1.0/eukcc.egg-info/PKG-INFO +0 -101
  31. eukcc-2.1.0/eukcc.egg-info/requires.txt +0 -3
  32. eukcc-2.1.0/eukcc.egg-info/top_level.txt +0 -1
  33. {eukcc-2.1.0 → eukcc-2.2.0}/LICENSE +0 -0
  34. {eukcc-2.1.0 → eukcc-2.2.0}/MANIFEST.in +0 -0
  35. {eukcc-2.1.0 → eukcc-2.2.0}/eukcc/__init__.py +0 -0
  36. {eukcc-2.1.0 → eukcc-2.2.0}/eukcc/file.py +0 -0
  37. {eukcc-2.1.0 → eukcc-2.2.0}/eukcc/find_markerset.py +0 -0
  38. {eukcc-2.1.0 → eukcc-2.2.0}/eukcc.egg-info/dependency_links.txt +0 -0
  39. {eukcc-2.1.0 → eukcc-2.2.0}/scripts/filter_euk_bins.py +0 -0
  40. {eukcc-2.1.0 → eukcc-2.2.0}/setup.cfg +0 -0
eukcc-2.2.0/PKG-INFO ADDED
@@ -0,0 +1,146 @@
1
+ Metadata-Version: 2.1
2
+ Name: eukcc
3
+ Version: 2.2.0
4
+ Summary: Check eukaryotic genomes or MAGs for completeness and contamination
5
+ Home-page: https://github.com/EBI-Metagenomics/EukCC
6
+ Author: Paul Saary
7
+ Author-email: Paul Saary <eukcc@paulsaary.de>, MGnify team <metagenomics-help@ebi.ac.uk>
8
+ License: GPLv3
9
+ Project-URL: Homepage, https://github.com/EBI-Metagenomics/EukCC
10
+ Project-URL: Documentation, https://eukcc.readthedocs.io/
11
+ Keywords: bioinformatics,mags,eukaryotic,completeness,contamination
12
+ Classifier: Programming Language :: Python :: 3.6
13
+ Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
14
+ Classifier: Operating System :: Unix
15
+ Requires-Python: >=3.6
16
+ Description-Content-Type: text/markdown
17
+ Requires-Dist: ete3>=3.1.3
18
+ Requires-Dist: numpy>=1.24.4
19
+ Requires-Dist: jsonpickle>=4.0.0
20
+ Requires-Dist: biopython==1.83
21
+ Requires-Dist: pysam==0.21.0
22
+ Requires-Dist: six
23
+ Provides-Extra: tests
24
+ Requires-Dist: pytest==7.4.0; extra == "tests"
25
+ Requires-Dist: pytest-md==0.2.0; extra == "tests"
26
+ Requires-Dist: pytest-workflow==2.0.1; extra == "tests"
27
+ Requires-Dist: ete3>=3.1.3; extra == "tests"
28
+ Requires-Dist: numpy>=1.24.4; extra == "tests"
29
+ Requires-Dist: jsonpickle>=4.0.0; extra == "tests"
30
+ Requires-Dist: biopython==1.83; extra == "tests"
31
+ Requires-Dist: pysam==0.21.0; extra == "tests"
32
+ Requires-Dist: six; extra == "tests"
33
+ Provides-Extra: dev
34
+ Requires-Dist: pre-commit==3.8.0; extra == "dev"
35
+ Requires-Dist: black==24.8.0; extra == "dev"
36
+ Requires-Dist: flake8==7.1.1; extra == "dev"
37
+ Requires-Dist: pep8-naming==0.14.1; extra == "dev"
38
+
39
+ # EukCC
40
+
41
+ ![Coverage.py coverage](badges/coverage.svg)
42
+
43
+ EukCC is a completeness and contamination estimator for metagenomic assembled
44
+ microbial eukaryotic genomes.
45
+
46
+ ## Documentation
47
+ Head over to https://eukcc.readthedocs.io/ to check out the documentation.
48
+
49
+ ## Run
50
+
51
+ ### Download EukCC2 database from [FTP](http://ftp.ebi.ac.uk/pub/databases/metagenomics/eukcc/)
52
+ ```
53
+ # create a folder were to keep the database
54
+ mkdir eukccdb
55
+ cd eukccdb
56
+ wget http://ftp.ebi.ac.uk/pub/databases/metagenomics/eukcc/eukcc2_db_ver_1.2.tar.gz
57
+ tar -xzvf eukcc2_db_ver_1.2.tar.gz
58
+ export EUKCC2_DB=$(realpath eukcc2_db_ver_1.2)
59
+ ```
60
+
61
+ ### Quickstart using container
62
+
63
+ Get EukCC quickly by fetching the container.
64
+
65
+ The container is hosted and automatically build from the master branch here:
66
+ https://quay.io/repository/microbiome-informatics/eukcc
67
+ ```
68
+ docker pull quay.io/microbiome-informatics/eukcc
69
+ singularity pull docker://quay.io/microbiome-informatics/eukcc
70
+ ```
71
+
72
+ ### Bioconda / pip
73
+
74
+ Alternatively you can install EukCC using conda or pip.
75
+
76
+ In addition, you need to install mandatory requirements:
77
+ - metaeuk=4.a0f584d
78
+ - pplacer
79
+ - epa-ng=0.3.8
80
+ - hmmer=3.3
81
+ - minimap2
82
+ - bwa
83
+ - samtools
84
+
85
+
86
+ ## Outputs explanation
87
+
88
+ - `eukcc.log` - log of execution
89
+
90
+ ### eukcc single
91
+
92
+ - `eukcc.tsv` - table with estimated completeness, contamination and taxonomy lineage
93
+
94
+ ### eukcc folder
95
+
96
+ - `eukcc.tsv` - table with estimated completeness, contamination and taxonomy lineage for **good** quality bins
97
+ - `merged_bins.csv` - table of merged refined bins
98
+ - `bad_quality.csv`- table with estimated completeness, contamination and taxonomy lineage for **bad** quality bins (chosen marker gene set is supported by less than half of the alignments)
99
+ - `missing_marker_genes.txt` - line separated list of bins with not defined set of marker genes
100
+ - `merged_bins` - folder with merged bins sequences
101
+ - `refine_workdir` - working directory with intermediate steps results
102
+
103
+
104
+ ## Don't use EukCC on already pubished data
105
+ Or at least not without thinking about it:
106
+
107
+ You should not use EukCC on already published genomes, if they have used during training of the marker
108
+ gene sets. If you want to make sure, you can see all used accessions in the database file `db_base/backbone/base_taxinfo.csv`.
109
+
110
+ ## Cite
111
+
112
+ If you use EukCC make sure to cite:
113
+
114
+ ```
115
+ Saary, Paul, Alex L. Mitchell, and Robert D. Finn.
116
+ "Estimating the quality of eukaryotic genomes recovered from metagenomic analysis with EukCC."
117
+ Genome biology 21.1 (2020): 1-21.
118
+ ```
119
+
120
+ EukCC also uses metaEUK, hmmer, pplacer, ete3 and epa-ng.
121
+
122
+
123
+ ## Changed compared to EukCC 1
124
+ **Note**:
125
+ With version 2, EukCC should provide a better experience than
126
+ version 1. Version 2 is not compatible with previous versions, most commandline arguments changed.
127
+ So version 2 is **not** a drop in replacement.
128
+
129
+ - Users can set the prevalence threshold for marker sets. In EukCC 1
130
+ this was fixed to 98% single copy prevalence. Now users could change that to be more strict.
131
+ We find that often 100% single copy prevalence can be found.
132
+
133
+ ## Issues and bugs
134
+
135
+ Please report any bugs and issues here on GitHub. Make sure to
136
+ include the debug log (run eukcc using `--debug` flag).
137
+
138
+ ### used exit codes
139
+ - 200: File not found
140
+ - 201: No Marker gene set could be defined
141
+ - 202: No database provided
142
+ - 203: Corrupted file
143
+ - 204: Predicted zero proteins
144
+ - 222: Invalid settings
145
+
146
+
eukcc-2.2.0/README.md ADDED
@@ -0,0 +1,108 @@
1
+ # EukCC
2
+
3
+ ![Coverage.py coverage](badges/coverage.svg)
4
+
5
+ EukCC is a completeness and contamination estimator for metagenomic assembled
6
+ microbial eukaryotic genomes.
7
+
8
+ ## Documentation
9
+ Head over to https://eukcc.readthedocs.io/ to check out the documentation.
10
+
11
+ ## Run
12
+
13
+ ### Download EukCC2 database from [FTP](http://ftp.ebi.ac.uk/pub/databases/metagenomics/eukcc/)
14
+ ```
15
+ # create a folder were to keep the database
16
+ mkdir eukccdb
17
+ cd eukccdb
18
+ wget http://ftp.ebi.ac.uk/pub/databases/metagenomics/eukcc/eukcc2_db_ver_1.2.tar.gz
19
+ tar -xzvf eukcc2_db_ver_1.2.tar.gz
20
+ export EUKCC2_DB=$(realpath eukcc2_db_ver_1.2)
21
+ ```
22
+
23
+ ### Quickstart using container
24
+
25
+ Get EukCC quickly by fetching the container.
26
+
27
+ The container is hosted and automatically build from the master branch here:
28
+ https://quay.io/repository/microbiome-informatics/eukcc
29
+ ```
30
+ docker pull quay.io/microbiome-informatics/eukcc
31
+ singularity pull docker://quay.io/microbiome-informatics/eukcc
32
+ ```
33
+
34
+ ### Bioconda / pip
35
+
36
+ Alternatively you can install EukCC using conda or pip.
37
+
38
+ In addition, you need to install mandatory requirements:
39
+ - metaeuk=4.a0f584d
40
+ - pplacer
41
+ - epa-ng=0.3.8
42
+ - hmmer=3.3
43
+ - minimap2
44
+ - bwa
45
+ - samtools
46
+
47
+
48
+ ## Outputs explanation
49
+
50
+ - `eukcc.log` - log of execution
51
+
52
+ ### eukcc single
53
+
54
+ - `eukcc.tsv` - table with estimated completeness, contamination and taxonomy lineage
55
+
56
+ ### eukcc folder
57
+
58
+ - `eukcc.tsv` - table with estimated completeness, contamination and taxonomy lineage for **good** quality bins
59
+ - `merged_bins.csv` - table of merged refined bins
60
+ - `bad_quality.csv`- table with estimated completeness, contamination and taxonomy lineage for **bad** quality bins (chosen marker gene set is supported by less than half of the alignments)
61
+ - `missing_marker_genes.txt` - line separated list of bins with not defined set of marker genes
62
+ - `merged_bins` - folder with merged bins sequences
63
+ - `refine_workdir` - working directory with intermediate steps results
64
+
65
+
66
+ ## Don't use EukCC on already pubished data
67
+ Or at least not without thinking about it:
68
+
69
+ You should not use EukCC on already published genomes, if they have used during training of the marker
70
+ gene sets. If you want to make sure, you can see all used accessions in the database file `db_base/backbone/base_taxinfo.csv`.
71
+
72
+ ## Cite
73
+
74
+ If you use EukCC make sure to cite:
75
+
76
+ ```
77
+ Saary, Paul, Alex L. Mitchell, and Robert D. Finn.
78
+ "Estimating the quality of eukaryotic genomes recovered from metagenomic analysis with EukCC."
79
+ Genome biology 21.1 (2020): 1-21.
80
+ ```
81
+
82
+ EukCC also uses metaEUK, hmmer, pplacer, ete3 and epa-ng.
83
+
84
+
85
+ ## Changed compared to EukCC 1
86
+ **Note**:
87
+ With version 2, EukCC should provide a better experience than
88
+ version 1. Version 2 is not compatible with previous versions, most commandline arguments changed.
89
+ So version 2 is **not** a drop in replacement.
90
+
91
+ - Users can set the prevalence threshold for marker sets. In EukCC 1
92
+ this was fixed to 98% single copy prevalence. Now users could change that to be more strict.
93
+ We find that often 100% single copy prevalence can be found.
94
+
95
+ ## Issues and bugs
96
+
97
+ Please report any bugs and issues here on GitHub. Make sure to
98
+ include the debug log (run eukcc using `--debug` flag).
99
+
100
+ ### used exit codes
101
+ - 200: File not found
102
+ - 201: No Marker gene set could be defined
103
+ - 202: No database provided
104
+ - 203: Corrupted file
105
+ - 204: Predicted zero proteins
106
+ - 222: Invalid settings
107
+
108
+
@@ -6,6 +6,7 @@ from eukcc.fasta import determine_type
6
6
  import eukcc.version as version
7
7
  from eukcc.refine import eukcc_folder
8
8
  from eukcc.file import file
9
+ from eukcc.set_define import define_set_w_genomes
9
10
 
10
11
 
11
12
  publications = {
@@ -17,15 +18,23 @@ publications = {
17
18
  }
18
19
 
19
20
 
20
- def update():
21
+ def update(dbfile=None):
21
22
  from ete3 import NCBITaxa
22
23
 
23
24
  logging.info("Going to fetch NCBI info using ete3")
24
- ncbi = NCBITaxa()
25
+ ncbi = NCBITaxa(dbfile=dbfile)
25
26
 
26
27
  ncbi.update_taxonomy_database()
27
28
 
28
29
 
30
+ def no_estimate(E, state):
31
+ """
32
+ It was not possible to estimate this genome. Write the result file with NAs and stop.
33
+ """
34
+ E.write_result(E.state, os.path.join(state["out"], "eukcc.tsv"))
35
+ E.terminate(201)
36
+
37
+
29
38
  def run_eukcc(args):
30
39
  state = eukcc_state(workdir=None, options=vars(args))
31
40
  logging.info("EukCC version {}".format(version.__version__))
@@ -66,10 +75,17 @@ def run_eukcc(args):
66
75
  # copy fna to faa state entry
67
76
  E.state["faa"] = E.state["fasta"]
68
77
 
69
- if not E.state["simple"] and E.state["clade"] == "base":
78
+ if E.state["comparison_genomes"] is not None:
79
+ E.state["simple"] = True
80
+
81
+ if (
82
+ not E.state["simple"]
83
+ and E.state["clade"] == "base"
84
+ and E.state["use_placement"] is None
85
+ ):
70
86
  logging.info("Doing a first pass placement in the base database")
71
87
  if E.placement() is None:
72
- E.terminate(1)
88
+ no_estimate(E, state)
73
89
  # decide which db to use
74
90
  clade = E.determine_subdb()
75
91
  if clade != "base":
@@ -79,18 +95,20 @@ def run_eukcc(args):
79
95
 
80
96
  # pick marker set and placement in one step
81
97
  if E.pick_marker_set() is None:
82
- E.terminate(1)
98
+ no_estimate(E, state)
83
99
  E.state["scmg_data"] = E.hmmsearch_scmg(
84
100
  E.state["workdir"], E.state["faa"], E.state["marker_set"]["profiles"]
85
101
  )
86
102
 
87
103
  logging.debug("Estimating quality")
88
104
  E.compute_quality(E.state["scmg_data"], E.state["marker_set"]["profiles"])
105
+ E.add_taxid()
89
106
 
90
107
  # aggregate all the interesting data, such as estimated quality
91
108
  # the expected and found SCMGs and the workdir
92
109
  E.state.save_state()
93
- E.write_result(E.state, os.path.join(state["out"], "eukcc.csv"))
110
+ E.write_result(E.state, os.path.join(state["out"], "eukcc.tsv"))
111
+
94
112
  if state["extra"]:
95
113
  # search for missing markers
96
114
 
@@ -120,6 +138,7 @@ def main():
120
138
  subparsers = parser.add_subparsers(title="Subcommands", dest="command")
121
139
  pars_eukcc = subparsers.add_parser("single")
122
140
  pars_folder = subparsers.add_parser("folder")
141
+ pars_set = subparsers.add_parser("define_set")
123
142
  subparsers.add_parser("ncbi_update")
124
143
 
125
144
  parser.add_argument(
@@ -274,6 +293,15 @@ def main():
274
293
  default=None,
275
294
  nargs="+",
276
295
  )
296
+ pars_eukcc.add_argument(
297
+ "--genomes",
298
+ type=str,
299
+ required=False,
300
+ help="Genome files to base a SCMG set upon",
301
+ dest="comparison_genomes",
302
+ default=None,
303
+ nargs="+",
304
+ )
277
305
  pars_eukcc.add_argument(
278
306
  "--set_size",
279
307
  type=int,
@@ -387,6 +415,66 @@ def main():
387
415
  default=True,
388
416
  help="Keep workdir after the run.",
389
417
  )
418
+ pars_set.add_argument(
419
+ "genomes",
420
+ type=str,
421
+ help="Genome files to base a SCMG set upon",
422
+ nargs="+",
423
+ )
424
+ pars_set.add_argument(
425
+ "--name", "-n", type=str, default="set", help="Name of the set (default: set)"
426
+ )
427
+ pars_set.add_argument(
428
+ "--out",
429
+ "-o",
430
+ type=str,
431
+ default=".",
432
+ help="Folder to store the output (default: .)",
433
+ )
434
+ pars_set.add_argument("--db", type=str, default=None, help="Path to EukCC DB")
435
+ pars_set.add_argument(
436
+ "--threads",
437
+ "-t",
438
+ type=int,
439
+ help="Number of threads to use (Default: 1)",
440
+ default=1,
441
+ )
442
+ pars_set.add_argument(
443
+ "--max_set_size",
444
+ type=int,
445
+ required=False,
446
+ help="Maximal number of marker genes used, set to 0 to include all possible marker genes (Default: 500)",
447
+ default=500,
448
+ )
449
+ pars_set.add_argument(
450
+ "--set_size",
451
+ type=int,
452
+ required=False,
453
+ help="Minimal number of marker genes to use (Default: 20)",
454
+ default=20,
455
+ )
456
+ pars_set.add_argument(
457
+ "--marker_prevalence",
458
+ type=float,
459
+ required=False,
460
+ help="Percentage of species in which markers should be found (Default: 95)",
461
+ default=95,
462
+ )
463
+ pars_set.add_argument(
464
+ "--AA",
465
+ dest="seqtype",
466
+ action="store_const",
467
+ const="AA",
468
+ help="All provided files are proteins, no gene prediction will be performed. Omit if input is mixed or genomic",
469
+ )
470
+ pars_set.add_argument(
471
+ "--DNA",
472
+ dest="seqtype",
473
+ action="store_const",
474
+ const="DNA",
475
+ default=None,
476
+ help="The fasta file contains DNA sequenes",
477
+ )
390
478
  args = parser.parse_args()
391
479
  if args.command is None:
392
480
  parser.print_help()
@@ -418,5 +506,7 @@ def main():
418
506
  eukcc_folder(args)
419
507
  elif args.command == "ncbi_update":
420
508
  update()
509
+ elif args.command == "define_set":
510
+ define_set_w_genomes(args)
421
511
  else:
422
512
  parser.print_help()
@@ -42,7 +42,7 @@ def compute_silent_contamination(fna, faa, hits):
42
42
  return stats
43
43
 
44
44
 
45
- def load_tax_info(path, sep=","):
45
+ def load_tax_info(path, sep=",", dbfile=None):
46
46
  """
47
47
  Load taxonomic information from two column csv
48
48
  With first column giving the taxid and second column
@@ -52,7 +52,8 @@ def load_tax_info(path, sep=","):
52
52
  :param path: path to csv file
53
53
  :return: dictionary of accession: ncbi_lineage
54
54
  """
55
- ncbi = NCBITaxa()
55
+ logging.debug("Loading NCBI database {}".format(dbfile))
56
+ ncbi = NCBITaxa(dbfile=dbfile)
56
57
  d = {}
57
58
  with open(path) as fin:
58
59
  reader = csv.reader(fin, delimiter=sep)
@@ -94,7 +95,9 @@ def percentage_sets(sets, percent=50, atmost=500):
94
95
  # reduce set to at most n elements
95
96
  # sorting and seting of the seed, will enforce reproducibility
96
97
  if len(keep) > atmost and atmost > 0:
97
- seed(125465) # seed required to obtain the same SCMGs when reducing size, leading to reproducible results
98
+ seed(
99
+ 125465
100
+ ) # seed required to obtain the same SCMGs when reducing size, leading to reproducible results
98
101
  x = list(keep)
99
102
  x.sort()
100
103
  keep = set(sample(x, atmost))
@@ -56,6 +56,7 @@ class bin:
56
56
  keep_hmm=True,
57
57
  )
58
58
  E.compute_quality(E.state["scmg_data"], E.state["marker_set"]["profiles"])
59
+ E.add_taxid()
59
60
 
60
61
  def __str__(self):
61
62
  return "Bin: {}".format(self.name)