mgnify-pipelines-toolkit 0.1.0__tar.gz → 0.1.2__tar.gz

{mgnify_pipelines_toolkit-0.1.0 → mgnify_pipelines_toolkit-0.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mgnify_pipelines_toolkit
-Version: 0.1.0
+Version: 0.1.2
 Summary: Collection of scripts and tools for MGnify pipelines
 Author-email: MGnify team <metagenomics-help@ebi.ac.uk>
 License: Apache Software License 2.0
@@ -32,46 +32,37 @@ This Python package contains a collection of scripts and tools for including in
 - Scripts that don't have existing containers built to run them
 - Scripts for which building an entire container would be too bulky of a solution to deploy in pipelines
 
-This package can be built and uploaded to PyPi, to be installed using pip. The package bundles scripts and makes them executable from the command-line when this package is installed.
-
-> **Soon: this repository will be made available on bioconda for even easier integration in nextflow/nf-core pipelines**.
+This package is built and uploaded to PyPi and bioconda. The package bundles scripts and makes them executable from the command-line when this package is installed.
 
 ## How to install
 
-Currently this package is only available on TestPyPi and is installed like this:
-
-`pip install -i https://test.pypi.org/simple/ --no-deps mgnify-pipelines-toolkit`
-
-You should then be able to run the packages from the command-line. For example to run the `get_subunits.py` script:
-
-`get_subunits -i ${easel_coords} -n ${meta.id}`
-
+This package is available both on [PyPi](https://pypi.org/project/mgnify-pipelines-toolkit/) and bioconda.
 
-## Building and uploading to PyPi
-This command this build the package:
+To install from PyPi with pip:
 
-`python3 -m build`
+`pip install mgnify-pipelines-toolkit`
 
-Then this command will upload it to Test-PyPi (you will need to generate an API token)
+To install from bioconda with conda/mamba:
 
-`python3 -m twine upload --repository testpypi dist/mgnify_pipelines_toolkit-0.0.x*`
+`conda install -c bioconda mgnify-pipelines-toolkit`
 
-To upload it to actual PyPi:
+You should then be able to run the packages from the command-line. For example to run the `get_subunits.py` script:
 
-`python3 -m twine upload dist/mgnify_pipelines_toolkit-0.0.x*`
+`get_subunits -i ${easel_coords} -n ${meta.id}`
 
 ## Adding a new script to the package
 
 ### New script requirements
 
 There are a few requirements for your script:
+
 - It needs to have a named main function of some kind. See `mgnify_pipelines_toolkit/analysis/shared/get_subunits.py` and the `main()` function for an example
 - Because this package is meant to be run from the command-line, make sure your script can easily pass arguments using tools like `argparse` or `click`
 - A small amount of dependencies. This requirement is subjective, but for example if your script only requires a handful of basic packages like `Biopython`, `numpy`, `pandas`, etc., then it's fine. However if the script has a more extensive list of dependencies, a container is probably a better fit.
 
 ### How to add a new script
 
-To add a new Python script, first copy it over to the `mgnify_pipelines_toolkit` directory in this repository, specifically to the subdirectory that makes the most sense. If none of the subdirectories make sense for your script, create a new one. If your script doesn't have a `main()` type function yet, write one. 
+To add a new Python script, first copy it over to the `mgnify_pipelines_toolkit` directory in this repository, specifically to the subdirectory that makes the most sense. If none of the subdirectories make sense for your script, create a new one. If your script doesn't have a `main()` type function yet, write one.
 
 Then, open `pyproject.toml` as you will need to add some bits. First, add any missing dependencies (include the version) to the `dependencies` field.
 
@@ -83,12 +74,18 @@ Then, scroll down to the `[project.scripts]` line. Here, you will create an alia
 
 - `get_subunits` is the alias
 - `mgnify_pipelines_toolkit.analysis.shared.get_subunits` will link the alias to the script with the path `mgnify_pipelines_toolkit/analysis/shared/get_subunits.py`
-- `:main` will specifically call the function named `main()` when the alias is run. 
+- `:main` will specifically call the function named `main()` when the alias is run.
 
 When you have setup this command, executing `get_subunits` on the command-line will be the equivalent of doing:
 
 `from mgnify_pipelines_toolkit.analysis.shared.get_subunits import main; main()`
 
-Finally, you will need to bump up the version in the `version` line. How/when we bump versions is to be determined.
+You should then write at least one unit test for your addition. This package uses `pytest` at the moment for this purpose. A GitHub Action workflow will run all of the unit tests whenever a commit is pushed to any branch.
+
+Finally, you will need to bump up the version in the `version` line.
 
 At the moment, these should be the only steps required to setup your script in this package (which is subject to change).
+
+### Building and uploading to PyPi
+
+The building and pushing of the package is automated by GitHub Actions, which will activate only on a new release. Bioconda should then automatically pick up the new PyPi release and push it to their recipes, though it's worth keeping an eye on their automated pull requests just in case [here](https://github.com/bioconda/bioconda-recipes/pulls).

{mgnify_pipelines_toolkit-0.1.0 → mgnify_pipelines_toolkit-0.1.2}/README.md

@@ -7,46 +7,37 @@ This Python package contains a collection of scripts and tools for including in
 - Scripts that don't have existing containers built to run them
 - Scripts for which building an entire container would be too bulky of a solution to deploy in pipelines
 
-This package can be built and uploaded to PyPi, to be installed using pip. The package bundles scripts and makes them executable from the command-line when this package is installed.
-
-> **Soon: this repository will be made available on bioconda for even easier integration in nextflow/nf-core pipelines**.
+This package is built and uploaded to PyPi and bioconda. The package bundles scripts and makes them executable from the command-line when this package is installed.
 
 ## How to install
 
-Currently this package is only available on TestPyPi and is installed like this:
-
-`pip install -i https://test.pypi.org/simple/ --no-deps mgnify-pipelines-toolkit`
-
-You should then be able to run the packages from the command-line. For example to run the `get_subunits.py` script:
-
-`get_subunits -i ${easel_coords} -n ${meta.id}`
-
+This package is available both on [PyPi](https://pypi.org/project/mgnify-pipelines-toolkit/) and bioconda.
 
-## Building and uploading to PyPi
-This command this build the package:
+To install from PyPi with pip:
 
-`python3 -m build`
+`pip install mgnify-pipelines-toolkit`
 
-Then this command will upload it to Test-PyPi (you will need to generate an API token)
+To install from bioconda with conda/mamba:
 
-`python3 -m twine upload --repository testpypi dist/mgnify_pipelines_toolkit-0.0.x*`
+`conda install -c bioconda mgnify-pipelines-toolkit`
 
-To upload it to actual PyPi:
+You should then be able to run the packages from the command-line. For example to run the `get_subunits.py` script:
 
-`python3 -m twine upload dist/mgnify_pipelines_toolkit-0.0.x*`
+`get_subunits -i ${easel_coords} -n ${meta.id}`
 
 ## Adding a new script to the package
 
 ### New script requirements
 
 There are a few requirements for your script:
+
 - It needs to have a named main function of some kind. See `mgnify_pipelines_toolkit/analysis/shared/get_subunits.py` and the `main()` function for an example
 - Because this package is meant to be run from the command-line, make sure your script can easily pass arguments using tools like `argparse` or `click`
 - A small amount of dependencies. This requirement is subjective, but for example if your script only requires a handful of basic packages like `Biopython`, `numpy`, `pandas`, etc., then it's fine. However if the script has a more extensive list of dependencies, a container is probably a better fit.
 
 ### How to add a new script
 
-To add a new Python script, first copy it over to the `mgnify_pipelines_toolkit` directory in this repository, specifically to the subdirectory that makes the most sense. If none of the subdirectories make sense for your script, create a new one. If your script doesn't have a `main()` type function yet, write one. 
+To add a new Python script, first copy it over to the `mgnify_pipelines_toolkit` directory in this repository, specifically to the subdirectory that makes the most sense. If none of the subdirectories make sense for your script, create a new one. If your script doesn't have a `main()` type function yet, write one.
 
 Then, open `pyproject.toml` as you will need to add some bits. First, add any missing dependencies (include the version) to the `dependencies` field.
 
@@ -58,12 +49,18 @@ Then, scroll down to the `[project.scripts]` line. Here, you will create an alia
 
 - `get_subunits` is the alias
 - `mgnify_pipelines_toolkit.analysis.shared.get_subunits` will link the alias to the script with the path `mgnify_pipelines_toolkit/analysis/shared/get_subunits.py`
-- `:main` will specifically call the function named `main()` when the alias is run. 
+- `:main` will specifically call the function named `main()` when the alias is run.
 
 When you have setup this command, executing `get_subunits` on the command-line will be the equivalent of doing:
 
 `from mgnify_pipelines_toolkit.analysis.shared.get_subunits import main; main()`
 
-Finally, you will need to bump up the version in the `version` line. How/when we bump versions is to be determined.
+You should then write at least one unit test for your addition. This package uses `pytest` at the moment for this purpose. A GitHub Action workflow will run all of the unit tests whenever a commit is pushed to any branch.
+
+Finally, you will need to bump up the version in the `version` line.
+
+At the moment, these should be the only steps required to setup your script in this package (which is subject to change).
+
+### Building and uploading to PyPi
 
-At the moment, these should be the only steps required to setup your script in this package (which is subject to change).
+The building and pushing of the package is automated by GitHub Actions, which will activate only on a new release. Bioconda should then automatically pick up the new PyPi release and push it to their recipes, though it's worth keeping an eye on their automated pull requests just in case [here](https://github.com/bioconda/bioconda-recipes/pulls).

{mgnify_pipelines_toolkit-0.1.0 → mgnify_pipelines_toolkit-0.1.2}/mgnify_pipelines_toolkit/analysis/amplicon/amplicon_utils.py

@@ -15,12 +15,15 @@
 # limitations under the License.
 
 from collections import defaultdict, Counter
+import logging
 import gzip
 import os
 import subprocess
 
 from mgnify_pipelines_toolkit.constants.regex_ambiguous_bases import _AMBIGUOUS_BASES_DICT, _AMBIGUOUS_BASES_DICT_REV
 
+logging.basicConfig(level=logging.DEBUG)
+
 def split_dir_into_sample_paths(_DIR):
 
     file_list = os.listdir(_DIR)
@@ -34,14 +37,28 @@ def split_dir_into_sample_paths(_DIR):
 def get_read_count(read_path, type='fastq'):
 
     cmd = []
+    stdout = ''
     
     if type == 'fastq':
         cmd = [
-            'zgrep',
-            '-c',
-            '^@',
+            'zcat',
             read_path
         ]
+        zcat_proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+        cmd = [
+            'sed',
+            '-n',
+            '1~4p',
+        ]
+        sed_proc = subprocess.Popen(cmd, stdin=zcat_proc.stdout, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+        cmd = [
+            'wc',
+            '-l'
+        ]
+        wc_proc = subprocess.Popen(cmd, stdin=sed_proc.stdout, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        stdout, stderr = wc_proc.communicate()
 
     elif type == 'fasta':
         cmd = [
@@ -50,15 +67,20 @@ def get_read_count(read_path, type='fastq'):
             '^>',
             read_path
         ]
+        grep_proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        stdout, stderr = grep_proc.communicate()
+
+    read_count = stdout.strip() if stdout is not None else "" 
 
-    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    stdout, stderr = proc.communicate()
+    if not read_count.isdigit():
+        logging.error(f"Read count is not a digit, something is wrong. stdout: '{stdout}', stderr: '{stderr}'")
+        exit(1)
 
-    read_count = int(stdout.strip())
+    read_count = int(read_count)
 
     return read_count
 
-def build_cons_seq(cons_list, read_count, cons_threshold=0.80, do_not_include=None, counter=1):
+def build_cons_seq(cons_list, read_count, cons_threshold=0.80, do_not_include=None, counter=1, max_line_count=None):
     """
     Generate consensus sequence using a list of base conservation dictionaries most likely
     generated by the `build_mcp_cons_dict_list()` function.
@@ -85,7 +107,10 @@ def build_cons_seq(cons_list, read_count, cons_threshold=0.80, do_not_include=No
             if base not in ('A', 'T', 'C', 'G'):
                 continue
 
-            cons_dict[base] = count/read_count
+            if max_line_count is None:
+                cons_dict[base] = count/read_count
+            else:
+                cons_dict[base] = count/max_line_count
             
             if count > max_count:
                 max_count = count
@@ -160,7 +185,7 @@ def build_mcp_cons_dict_list(mcp_count_dict, mcp_len):
     
     return mcp_cons_list
 
-def fetch_mcp(fastq, prefix_len, start=1, rev=False):
+def fetch_mcp(fastq, prefix_len, start=1, rev=False, max_line_count=None):
     """
     Generates the most common prefix sequences along with their counts in a fastq file.
     Outputs dictionary containing counts for each generated MCP in the fastq.
@@ -177,6 +202,9 @@ def fetch_mcp(fastq, prefix_len, start=1, rev=False):
                 else:
                     rev_line = line[::-1]
                     selected_lines.append(rev_line[start-1:start+prefix_len-1])
+            if max_line_count != None:
+                if len(selected_lines) > max_line_count:
+                    break
 
     sequence_counts = Counter(selected_lines)
     mcp_count_dict = dict(sorted(sequence_counts.items(), key=lambda x: x[1], reverse=True))

{mgnify_pipelines_toolkit-0.1.0 → mgnify_pipelines_toolkit-0.1.2}/mgnify_pipelines_toolkit/analysis/amplicon/assess_inflection_point_mcp.py

@@ -21,6 +21,7 @@ import numpy as np
 import pandas as pd
 
 from mgnify_pipelines_toolkit.analysis.amplicon.amplicon_utils import get_read_count, build_cons_seq, build_mcp_cons_dict_list, fetch_mcp
+from mgnify_pipelines_toolkit.constants.thresholds import MCP_MAX_LINE_COUNT
 
 def parse_args():
 
@@ -65,14 +66,18 @@ def assess_inflection_point_mcp_for_sample(_PATH, inf_point_list, rev=False):
 
     read_count = get_read_count(_PATH) # get readcount from fastq
 
+    max_line_count = None
+    if read_count > MCP_MAX_LINE_COUNT:
+        max_line_count = MCP_MAX_LINE_COUNT
+
     n_prop = 0.8
 
     for start in inf_point_list: # Looping through the pre-inflection point mcps
         mcp_len = start + 4 # length of pre-inf mcps is inflection point + 4
 
-        mcp_count_dict = fetch_mcp(_PATH, mcp_len, rev=rev) # get MCP count dict 
+        mcp_count_dict = fetch_mcp(_PATH, mcp_len, rev=rev, max_line_count=max_line_count) # get MCP count dict 
         mcp_cons_list = build_mcp_cons_dict_list(mcp_count_dict, mcp_len) # list of base conservation dicts for mcps
-        cons_seq, cons_confs = build_cons_seq(mcp_cons_list, read_count, n_prop, do_not_include_list) # get list of max base conservations for each index
+        cons_seq, cons_confs = build_cons_seq(mcp_cons_list, read_count, n_prop, do_not_include_list, max_line_count=max_line_count) # get list of max base conservations for each index
                                                                                                     # also get consensus sequence
         cons_seq_list.append(cons_seq)
         start_confs.append(np.mean(cons_confs))
@@ -83,9 +88,9 @@ def assess_inflection_point_mcp_for_sample(_PATH, inf_point_list, rev=False):
         subs_len = start_cons_lens[i] # length of respective pre-inf point sequence
         l = mcp_len + subs_len - 1 # final index of MCP
 
-        mcp_count_dict = fetch_mcp(_PATH, l, mcp_len, rev=rev)
+        mcp_count_dict = fetch_mcp(_PATH, l, mcp_len, rev=rev, max_line_count=max_line_count)
         mcp_cons_list = build_mcp_cons_dict_list(mcp_count_dict, subs_len)
-        cons_seq, cons_confs = build_cons_seq(mcp_cons_list, read_count, n_prop, do_not_include_list, subs_len)
+        cons_seq, cons_confs = build_cons_seq(mcp_cons_list, read_count, n_prop, do_not_include_list, subs_len, max_line_count=max_line_count)
 
         end_confs.append(np.mean(cons_confs))
 

{mgnify_pipelines_toolkit-0.1.0 → mgnify_pipelines_toolkit-0.1.2}/mgnify_pipelines_toolkit/analysis/amplicon/assess_mcp_proportions.py

@@ -22,6 +22,7 @@ import pandas as pd
 import numpy as np
 
 from mgnify_pipelines_toolkit.analysis.amplicon.amplicon_utils import get_read_count, build_cons_seq, build_mcp_cons_dict_list, fetch_mcp
+from mgnify_pipelines_toolkit.constants.thresholds import MCP_MAX_LINE_COUNT
 
 def parse_args():
 
@@ -67,9 +68,14 @@ def find_mcp_props_for_sample(_PATH, rev=False):
         end = start+mcp_len-1 # compute the final index for the mcp (inclusive). Indices are of base 1 not 0.
 
         read_count = get_read_count(_PATH, type='fastq') # get read count for fastq file
-        mcp_count_dict = fetch_mcp(_PATH, end, start, rev) # get MCP count dict
+        
+        max_line_count = None
+        if read_count > MCP_MAX_LINE_COUNT:
+            max_line_count = MCP_MAX_LINE_COUNT
+
+        mcp_count_dict = fetch_mcp(_PATH, end, start, rev, max_line_count) # get MCP count dict
         mcp_cons_list = build_mcp_cons_dict_list(mcp_count_dict, mcp_len) # list of base conservation dicts for mcps
-        cons_seq, cons_conf = build_cons_seq(mcp_cons_list, read_count) # get list of max base conservations for each index
+        cons_seq, cons_conf = build_cons_seq(mcp_cons_list, read_count, max_line_count=max_line_count) # get list of max base conservations for each index
         
         res_dict[start] = np.mean(cons_conf) # compute the mean
 

{mgnify_pipelines_toolkit-0.1.0 → mgnify_pipelines_toolkit-0.1.2}/mgnify_pipelines_toolkit/analysis/amplicon/classify_var_regions.py

@@ -24,55 +24,14 @@ import sys
 import json
 import time
 
+from mgnify_pipelines_toolkit.constants.thresholds import MIN_OVERLAP, MIN_SEQ_COUNT, MAX_ERROR_PROPORTION,MAX_INTERNAL_PRIMER_PROPORTION
+from mgnify_pipelines_toolkit.constants.var_region_coordinates import REGIONS_16S_BACTERIA, REGIONS_16S_ARCHAEA, REGIONS_18S
+
 raw_f_regex = re.compile(
     "([A-z0-9\.\-\:]+)\s+-\s+(\w+)\s+(\w+)\s+(\w+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+([-+])\s+([-+])\s+(\d+)\s+(\d+[\.\d]*)\s+(\d+[\.\d]*)\s+(\d+[\.\d]*)\s+(.+)\s!\s+.*")
 
-MIN_OVERLAP = 0.95
-
-MIN_SEQ_COUNT = 5000
-
-MAX_ERROR_PROPORTION = 0.01
-
-MAX_INTERNAL_PRIMER_PROPORTION = 0.2
-
-regions_16S_bacteria = {
-    'V1': [69, 92],
-    'V2': [131, 239],
-    'V3': [430, 487],
-    'V4': [566, 672],
-    'V5': [812, 869],
-    'V6': [976, 1033],
-    'V7': [1107, 1164],
-    'V8': [1234, 1285],
-    'V9': [1426, 1456]
-}
-
-regions_16S_archaea = {
-    'V1': [61, 79],
-    'V2': [114, 223],
-    'V3': [397, 436],
-    'V4': [516, 623],
-    'V5': [763, 824],
-    'V6': [932, 982],
-    'V7': [1056, 1119],
-    'V8': [1189, 1240],
-    'V9': [1372, 1410]
-}
-
-regions_18S = {
-    'V1': [69, 109],
-    'V2': [136, 298],
-    'V3': [474, 545],
-    'V4': [627, 873],
-    'V5': [1059, 1102],
-    'V7': [1366, 1454],
-    'V8': [1526, 1608],
-    'V9': [1728, 1795]
-}
-
 logging.basicConfig(level=logging.DEBUG)
 
-
 def calc_overlap(read, reg):
     read_s, read_f = read
     reg_s, reg_f = reg
@@ -207,11 +166,11 @@ def determine_cm(cm_detected):
         model: A dictionary containing the coordinates of the variable regions for the matched model.
     """
     if cm_detected == 'RF00177':
-        model = regions_16S_bacteria
+        model = REGIONS_16S_BACTERIA
     elif cm_detected == 'RF01959':
-        model = regions_16S_archaea
+        model = REGIONS_16S_ARCHAEA
     elif cm_detected == 'RF01960':
-        model = regions_18S
+        model = REGIONS_18S
     else:
         model = 'unsupported'
     return model

{mgnify_pipelines_toolkit-0.1.0 → mgnify_pipelines_toolkit-0.1.2}/mgnify_pipelines_toolkit/analysis/amplicon/make_asv_count_table.py

@@ -28,7 +28,7 @@ def parse_args():
 
     parser = argparse.ArgumentParser()
 
-    parser.add_argument("-t", "--taxa", required=True, type=str, help="Path to DADA2 taxa file")
+    parser.add_argument("-t", "--taxa", required=True, type=str, help="Path to taxa file")
     parser.add_argument("-f", "--fwd", required=True, type=str, help="Path to DADA2 forward map file")
     parser.add_argument("-r", "--rev", required=False, type=str, help="Path to DADA2 reverse map file")
     parser.add_argument("-a", "--amp", required=True, type=str, help="Path to extracted amp_region reads from inference subworkflow")
@@ -49,7 +49,7 @@ def parse_args():
 
 def order_df(taxa_df):
 
-    if len(taxa_df.columns) == 8:
+    if len(taxa_df.columns) == 9:
         taxa_df = taxa_df.sort_values(_SILVA_TAX_RANKS, ascending=True)
     elif len(taxa_df.columns) == 10:
         taxa_df = taxa_df.sort_values(_PR2_TAX_RANKS, ascending=True)
@@ -66,11 +66,13 @@ def make_tax_assignment_dict_silva(taxa_df, asv_dict):
     for i in range(len(taxa_df)):
         
         sorted_index = taxa_df.index[i]
-        asv_count = asv_dict[sorted_index]
+        asv_num = taxa_df.iloc[i, 0]
+        asv_count = asv_dict[asv_num]
 
         if asv_count == 0:
             continue
 
+        sk = taxa_df.loc[sorted_index, "Superkingdom"]
         k = taxa_df.loc[sorted_index, "Kingdom"]
         p = taxa_df.loc[sorted_index, "Phylum"]
         c = taxa_df.loc[sorted_index, "Class"]
@@ -83,47 +85,53 @@ def make_tax_assignment_dict_silva(taxa_df, asv_dict):
 
         while True:
 
+            if sk != "0":
+                sk = "_".join(sk.split(" "))
+                tax_assignment += sk
+            else:
+                break
+
             if k != "0":
                 k = "_".join(k.split(" "))
-                if k == "Archaea" or k == "Bacteria":
-                    tax_assignment += f"sk__{k}"
-                elif k == "Eukaryota":
-                    tax_assignment += f"sk__Eukaryota"
-                else:
-                    tax_assignment += f"sk__Eukaryota\tk__{k}"
+                tax_assignment += f"\t{k}"
+            elif sk != "0":
+                tax_assignment += f"\tk__"
             else:
                 break
 
             if p != "0":
-                if k == "Archaea" or k == "Bacteria":
-                    tax_assignment += f"\tk__"
                 p = "_".join(p.split(" "))
-                tax_assignment += f"\tp__{p}"
+                tax_assignment += f"\t{p}"
             else:
                 break
+
             if c != "0":
                 c = "_".join(c.split(" "))
-                tax_assignment += f"\tc__{c}"
+                tax_assignment += f"\t{c}"
             else:
                 break
+
             if o != "0":
                 o = "_".join(o.split(" "))
-                tax_assignment += f"\to__{o}"
+                tax_assignment += f"\t{o}"
             else:
                 break
+
             if f != "0":
                 f = "_".join(f.split(" "))
-                tax_assignment += f"\tf__{f}"
+                tax_assignment += f"\t{f}"
             else:
                 break
+
             if g != "0":
                 g = "_".join(g.split(" "))
-                tax_assignment += f"\tg__{g}"
+                tax_assignment += f"\t{g}"
             else:
                 break
+
             if s != "0":
                 s = "_".join(s.split(" "))
-                tax_assignment += f"\ts__{s}"
+                tax_assignment += f"\t{s}"
             break
 
         if tax_assignment == "":
@@ -140,7 +148,8 @@ def make_tax_assignment_dict_pr2(taxa_df, asv_dict):
     for i in range(len(taxa_df)):
         
         sorted_index = taxa_df.index[i]
-        asv_count = asv_dict[sorted_index]
+        asv_num = taxa_df.iloc[i, 0]
+        asv_count = asv_dict[asv_num]
 
         if asv_count == 0:
             continue
@@ -161,45 +170,55 @@ def make_tax_assignment_dict_pr2(taxa_df, asv_dict):
 
             if d != "0":
                 d = "_".join(d.split(" "))
-                tax_assignment += f"d__{d}"
+                tax_assignment += d
             else:
                 break
 
             if sg != "0":
                 sg = "_".join(sg.split(" "))
-                tax_assignment += f"\tsg__{sg}"
+                tax_assignment += f"\t{sg}"
             else:
                 break
+
             if dv != "0":
                 dv = "_".join(dv.split(" "))
-                tax_assignment += f"\tdv__{dv}"
+                tax_assignment += f"\t{dv}"
+            else:
+                break
 
             if sdv != "0":
                 sdv = "_".join(sdv.split(" "))
-                tax_assignment += f"\tsdv__{sdv}"
+                tax_assignment += f"\t{sdv}"
+            else:
+                break
+
             if c != "0":
                 c = "_".join(c.split(" "))
-                tax_assignment += f"\tc__{c}"
+                tax_assignment += f"\t{c}"
             else:
                 break
+
             if o != "0":
                 o = "_".join(o.split(" "))
-                tax_assignment += f"\to__{o}"
+                tax_assignment += f"\t{o}"
             else:
                 break
+
             if f != "0":
                 f = "_".join(f.split(" "))
-                tax_assignment += f"\tf__{f}"
+                tax_assignment += f"\t{f}"
             else:
                 break
+
             if g != "0":
                 g = "_".join(g.split(" "))
-                tax_assignment += f"\tg__{g}"
+                tax_assignment += f"\t{g}"
             else:
                 break
+
             if s != "0":
                 s = "_".join(s.split(" "))
-                tax_assignment += f"\ts__{s}"
+                tax_assignment += f"\t{s}"
             break
 
         if tax_assignment == "":
@@ -253,7 +272,7 @@ def main():
             asv_intersection = fwd_asvs
 
         if headers[counter] in amp_reads:
-            asv_dict[int(asv_intersection[0]) - 1] += 1
+            asv_dict[f"seq_{int(asv_intersection[0]) - 1}"] += 1
     
     fwd_fr.close()
     if paired_end:
@@ -261,7 +280,7 @@ def main():
 
     ref_db = ""
 
-    if len(taxa_df.columns) == 8:
+    if len(taxa_df.columns) == 9:
         tax_assignment_dict = make_tax_assignment_dict_silva(taxa_df, asv_dict)
         ref_db = "silva"
     elif len(taxa_df.columns) == 10:

mgnify_pipelines_toolkit-0.1.2/mgnify_pipelines_toolkit/analysis/amplicon/mapseq_to_asv_table.py

@@ -0,0 +1,126 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright 2024 EMBL - European Bioinformatics Institute
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+from collections import defaultdict
+import logging
+
+import pandas as pd
+
+logging.basicConfig(level=logging.DEBUG)
+
+def parse_args():
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-i", "--input", required=True, type=str, help="Input from MAPseq output")
+    parser.add_argument("-l", "--label", choices=['DADA2-SILVA', 'DADA2-PR2'], required=True, type=str, help="Database label - either DADA2-SILVA or DADA2-PR2")
+    parser.add_argument("-s", "--sample", required=True, type=str, help="Sample ID")
+
+    args = parser.parse_args()
+
+    _INPUT = args.input
+    _LABEL = args.label
+    _SAMPLE = args.sample
+
+    return _INPUT, _LABEL, _SAMPLE
+
+def parse_label(label):
+
+    silva_short_ranks = ["sk__", "k__", "p__", "c__", "o__", "f__", "g__", "s__"]
+    pr2_short_ranks = ["d__", "sg__", "dv__", "sdv__", "c__", "o__", "f__", "g__", "s__"]
+
+    silva_long_ranks = ["Superkingdom", "Kingdom", "Phylum", "Class", "Order", "Family", "Genus", "Species"]
+    pr2_long_ranks = ["Domain", "Supergroup", "Division", "Subdivision", "Class", "Order", "Family", "Genus", "Species"]
+
+    chosen_short_ranks = ''
+    chosen_long_ranks = ''
+
+    if label == 'DADA2-SILVA':
+        chosen_short_ranks = silva_short_ranks
+        chosen_long_ranks = silva_long_ranks
+    elif label == 'DADA2-PR2':
+        chosen_short_ranks = pr2_short_ranks
+        chosen_long_ranks = pr2_long_ranks
+    else:
+        logging.error("Incorrect database label - exiting.")
+        exit(1)
+
+    return chosen_short_ranks, chosen_long_ranks
+
+def parse_mapseq(mseq_df, short_ranks, long_ranks):
+
+    res_dict = defaultdict(list)
+
+    for i in range(len(mseq_df)):
+        asv_id = mseq_df.iloc[i, 0]
+        tax_ass = mseq_df.iloc[i, 1].split(';')
+
+        res_dict['ASV'].append(asv_id)
+        
+        for j in range(len(short_ranks)):
+
+            curr_rank = long_ranks[j]
+            
+            if j >= len(tax_ass):
+                # This would only be true if the assigned taxonomy is shorter than the total reference database taxononmy
+                # so fill each remaining rank with its respective short rank blank
+                curr_tax = short_ranks[j]
+            else:
+                curr_tax = tax_ass[j]
+            
+            res_dict[curr_rank].append(curr_tax)
+    res_df = pd.DataFrame.from_dict(res_dict)
+
+    return(res_df)
+
+def process_blank_tax_ends(res_df, ranks):
+    # Necessary function as we want to replace consecutive blank assignments that start at the last rank as NAs
+    # while avoiding making blanks in the middle as NAs
+
+    for i in range(len(res_df)):
+        last_empty_rank = ''
+        currently_empty = False
+        for j in reversed(range(len(ranks))): # Parse an assignment backwards, from Species all the way to Superkingdom/Domain
+            curr_rank = res_df.iloc[i, j+1]
+            if curr_rank in ranks:
+                if last_empty_rank == '': # Last rank is empty, start window of consecutive blanks 
+                    last_empty_rank = j+1
+                    currently_empty = True
+                elif currently_empty: # If we're in a window of consecutive blank assignments that started at the beginning
+                    last_empty_rank = j+1
+                else:
+                    break
+            else:
+                break
+        if last_empty_rank != '':
+            res_df.iloc[i, last_empty_rank:] = 'NA'
+
+    return res_df
+
+def main():
+    
+    _INPUT, _LABEL, _SAMPLE = parse_args()
+
+    mseq_df = pd.read_csv(_INPUT, header=1, delim_whitespace=True, usecols=[0, 12])
+
+    short_ranks, long_ranks = parse_label(_LABEL)
+    res_df = parse_mapseq(mseq_df, short_ranks, long_ranks)
+    final_res_df = process_blank_tax_ends(res_df, short_ranks)
+
+    final_res_df.to_csv(f"./{_SAMPLE}_{_LABEL}_asv_taxa.tsv", sep="\t", index=False)
+
+if __name__ == "__main__":
+    main()

{mgnify_pipelines_toolkit-0.1.0 → mgnify_pipelines_toolkit-0.1.2}/mgnify_pipelines_toolkit/constants/tax_ranks.py

@@ -14,5 +14,5 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-_SILVA_TAX_RANKS = ["Kingdom", "Phylum", "Class", "Order", "Family", "Genus", "Species"]
+_SILVA_TAX_RANKS = ["Superkingdom", "Kingdom", "Phylum", "Class", "Order", "Family", "Genus", "Species"]
 _PR2_TAX_RANKS = ["Domain", "Supergroup", "Division", "Subdivision", "Class", "Order", "Family", "Genus", "Species"]

mgnify_pipelines_toolkit-0.1.2/mgnify_pipelines_toolkit/constants/thresholds.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright 2024 EMBL - European Bioinformatics Institute
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# used by fetch_mcp in analysis.amplicon
+MCP_MAX_LINE_COUNT = 300_000
+
+# used by classify_var_regions in analysis.amplicon
+MIN_OVERLAP = 0.95
+MIN_SEQ_COUNT = 5000
+MAX_ERROR_PROPORTION = 0.01
+MAX_INTERNAL_PRIMER_PROPORTION = 0.2

mgnify_pipelines_toolkit-0.1.2/mgnify_pipelines_toolkit/constants/var_region_coordinates.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright 2024 EMBL - European Bioinformatics Institute
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+REGIONS_16S_BACTERIA = {
+    'V1': [69, 92],
+    'V2': [131, 239],
+    'V3': [430, 487],
+    'V4': [566, 672],
+    'V5': [812, 869],
+    'V6': [976, 1033],
+    'V7': [1107, 1164],
+    'V8': [1234, 1285],
+    'V9': [1426, 1456]
+}
+
+REGIONS_16S_ARCHAEA = {
+    'V1': [61, 79],
+    'V2': [114, 223],
+    'V3': [397, 436],
+    'V4': [516, 623],
+    'V5': [763, 824],
+    'V6': [932, 982],
+    'V7': [1056, 1119],
+    'V8': [1189, 1240],
+    'V9': [1372, 1410]
+}
+
+REGIONS_18S = {
+    'V1': [69, 109],
+    'V2': [136, 298],
+    'V3': [474, 545],
+    'V4': [627, 873],
+    'V5': [1059, 1102],
+    'V7': [1366, 1454],
+    'V8': [1526, 1608],
+    'V9': [1728, 1795]
+}

{mgnify_pipelines_toolkit-0.1.0 → mgnify_pipelines_toolkit-0.1.2}/mgnify_pipelines_toolkit.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mgnify_pipelines_toolkit
-Version: 0.1.0
+Version: 0.1.2
 Summary: Collection of scripts and tools for MGnify pipelines
 Author-email: MGnify team <metagenomics-help@ebi.ac.uk>
 License: Apache Software License 2.0
@@ -32,46 +32,37 @@ This Python package contains a collection of scripts and tools for including in
 - Scripts that don't have existing containers built to run them
 - Scripts for which building an entire container would be too bulky of a solution to deploy in pipelines
 
-This package can be built and uploaded to PyPi, to be installed using pip. The package bundles scripts and makes them executable from the command-line when this package is installed.
-
-> **Soon: this repository will be made available on bioconda for even easier integration in nextflow/nf-core pipelines**.
+This package is built and uploaded to PyPi and bioconda. The package bundles scripts and makes them executable from the command-line when this package is installed.
 
 ## How to install
 
-Currently this package is only available on TestPyPi and is installed like this:
-
-`pip install -i https://test.pypi.org/simple/ --no-deps mgnify-pipelines-toolkit`
-
-You should then be able to run the packages from the command-line. For example to run the `get_subunits.py` script:
-
-`get_subunits -i ${easel_coords} -n ${meta.id}`
-
+This package is available both on [PyPi](https://pypi.org/project/mgnify-pipelines-toolkit/) and bioconda.
 
-## Building and uploading to PyPi
-This command this build the package:
+To install from PyPi with pip:
 
-`python3 -m build`
+`pip install mgnify-pipelines-toolkit`
 
-Then this command will upload it to Test-PyPi (you will need to generate an API token)
+To install from bioconda with conda/mamba:
 
-`python3 -m twine upload --repository testpypi dist/mgnify_pipelines_toolkit-0.0.x*`
+`conda install -c bioconda mgnify-pipelines-toolkit`
 
-To upload it to actual PyPi:
+You should then be able to run the packages from the command-line. For example to run the `get_subunits.py` script:
 
-`python3 -m twine upload dist/mgnify_pipelines_toolkit-0.0.x*`
+`get_subunits -i ${easel_coords} -n ${meta.id}`
 
 ## Adding a new script to the package
 
 ### New script requirements
 
 There are a few requirements for your script:
+
 - It needs to have a named main function of some kind. See `mgnify_pipelines_toolkit/analysis/shared/get_subunits.py` and the `main()` function for an example
 - Because this package is meant to be run from the command-line, make sure your script can easily pass arguments using tools like `argparse` or `click`
 - A small amount of dependencies. This requirement is subjective, but for example if your script only requires a handful of basic packages like `Biopython`, `numpy`, `pandas`, etc., then it's fine. However if the script has a more extensive list of dependencies, a container is probably a better fit.
 
 ### How to add a new script
 
-To add a new Python script, first copy it over to the `mgnify_pipelines_toolkit` directory in this repository, specifically to the subdirectory that makes the most sense. If none of the subdirectories make sense for your script, create a new one. If your script doesn't have a `main()` type function yet, write one. 
+To add a new Python script, first copy it over to the `mgnify_pipelines_toolkit` directory in this repository, specifically to the subdirectory that makes the most sense. If none of the subdirectories make sense for your script, create a new one. If your script doesn't have a `main()` type function yet, write one.
 
 Then, open `pyproject.toml` as you will need to add some bits. First, add any missing dependencies (include the version) to the `dependencies` field.
 
@@ -83,12 +74,18 @@ Then, scroll down to the `[project.scripts]` line. Here, you will create an alia
 
 - `get_subunits` is the alias
 - `mgnify_pipelines_toolkit.analysis.shared.get_subunits` will link the alias to the script with the path `mgnify_pipelines_toolkit/analysis/shared/get_subunits.py`
-- `:main` will specifically call the function named `main()` when the alias is run. 
+- `:main` will specifically call the function named `main()` when the alias is run.
 
 When you have setup this command, executing `get_subunits` on the command-line will be the equivalent of doing:
 
 `from mgnify_pipelines_toolkit.analysis.shared.get_subunits import main; main()`
 
-Finally, you will need to bump up the version in the `version` line. How/when we bump versions is to be determined.
+You should then write at least one unit test for your addition. This package uses `pytest` at the moment for this purpose. A GitHub Action workflow will run all of the unit tests whenever a commit is pushed to any branch.
+
+Finally, you will need to bump up the version in the `version` line.
 
 At the moment, these should be the only steps required to setup your script in this package (which is subject to change).
+
+### Building and uploading to PyPi
+
+The building and pushing of the package is automated by GitHub Actions, which will activate only on a new release. Bioconda should then automatically pick up the new PyPi release and push it to their recipes, though it's worth keeping an eye on their automated pull requests just in case [here](https://github.com/bioconda/bioconda-recipes/pulls).

{mgnify_pipelines_toolkit-0.1.0 → mgnify_pipelines_toolkit-0.1.2}/mgnify_pipelines_toolkit.egg-info/SOURCES.txt

@@ -16,6 +16,7 @@ mgnify_pipelines_toolkit/analysis/amplicon/assess_mcp_proportions.py
 mgnify_pipelines_toolkit/analysis/amplicon/classify_var_regions.py
 mgnify_pipelines_toolkit/analysis/amplicon/find_mcp_inflection_points.py
 mgnify_pipelines_toolkit/analysis/amplicon/make_asv_count_table.py
+mgnify_pipelines_toolkit/analysis/amplicon/mapseq_to_asv_table.py
 mgnify_pipelines_toolkit/analysis/amplicon/remove_ambiguous_reads.py
 mgnify_pipelines_toolkit/analysis/amplicon/rev_comp_se_primers.py
 mgnify_pipelines_toolkit/analysis/amplicon/standard_primer_matching.py
@@ -24,4 +25,6 @@ mgnify_pipelines_toolkit/analysis/shared/get_subunits.py
 mgnify_pipelines_toolkit/analysis/shared/get_subunits_coords.py
 mgnify_pipelines_toolkit/analysis/shared/mapseq2biom.py
 mgnify_pipelines_toolkit/constants/regex_ambiguous_bases.py
-mgnify_pipelines_toolkit/constants/tax_ranks.py
+mgnify_pipelines_toolkit/constants/tax_ranks.py
+mgnify_pipelines_toolkit/constants/thresholds.py
+mgnify_pipelines_toolkit/constants/var_region_coordinates.py

{mgnify_pipelines_toolkit-0.1.0 → mgnify_pipelines_toolkit-0.1.2}/mgnify_pipelines_toolkit.egg-info/entry_points.txt

@@ -8,6 +8,7 @@ get_subunits = mgnify_pipelines_toolkit.analysis.shared.get_subunits:main
 get_subunits_coords = mgnify_pipelines_toolkit.analysis.shared.get_subunits_coords:main
 make_asv_count_table = mgnify_pipelines_toolkit.analysis.amplicon.make_asv_count_table:main
 mapseq2biom = mgnify_pipelines_toolkit.analysis.shared.mapseq2biom:main
+mapseq_to_asv_table = mgnify_pipelines_toolkit.analysis.amplicon.mapseq_to_asv_table:main
 remove_ambiguous_reads = mgnify_pipelines_toolkit.analysis.amplicon.remove_ambiguous_reads:main
 rev_comp_se_primers = mgnify_pipelines_toolkit.analysis.amplicon.rev_comp_se_primers:main
 standard_primer_matching = mgnify_pipelines_toolkit.analysis.amplicon.standard_primer_matching:main

{mgnify_pipelines_toolkit-0.1.0 → mgnify_pipelines_toolkit-0.1.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "mgnify_pipelines_toolkit"
-version = "0.1.0"
+version = "0.1.2"
 readme = "README.md"
 license = {text = "Apache Software License 2.0"}
 authors = [
@@ -49,6 +49,7 @@ make_asv_count_table = "mgnify_pipelines_toolkit.analysis.amplicon.make_asv_coun
 remove_ambiguous_reads = "mgnify_pipelines_toolkit.analysis.amplicon.remove_ambiguous_reads:main"
 rev_comp_se_primers = "mgnify_pipelines_toolkit.analysis.amplicon.rev_comp_se_primers:main"
 standard_primer_matching = "mgnify_pipelines_toolkit.analysis.amplicon.standard_primer_matching:main"
+mapseq_to_asv_table = "mgnify_pipelines_toolkit.analysis.amplicon.mapseq_to_asv_table:main"
 
 [project.optional-dependencies]
 tests = [