mgnify-pipelines-toolkit 1.0.0__py3-none-any.whl → 1.0.1__py3-none-any.whl

mgnify_pipelines_toolkit/analysis/shared/study_summary_generator.py

@@ -1,5 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
+import shutil
+from shutil import SameFileError
 
 # Copyright 2024-2025 EMBL - European Bioinformatics Institute
 #
@@ -33,6 +35,7 @@ from mgnify_pipelines_toolkit.schemas.schemas import (
     AmpliconNonINSDCPassedRunsSchema,
     TaxonSchema,
     PR2TaxonSchema,
+    validate_dataframe,
 )
 
 logging.basicConfig(level=logging.DEBUG)
@@ -127,9 +130,9 @@ def parse_one_tax_file(
     # Two different schemas used for validation depending on the database
     # because PR2 schema has different taxonomic ranks than the standard
     if len(long_tax_ranks) == 8:
-        TaxonSchema(res_df)
+        validate_dataframe(res_df, TaxonSchema, str(tax_file))
     elif len(long_tax_ranks) == 9:
-        PR2TaxonSchema(res_df)
+        validate_dataframe(res_df, PR2TaxonSchema, str(tax_file))
 
     res_df["full_taxon"] = res_df.iloc[:, 1:].apply(
         lambda x: ";".join(x).strip(";"), axis=1
@@ -205,9 +208,7 @@ def generate_db_summary(
                 amp_region_dict[amp_region].append(amp_region_df)
 
         for amp_region, amp_region_dfs in amp_region_dict.items():
-            if (
-                len(amp_region_dfs) > 1
-            ):  # Need at least two analyses with this amp_region to bother with the summary
+            if amp_region_dfs:
                 amp_res_df = amp_region_dfs[0]
                 for amp_df in amp_region_dfs[1:]:
                     amp_res_df = amp_res_df.join(amp_df, how="outer")
@@ -319,9 +320,7 @@ def summarise_analyses(
             if tax_file:
                 tax_files[run_acc] = tax_file
 
-        if (
-            len(tax_files) > 1
-        ):  # If at least two analyses have results from the current DB, generate a study-level summary for it
+        if tax_files:
             generate_db_summary(db_label, tax_files, output_prefix)
 
 
@@ -356,12 +355,12 @@ def merge_summaries(analyses_dir: str, output_prefix: str) -> None:
     :type output_prefix: str
     """
 
-    # TODO: The way we grab all the summaries might change depending on how the prefect side does things
     all_study_summaries = glob.glob(f"{analyses_dir}/*_study_summary.tsv")
 
     summaries_dict = organise_study_summaries(all_study_summaries)
 
     for db_label, summaries in summaries_dict.items():
+        merged_summary_name = f"{output_prefix}_{db_label}_study_summary.tsv"
         if len(summaries) > 1:
             res_df = pd.read_csv(summaries[0], sep="\t", index_col=0)
             for summary in summaries[1:]:
@@ -372,10 +371,18 @@ def merge_summaries(analyses_dir: str, output_prefix: str) -> None:
 
             res_df = res_df.reindex(sorted(res_df.columns), axis=1)
             res_df.to_csv(
-                f"{output_prefix}_{db_label}_study_summary.tsv",
+                merged_summary_name,
                 sep="\t",
                 index_label="taxonomy",
             )
+        elif len(summaries) == 1:
+            logging.info(
+                f"Only one summary ({summaries[0]}) so will use that as {merged_summary_name}"
+            )
+            try:
+                shutil.copyfile(summaries[0], merged_summary_name)
+            except SameFileError:
+                pass
 
 
 if __name__ == "__main__":

mgnify_pipelines_toolkit/schemas/schemas.py

@@ -1,6 +1,5 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-
 # Copyright 2024-2025 EMBL - European Bioinformatics Institute
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,11 +13,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import logging
 import re
 
 from enum import Enum
-from typing import ClassVar, Optional
+from typing import ClassVar, Optional, Type
+
+import pandas as pd
 import pandera as pa
+from pandera.typing.common import DataFrameBase
 
 from pydantic import (
     Field,
@@ -215,3 +218,18 @@ class PR2TaxonSchema(pa.DataFrameModel):
 
         dtype = PydanticModel(PR2TaxonRecord)
         coerce = True
+
+
+def validate_dataframe(
+    df: pd.DataFrame, schema: Type[pa.DataFrameModel], df_metadata: str
+) -> DataFrameBase:
+    """
+    Validate a pandas dataframe using a pandera schema.
+    df_metadata will be shown in logs on failure: example, the TSV filename from which the df was read.
+    """
+    try:
+        dfs = schema.validate(df, lazy=True)
+    except pa.errors.SchemaErrors as e:
+        logging.error(f"{schema.__name__} validation failure for {df_metadata}")
+        raise e
+    return dfs

{mgnify_pipelines_toolkit-1.0.0.dist-info → mgnify_pipelines_toolkit-1.0.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mgnify_pipelines_toolkit
-Version: 1.0.0
+Version: 1.0.1
 Summary: Collection of scripts and tools for MGnify pipelines
 Author-email: MGnify team <metagenomics-help@ebi.ac.uk>
 License: Apache Software License 2.0

{mgnify_pipelines_toolkit-1.0.0.dist-info → mgnify_pipelines_toolkit-1.0.1.dist-info}/RECORD

@@ -29,20 +29,20 @@ mgnify_pipelines_toolkit/analysis/shared/get_subunits_coords.py,sha256=EH5RyzesL
 mgnify_pipelines_toolkit/analysis/shared/library_strategy_check.py,sha256=6Ck2NhwRWw66GctUtKDdPT5fwJhWFR_YOZq-Vxwoa8A,1996
 mgnify_pipelines_toolkit/analysis/shared/mapseq2biom.py,sha256=7-U0DN1joVu0ifLOoDUK2Pfqy8rb1RDKT6khVg3jky0,5559
 mgnify_pipelines_toolkit/analysis/shared/markergene_study_summary.py,sha256=sKAo_rKEyVAZXSaIFMkpSoYZxiWwXMA3XDA6Z-hbHgg,7904
-mgnify_pipelines_toolkit/analysis/shared/study_summary_generator.py,sha256=SosRFtW2PWr3dzvLEvYHQFZgGFX0LkQe30sGl3ozThA,13685
+mgnify_pipelines_toolkit/analysis/shared/study_summary_generator.py,sha256=OOqKaQmKGAya6_BZgfcWBZSVlmZ918PQTVMv6KwGIns,13827
 mgnify_pipelines_toolkit/constants/db_labels.py,sha256=omPINMylAjO2PxeFhSk2MbYNcGZH3P82optSlMey3dw,858
 mgnify_pipelines_toolkit/constants/regex_ambiguous_bases.py,sha256=7nEOODQq35y9wx9YnvJuo29oBpwTpXg_kIbf_t7N4TQ,1093
 mgnify_pipelines_toolkit/constants/regex_fasta_header.py,sha256=G-xrc9b8zdmPTaOICD2b3RCVeFAEOVkfRkIfotQ7gek,1193
 mgnify_pipelines_toolkit/constants/tax_ranks.py,sha256=kMq__kOJcbiwsgolkdvb-XLo3WMnJdEXgedjUyMOYjI,1081
 mgnify_pipelines_toolkit/constants/thresholds.py,sha256=guDE7c4KrVJEfg_AcO_cQoJM6LGGaRlmo_U2i8d4N7g,1157
 mgnify_pipelines_toolkit/constants/var_region_coordinates.py,sha256=0bM4MwarFiM5yTcp5AbAmQ0o-q-gWy7kknir9zJ9R0A,1312
-mgnify_pipelines_toolkit/schemas/schemas.py,sha256=Iwps_YtOrIzCuADBgjjJU5VSKb4G0OQZLJfvwRNGN3A,7103
+mgnify_pipelines_toolkit/schemas/schemas.py,sha256=pnH8LUH8i2ACNvFNWyG-n-eIHZcI5O9UDYulkh43mec,7692
 mgnify_pipelines_toolkit/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 mgnify_pipelines_toolkit/utils/fasta_to_delimited.py,sha256=lgYIR1S4crURY7C7nFtgE6QMV4u4zCNsUrVkcRnsEEo,3996
 mgnify_pipelines_toolkit/utils/get_mpt_version.py,sha256=aS9bWrC9CP7tpxoEVg6eEYt18-pmjG7fJl5Mchz4YOU,798
-mgnify_pipelines_toolkit-1.0.0.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-mgnify_pipelines_toolkit-1.0.0.dist-info/METADATA,sha256=46IhEb_9fA1DuCMiQDWyc3yv4EcoZ9KhZ77hWBmjHjA,6181
-mgnify_pipelines_toolkit-1.0.0.dist-info/WHEEL,sha256=52BFRY2Up02UkjOa29eZOS2VxUrpPORXg1pkohGGUS8,91
-mgnify_pipelines_toolkit-1.0.0.dist-info/entry_points.txt,sha256=cTTjlAPQafv9uLrsV4PUGWZgU61qaY8j6uvu0FEpO4A,2309
-mgnify_pipelines_toolkit-1.0.0.dist-info/top_level.txt,sha256=xA_wC7C01V3VwuDnqwRM2QYeJJ45WtvF6LVav4tYxuE,25
-mgnify_pipelines_toolkit-1.0.0.dist-info/RECORD,,
+mgnify_pipelines_toolkit-1.0.1.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+mgnify_pipelines_toolkit-1.0.1.dist-info/METADATA,sha256=3xW9nS84AonTMO6tWU03fii6CqyV5-oa7pa4XrlYvWE,6181
+mgnify_pipelines_toolkit-1.0.1.dist-info/WHEEL,sha256=52BFRY2Up02UkjOa29eZOS2VxUrpPORXg1pkohGGUS8,91
+mgnify_pipelines_toolkit-1.0.1.dist-info/entry_points.txt,sha256=MsQXFdzL_dd7-2V6kHtA-QCf_iSQ-FmDcB9nZMLzJ98,2301
+mgnify_pipelines_toolkit-1.0.1.dist-info/top_level.txt,sha256=xA_wC7C01V3VwuDnqwRM2QYeJJ45WtvF6LVav4tYxuE,25
+mgnify_pipelines_toolkit-1.0.1.dist-info/RECORD,,

{mgnify_pipelines_toolkit-1.0.0.dist-info → mgnify_pipelines_toolkit-1.0.1.dist-info}/entry_points.txt

@@ -4,7 +4,7 @@ are_there_primers = mgnify_pipelines_toolkit.analysis.amplicon.are_there_primers
 assess_inflection_point_mcp = mgnify_pipelines_toolkit.analysis.amplicon.assess_inflection_point_mcp:main
 assess_mcp_proportions = mgnify_pipelines_toolkit.analysis.amplicon.assess_mcp_proportions:main
 classify_var_regions = mgnify_pipelines_toolkit.analysis.amplicon.classify_var_regions:main
-combined_gene_caller_merge = mgnify_pipelines_toolkit.analysis.assembly.combined_gene_caller_merge:combine_main
+combined_gene_caller_merge = mgnify_pipelines_toolkit.analysis.assembly.combined_gene_caller_merge:main
 dwc_summary_generator = mgnify_pipelines_toolkit.analysis.assembly.dwc_summary_generator:main
 fasta_to_delimited = mgnify_pipelines_toolkit.utils.fasta_to_delimited:main
 fastq_suffix_header_check = mgnify_pipelines_toolkit.analysis.shared.fastq_suffix_header_check:main