pheval 0.3.0__tar.gz → 0.3.2__tar.gz

{pheval-0.3.0 → pheval-0.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pheval
-Version: 0.3.0
+Version: 0.3.2
 Summary: 
 Author: Yasemin Bridges
 Author-email: y.bridges@qmul.ac.uk

{pheval-0.3.0 → pheval-0.3.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pheval"
-version = "0.3.0"
+version = "0.3.2"
 description = ""
 authors = ["Yasemin Bridges <y.bridges@qmul.ac.uk>",
   "Julius Jacobsen <j.jacobsen@qmul.ac.uk>",

{pheval-0.3.0 → pheval-0.3.2}/src/pheval/analyse/analysis.py

@@ -41,7 +41,11 @@ def _run_benchmark(
     benchmark_result = benchmark_generator.generate_benchmark_run_results(
         results_dir_and_input, score_order, threshold, rank_comparison
     )
-    stats_writer.write_row(results_dir_and_input.results_dir, benchmark_result.rank_stats)
+    stats_writer.write_row(
+        results_dir_and_input.results_dir,
+        benchmark_result.rank_stats,
+        benchmark_result.binary_classification_stats,
+    )
     generate_benchmark_output(benchmark_result, plot_type, benchmark_generator)
     stats_writer.close()
 
@@ -127,7 +131,11 @@ def _run_benchmark_comparison(
         benchmark_result = benchmark_generator.generate_benchmark_run_results(
             results_dir_and_input, score_order, threshold, rank_comparison
         )
-        stats_writer.write_row(results_dir_and_input.results_dir, benchmark_result.rank_stats)
+        stats_writer.write_row(
+            results_dir_and_input.results_dir,
+            benchmark_result.rank_stats,
+            benchmark_result.binary_classification_stats,
+        )
         benchmarking_results.append(benchmark_result)
     generate_benchmark_comparison_output(benchmarking_results, plot_type, benchmark_generator)
     stats_writer.close()

{pheval-0.3.0 → pheval-0.3.2}/src/pheval/analyse/binary_classification_stats.py

@@ -1,4 +1,4 @@
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from math import sqrt
 from typing import List, Union
 
@@ -29,6 +29,8 @@ class BinaryClassificationStats:
     true_negatives: int = 0
     false_positives: int = 0
     false_negatives: int = 0
+    labels: List = field(default_factory=list)
+    scores: List = field(default_factory=list)
 
     @staticmethod
     def remove_relevant_ranks(
@@ -84,6 +86,31 @@ class BinaryClassificationStats:
             elif rank != 1:
                 self.true_negatives += 1
 
+    def add_labels_and_scores(
+        self,
+        pheval_results: Union[
+            List[RankedPhEvalGeneResult],
+            List[RankedPhEvalVariantResult],
+            List[RankedPhEvalDiseaseResult],
+        ],
+        relevant_ranks: List[int],
+    ):
+        """
+        Adds scores and labels from the PhEval results.
+
+        Args:
+            pheval_results (Union[List[RankedPhEvalGeneResult], List[RankedPhEvalVariantResult],
+                                  List[RankedPhEvalDiseaseResult]]):
+                List of all PhEval results
+            relevant_ranks (List[int]): A list of the ranks associated with the known entities.
+        """
+        relevant_ranks_copy = relevant_ranks.copy()
+        for result in pheval_results:
+            self.scores.append(result.score)
+            label = 1 if result.rank in relevant_ranks_copy else 0
+            self.labels.append(label)
+            relevant_ranks_copy.remove(result.rank) if label == 1 else None
+
     def add_classification(
         self,
         pheval_results: Union[
@@ -105,6 +132,7 @@ class BinaryClassificationStats:
         self.add_classification_for_other_entities(
             self.remove_relevant_ranks(pheval_results, relevant_ranks)
         )
+        self.add_labels_and_scores(pheval_results, relevant_ranks)
 
     def sensitivity(self) -> float:
         """

{pheval-0.3.0 → pheval-0.3.2}/src/pheval/analyse/disease_prioritisation_analysis.py

@@ -10,7 +10,11 @@ from pheval.analyse.prioritisation_result_types import DiseasePrioritisationResu
 from pheval.analyse.rank_stats import RankStats
 from pheval.analyse.run_data_parser import TrackInputOutputDirectories
 from pheval.post_processing.post_processing import RankedPhEvalDiseaseResult
-from pheval.utils.file_utils import all_files, files_with_suffix, obtain_closest_file_name
+from pheval.utils.file_utils import (
+    all_files,
+    files_with_suffix,
+    obtain_phenopacket_path_from_pheval_result,
+)
 from pheval.utils.phenopacket_utils import PhenopacketUtil, ProbandDisease, phenopacket_reader
 
 
@@ -234,7 +238,7 @@ def assess_phenopacket_disease_prioritisation(
         disease_rank_comparison (defaultdict): Default dictionary for disease rank comparisons.
         disease_binary_classification_stats (BinaryClassificationStats): BinaryClassificationStats class instance.
     """
-    phenopacket_path = obtain_closest_file_name(
+    phenopacket_path = obtain_phenopacket_path_from_pheval_result(
         standardised_disease_result, all_files(results_dir_and_input.phenopacket_dir)
     )
     pheval_disease_result = read_standardised_result(standardised_disease_result)

{pheval-0.3.0 → pheval-0.3.2}/src/pheval/analyse/gene_prioritisation_analysis.py

@@ -10,7 +10,11 @@ from pheval.analyse.prioritisation_result_types import GenePrioritisationResult
 from pheval.analyse.rank_stats import RankStats
 from pheval.analyse.run_data_parser import TrackInputOutputDirectories
 from pheval.post_processing.post_processing import RankedPhEvalGeneResult
-from pheval.utils.file_utils import all_files, files_with_suffix, obtain_closest_file_name
+from pheval.utils.file_utils import (
+    all_files,
+    files_with_suffix,
+    obtain_phenopacket_path_from_pheval_result,
+)
 from pheval.utils.phenopacket_utils import PhenopacketUtil, ProbandCausativeGene, phenopacket_reader
 
 
@@ -226,7 +230,7 @@ def assess_phenopacket_gene_prioritisation(
         gene_rank_comparison (defaultdict): Default dictionary for gene rank comparisons.
         gene_binary_classification_stats (BinaryClassificationStats): BinaryClassificationStats class instance.
     """
-    phenopacket_path = obtain_closest_file_name(
+    phenopacket_path = obtain_phenopacket_path_from_pheval_result(
         standardised_gene_result, all_files(results_dir_and_input.phenopacket_dir)
     )
     pheval_gene_result = read_standardised_result(standardised_gene_result)

{pheval-0.3.0 → pheval-0.3.2}/src/pheval/analyse/generate_plots.py

@@ -5,6 +5,7 @@ import matplotlib
 import pandas as pd
 import seaborn as sns
 from matplotlib import pyplot as plt
+from sklearn.metrics import auc, precision_recall_curve, roc_curve
 
 from pheval.analyse.benchmark_generator import (
     BenchmarkRunOutputGenerator,
@@ -357,6 +358,82 @@ class PlotGenerator:
             ]
         )
 
+    def generate_roc_curve(
+        self,
+        benchmarking_results: List[BenchmarkRunResults],
+        benchmark_generator: BenchmarkRunOutputGenerator,
+    ):
+        """
+        Generate and plot Receiver Operating Characteristic (ROC) curves for binary classification benchmark results.
+
+        Args:
+            benchmarking_results (List[BenchmarkRunResults]): List of benchmarking results for multiple runs.
+            benchmark_generator (BenchmarkRunOutputGenerator): Object containing benchmarking output generation details.
+        """
+        for i, benchmark_result in enumerate(benchmarking_results):
+            fpr, tpr, thresh = roc_curve(
+                benchmark_result.binary_classification_stats.labels,
+                benchmark_result.binary_classification_stats.scores,
+                pos_label=1,
+            )
+            roc_auc = auc(fpr, tpr)
+
+            plt.plot(
+                fpr,
+                tpr,
+                label=f"{self.return_benchmark_name(benchmark_result)} ROC Curve (AUC = {roc_auc:.2f})",
+                color=self.palette_hex_codes[i],
+            )
+
+        plt.plot(linestyle="--", color="gray")
+        plt.xlabel("False Positive Rate")
+        plt.ylabel("True Positive Rate")
+        plt.title("Receiver Operating Characteristic (ROC) Curve")
+        plt.legend(loc="upper center", bbox_to_anchor=(0.5, -0.15))
+        plt.savefig(
+            f"{benchmark_generator.prioritisation_type_file_prefix}_roc_curve.svg",
+            format="svg",
+            bbox_inches="tight",
+        )
+
+    def generate_precision_recall(
+        self,
+        benchmarking_results: List[BenchmarkRunResults],
+        benchmark_generator: BenchmarkRunOutputGenerator,
+    ):
+        """
+        Generate and plot Precision-Recall curves for binary classification benchmark results.
+
+        Args:
+            benchmarking_results (List[BenchmarkRunResults]): List of benchmarking results for multiple runs.
+            benchmark_generator (BenchmarkRunOutputGenerator): Object containing benchmarking output generation details.
+        """
+        plt.figure()
+        for i, benchmark_result in enumerate(benchmarking_results):
+            precision, recall, thresh = precision_recall_curve(
+                benchmark_result.binary_classification_stats.labels,
+                benchmark_result.binary_classification_stats.scores,
+            )
+            precision_recall_auc = auc(recall, precision)
+            plt.plot(
+                recall,
+                precision,
+                label=f"{self.return_benchmark_name(benchmark_result)} Precision-Recall Curve "
+                f"(AUC = {precision_recall_auc:.2f})",
+                color=self.palette_hex_codes[i],
+            )
+
+        plt.plot(linestyle="--", color="gray")
+        plt.xlabel("Recall")
+        plt.ylabel("Precision")
+        plt.title("Precision-Recall Curve")
+        plt.legend(loc="upper center", bbox_to_anchor=(0.5, -0.15))
+        plt.savefig(
+            f"{benchmark_generator.prioritisation_type_file_prefix}_precision_recall_curve.svg",
+            format="svg",
+            bbox_inches="tight",
+        )
+
     def generate_non_cumulative_bar(
         self,
         benchmarking_results: List[BenchmarkRunResults],
@@ -418,6 +495,8 @@ def generate_plots(
         title (str, optional): Title for the generated plot. Defaults to None.
     """
     plot_generator = PlotGenerator()
+    plot_generator.generate_roc_curve(benchmarking_results, benchmark_generator)
+    plot_generator.generate_precision_recall(benchmarking_results, benchmark_generator)
     if plot_type == "bar_stacked":
         plot_generator.generate_stacked_bar_plot(benchmarking_results, benchmark_generator, title)
     elif plot_type == "bar_cumulative":

{pheval-0.3.0 → pheval-0.3.2}/src/pheval/analyse/generate_summary_outputs.py

@@ -3,6 +3,7 @@ from collections import defaultdict
 from copy import deepcopy
 from typing import List
 
+import numpy as np
 import pandas as pd
 
 from pheval.analyse.benchmark_generator import BenchmarkRunOutputGenerator
@@ -40,7 +41,19 @@ class RankComparisonGenerator:
             pd.DataFrame: DataFrame containing the calculated rank differences.
         """
         comparison_df = self._generate_dataframe()
-        comparison_df["rank_decrease"] = comparison_df.iloc[:, 3] - comparison_df.iloc[:, 2]
+        comparison_df["rank_change"] = comparison_df.iloc[:, 2] - comparison_df.iloc[:, 3]
+        comparison_df["rank_change"] = np.where(
+            (comparison_df.iloc[:, 2] == 0) & (comparison_df.iloc[:, 3] != 0),
+            "GAINED",
+            np.where(
+                (comparison_df.iloc[:, 3] == 0) & (comparison_df.iloc[:, 2] != 0),
+                "LOST",
+                comparison_df["rank_change"],
+            ),
+        )
+        comparison_df["rank_change"] = comparison_df["rank_change"].apply(
+            lambda x: int(x) if str(x).lstrip("-").isdigit() else x
+        )
         return comparison_df
 
     def generate_output(self, prefix: str, suffix: str) -> None:

{pheval-0.3.0 → pheval-0.3.2}/src/pheval/analyse/variant_prioritisation_analysis.py

@@ -10,7 +10,11 @@ from pheval.analyse.prioritisation_result_types import VariantPrioritisationResu
 from pheval.analyse.rank_stats import RankStats
 from pheval.analyse.run_data_parser import TrackInputOutputDirectories
 from pheval.post_processing.post_processing import RankedPhEvalVariantResult
-from pheval.utils.file_utils import all_files, files_with_suffix, obtain_closest_file_name
+from pheval.utils.file_utils import (
+    all_files,
+    files_with_suffix,
+    obtain_phenopacket_path_from_pheval_result,
+)
 from pheval.utils.phenopacket_utils import GenomicVariant, PhenopacketUtil, phenopacket_reader
 
 
@@ -228,7 +232,7 @@ def assess_phenopacket_variant_prioritisation(
         variant_rank_comparison (defaultdict): Default dictionary for variant rank comparisons.
         variant_binary_classification_stats (BinaryClassificationStats): BinaryClassificationStats class instance.
     """
-    phenopacket_path = obtain_closest_file_name(
+    phenopacket_path = obtain_phenopacket_path_from_pheval_result(
         standardised_variant_result, all_files(results_dir_and_input.phenopacket_dir)
     )
     proband_causative_variants = _obtain_causative_variants(phenopacket_path)

{pheval-0.3.0 → pheval-0.3.2}/src/pheval/utils/file_utils.py

@@ -1,4 +1,3 @@
-import difflib
 import itertools
 import re
 import unicodedata
@@ -71,23 +70,33 @@ def normalise_file_name(file_path: Path) -> str:
     return re.sub("[\u0300-\u036f]", "", normalised_file_name)
 
 
-def obtain_closest_file_name(file_to_be_queried: Path, file_paths: list[Path]) -> Path:
+def obtain_phenopacket_path_from_pheval_result(
+    pheval_result_path: Path, phenopacket_paths: list[Path]
+) -> Path:
     """
-    Obtains the closest file name when given a template file name
-    and a list of full paths of files to be queried.
+    Obtains the phenopacket file name when given a pheval result file name
+    and a list of full paths of phenopackets to be queried.
 
     Args:
-        file_to_be_queried (Path): The template file name to find the closest match.
-        file_paths (list[Path]): List of full paths of files to be queried.
+        pheval_result_path (Path): The PhEval result.
+        phenopacket_paths (list[Path]): List of full paths of phenopackets to be queried.
 
     Returns:
-        Path: The closest matching file path from the provided list.
+        Path: The matching phenopacket file path from the provided list.
     """
-    stems = [Path(file_path).stem for file_path in file_paths]
-    closest_file_match = difflib.get_close_matches(
-        str(Path(file_to_be_queried).stem), stems, cutoff=0.1, n=1
-    )[0]
-    return [file_path for file_path in file_paths if closest_file_match == str(file_path.stem)][0]
+    pheval_result_path_stem_stripped = pheval_result_path.stem.split("-pheval_")[0]
+    matching_phenopacket_paths = [
+        phenopacket_path
+        for phenopacket_path in phenopacket_paths
+        if phenopacket_path.stem == pheval_result_path_stem_stripped
+    ]
+    if matching_phenopacket_paths:
+        return matching_phenopacket_paths[0]
+    else:
+        raise FileNotFoundError(
+            f"Unable to find matching phenopacket file named "
+            f"{pheval_result_path_stem_stripped}.json for {pheval_result_path.name}"
+        )
 
 
 def ensure_file_exists(*files: str):