pheval 0.1.0__py3-none-any.whl → 0.2.0__py3-none-any.whl

pheval/analyse/generate_plots.py

@@ -0,0 +1,312 @@
+from dataclasses import dataclass
+from pathlib import Path
+
+import matplotlib
+import pandas as pd
+import seaborn as sns
+from matplotlib import pyplot as plt
+
+from pheval.analyse.rank_stats import RankStats
+from pheval.constants import PHEVAL_RESULTS_DIRECTORY_SUFFIX
+
+
+def trim_corpus_results_directory_suffix(corpus_results_directory: Path) -> Path:
+    """Trim the end of the corpus results directory name."""
+    return Path(str(corpus_results_directory).replace(PHEVAL_RESULTS_DIRECTORY_SUFFIX, ""))
+
+
+@dataclass
+class TrackGenePrioritisation:
+    """Track gene prioritisation for a run."""
+
+    results_dir: Path
+    ranks: dict
+    rank_stats: RankStats
+
+
+@dataclass
+class TrackVariantPrioritisation:
+    """Track variant prioritisation for a run."""
+
+    results_dir: Path
+    ranks: dict
+    rank_stats: RankStats
+
+
+@dataclass
+class TrackPrioritisation:
+    """Track prioritisation for a run."""
+
+    gene_prioritisation: TrackGenePrioritisation
+    variant_prioritisation: TrackVariantPrioritisation
+
+
+class PlotGenerator:
+    def __init__(self, gene_analysis: bool):
+        self.gene_analysis = gene_analysis
+        self.stats, self.mrr = [], []
+        matplotlib.rcParams["axes.spines.right"] = False
+        matplotlib.rcParams["axes.spines.top"] = False
+
+    def _retrieve_prioritisation_data(self, prioritisation_result: TrackPrioritisation):
+        """Return either gene prioritisation or variant prioritisation stats."""
+        return (
+            prioritisation_result.gene_prioritisation
+            if self.gene_analysis
+            else prioritisation_result.variant_prioritisation
+        )
+
+    def _generate_stacked_bar_plot_data(self, prioritisation_result: TrackPrioritisation) -> None:
+        """Generate data in correct format for dataframe creation for stacked bar plot."""
+        result = self._retrieve_prioritisation_data(prioritisation_result)
+        rank_stats = result.rank_stats
+        self.stats.append(
+            {
+                "Run": f"{result.results_dir.parents[0].name}_"
+                f"{trim_corpus_results_directory_suffix(result.results_dir.name)}",
+                "Top": result.rank_stats.percentage_top(),
+                "2-3": rank_stats.percentage_difference(
+                    rank_stats.percentage_top3(), rank_stats.percentage_top()
+                ),
+                "4-5": rank_stats.percentage_difference(
+                    rank_stats.percentage_top5(), rank_stats.percentage_top3()
+                ),
+                "6-10": rank_stats.percentage_difference(
+                    rank_stats.percentage_top10(), rank_stats.percentage_top5()
+                ),
+                ">10": rank_stats.percentage_difference(
+                    rank_stats.percentage_found(), rank_stats.percentage_top10()
+                ),
+                "FO/NP": rank_stats.percentage_difference(100, rank_stats.percentage_found()),
+            }
+        )
+
+    def _generate_stats_mrr_bar_plot_data(self, prioritisation_result: TrackPrioritisation) -> None:
+        """Generate data in correct format for dataframe creation for MRR bar plot."""
+        result = self._retrieve_prioritisation_data(prioritisation_result)
+        self.mrr.extend(
+            [
+                {
+                    "Rank": "MRR",
+                    "Percentage": result.rank_stats.mean_reciprocal_rank(),
+                    "Run": f"{result.results_dir.parents[0].name}_"
+                    f"{trim_corpus_results_directory_suffix(result.results_dir.name)}",
+                }
+            ]
+        )
+
+    def generate_stacked_bar_gene(self, prioritisation_data: [TrackPrioritisation]) -> None:
+        """Generate stacked bar plot and MRR bar plot for gene prioritisation stats."""
+        for prioritisation_result in prioritisation_data:
+            self._generate_stacked_bar_plot_data(prioritisation_result)
+            self._generate_stats_mrr_bar_plot_data(prioritisation_result)
+        gene_prioritisation_stats_df = pd.DataFrame(self.stats)
+        gene_prioritisation_stats_df.set_index("Run").plot(
+            kind="bar",
+            stacked=True,
+            colormap="tab10",
+            ylabel="Disease-causing genes (%)",
+            figsize=(10, 8),
+            # rot=45,
+        ).legend(loc="center left", bbox_to_anchor=(1.0, 0.5))
+        plt.savefig("gene_rank_stats.svg", format="svg", bbox_inches="tight")
+        gene_mrr_df = pd.DataFrame(self.mrr)
+        gene_mrr_df.set_index("Run").plot(
+            kind="bar",
+            colormap="tab10",
+            ylabel="Gene prioritisation mean reciprocal rank",
+            legend=False,
+        )
+        plt.savefig("gene_mrr.svg", format="svg", bbox_inches="tight")
+
+    def generate_stacked_bar_variant(self, prioritisation_data: [TrackPrioritisation]):
+        """Generate stacked bar plot and MRR bar plot for variant prioritisation stats."""
+        for prioritisation_result in prioritisation_data:
+            self._generate_stacked_bar_plot_data(prioritisation_result)
+            self._generate_stats_mrr_bar_plot_data(prioritisation_result)
+        variant_prioritisation_stats_df = pd.DataFrame(self.stats)
+
+        variant_prioritisation_stats_df.set_index("Run").plot(
+            kind="bar", stacked=True, colormap="tab10", ylabel="Disease-causing variants (%)"
+        ).legend(loc="center left", bbox_to_anchor=(1.0, 0.5))
+        plt.savefig("variant_rank_stats.svg", format="svg", bbox_inches="tight")
+        gene_mrr_df = pd.DataFrame(self.mrr)
+        gene_mrr_df.set_index("Run").plot(
+            kind="bar",
+            colormap="tab10",
+            ylabel="Variant prioritisation mean reciprocal rank",
+            legend=False,
+        )
+        plt.savefig("variant_mrr.svg", format="svg", bbox_inches="tight")
+
+    def _generate_cumulative_bar_plot_data(self, prioritisation_result: TrackPrioritisation):
+        """Generate data in correct format for dataframe creation for cumulative bar plot."""
+        result = self._retrieve_prioritisation_data(prioritisation_result)
+        rank_stats = result.rank_stats
+        trimmed_corpus_results_dir = trim_corpus_results_directory_suffix(result.results_dir.name)
+        self.stats.extend(
+            [
+                {
+                    "Rank": "Top",
+                    "Percentage": rank_stats.percentage_top() / 100,
+                    "Run": f"{result.results_dir.parents[0].name}_" f"{trimmed_corpus_results_dir}",
+                },
+                {
+                    "Rank": "Top3",
+                    "Percentage": rank_stats.percentage_top3() / 100,
+                    "Run": f"{result.results_dir.parents[0].name}_" f"{trimmed_corpus_results_dir}",
+                },
+                {
+                    "Rank": "Top5",
+                    "Percentage": rank_stats.percentage_top5() / 100,
+                    "Run": f"{result.results_dir.parents[0].name}_" f"{trimmed_corpus_results_dir}",
+                },
+                {
+                    "Rank": "Top10",
+                    "Percentage": rank_stats.percentage_top10() / 100,
+                    "Run": f"{result.results_dir.parents[0].name}_" f"{trimmed_corpus_results_dir}",
+                },
+                {
+                    "Rank": "Found",
+                    "Percentage": rank_stats.percentage_found() / 100,
+                    "Run": f"{result.results_dir.parents[0].name}_" f"{trimmed_corpus_results_dir}",
+                },
+                {
+                    "Rank": "FO/NP",
+                    "Percentage": rank_stats.percentage_difference(
+                        100, rank_stats.percentage_found()
+                    )
+                    / 100,
+                    "Run": f"{result.results_dir.parents[0].name}_" f"{trimmed_corpus_results_dir}",
+                },
+                {
+                    "Rank": "MRR",
+                    "Percentage": rank_stats.mean_reciprocal_rank(),
+                    "Run": f"{result.results_dir.parents[0].name}_" f"{trimmed_corpus_results_dir}",
+                },
+            ]
+        )
+
+    def generate_cumulative_bar_gene(self, prioritisation_data: [TrackPrioritisation]):
+        """Generate cumulative bar plot for gene prioritisation stats."""
+        for prioritisation_result in prioritisation_data:
+            self._generate_cumulative_bar_plot_data(prioritisation_result)
+        gene_prioritisation_df = pd.DataFrame(self.stats)
+        sns.catplot(
+            data=gene_prioritisation_df, kind="bar", x="Rank", y="Percentage", hue="Run"
+        ).set(xlabel="Rank", ylabel="Disease-causing genes (%)")
+        plt.savefig("gene_rank_stats.svg", format="svg", bbox_inches="tight")
+
+    def generate_cumulative_bar_variant(self, prioritisation_data: [TrackPrioritisation]):
+        """Generate cumulative bar plot for variant prioritisation stats."""
+        for prioritisation_result in prioritisation_data:
+            self._generate_cumulative_bar_plot_data(prioritisation_result)
+        variant_prioritisation_df = pd.DataFrame(self.stats)
+        sns.catplot(
+            data=variant_prioritisation_df, kind="bar", x="Rank", y="Percentage", hue="Run"
+        ).set(xlabel="Rank", ylabel="Disease-causing variants (%)")
+        plt.savefig("variant_rank_stats.svg", format="svg", bbox_inches="tight")
+
+    def _generate_non_cumulative_bar_plot_data(
+        self, prioritisation_result: TrackPrioritisation
+    ) -> [dict]:
+        """Generate data in correct format for dataframe creation for non-cumulative bar plot."""
+        result = self._retrieve_prioritisation_data(prioritisation_result)
+        rank_stats = result.rank_stats
+        trimmed_corpus_results_dir = trim_corpus_results_directory_suffix(result.results_dir.name)
+        self.stats.extend(
+            [
+                {
+                    "Rank": "Top",
+                    "Percentage": rank_stats.percentage_top() / 100,
+                    "Run": f"{result.results_dir.parents[0].name}_" f"{trimmed_corpus_results_dir}",
+                },
+                {
+                    "Rank": "2-3",
+                    "Percentage": rank_stats.percentage_difference(
+                        rank_stats.percentage_top3(), rank_stats.percentage_top()
+                    )
+                    / 100,
+                    "Run": f"{result.results_dir.parents[0].name}_" f"{trimmed_corpus_results_dir}",
+                },
+                {
+                    "Rank": "4-5",
+                    "Percentage": rank_stats.percentage_difference(
+                        rank_stats.percentage_top5(), rank_stats.percentage_top3()
+                    )
+                    / 100,
+                    "Run": f"{result.results_dir.parents[0].name}_" f"{trimmed_corpus_results_dir}",
+                },
+                {
+                    "Rank": "6-10",
+                    "Percentage": rank_stats.percentage_difference(
+                        rank_stats.percentage_top10(), rank_stats.percentage_top5()
+                    )
+                    / 100,
+                    "Run": f"{result.results_dir.parents[0].name}_" f"{trimmed_corpus_results_dir}",
+                },
+                {
+                    "Rank": ">10",
+                    "Percentage": rank_stats.percentage_difference(
+                        rank_stats.percentage_found(), rank_stats.percentage_top10()
+                    )
+                    / 100,
+                    "Run": f"{result.results_dir.parents[0].name}_" f"{trimmed_corpus_results_dir}",
+                },
+                {
+                    "Rank": "FO/NP",
+                    "Percentage": rank_stats.percentage_difference(
+                        100, rank_stats.percentage_found()
+                    )
+                    / 100,
+                    "Run": f"{result.results_dir.parents[0].name}_" f"{trimmed_corpus_results_dir}",
+                },
+                {
+                    "Rank": "MRR",
+                    "Percentage": rank_stats.mean_reciprocal_rank(),
+                    "Run": f"{result.results_dir.parents[0].name}_" f"{trimmed_corpus_results_dir}",
+                },
+            ]
+        )
+
+    def generate_non_cumulative_bar_gene(self, prioritisation_data: [TrackPrioritisation]):
+        """Generate non-cumulative bar plot for gene prioritisation stats."""
+        for prioritisation_result in prioritisation_data:
+            self._generate_non_cumulative_bar_plot_data(prioritisation_result)
+        gene_prioritisation_df = pd.DataFrame(self.stats)
+        sns.catplot(
+            data=gene_prioritisation_df, kind="bar", x="Rank", y="Percentage", hue="Run"
+        ).set(xlabel="Rank", ylabel="Disease-causing genes (%)")
+        plt.savefig("gene_rank_stats.svg", format="svg", bbox_inches="tight")
+
+    def generate_non_cumulative_bar_variant(self, prioritisation_data: [TrackPrioritisation]):
+        """Generate non-cumulative bar plot for variant prioritisation stats."""
+        for prioritisation_result in prioritisation_data:
+            self._generate_non_cumulative_bar_plot_data(prioritisation_result)
+        variant_prioritisation_df = pd.DataFrame(self.stats)
+        sns.catplot(
+            data=variant_prioritisation_df, kind="bar", x="Rank", y="Percentage", hue="Run"
+        ).set(xlabel="Rank", ylabel="Disease-causing variants (%)")
+        plt.savefig("variant_rank_stats.svg", format="svg", bbox_inches="tight")
+
+
+def generate_gene_plots(prioritisation_data: [TrackPrioritisation], plot_type: str) -> None:
+    """Generate summary stats bar plot for gene prioritisation."""
+    plot_generator = PlotGenerator(gene_analysis=True)
+    if plot_type == "bar_stacked":
+        plot_generator.generate_stacked_bar_gene(prioritisation_data)
+    elif plot_type == "bar_cumulative":
+        plot_generator.generate_cumulative_bar_gene(prioritisation_data)
+    elif plot_type == "bar_non_cumulative":
+        plot_generator.generate_non_cumulative_bar_gene(prioritisation_data)
+
+
+def generate_variant_plots(prioritisation_data: [TrackPrioritisation], plot_type: str) -> None:
+    """Generate summary stats bar plot for variant prioritisation."""
+    plot_generator = PlotGenerator(gene_analysis=False)
+    if plot_type == "bar_stacked":
+        plot_generator.generate_stacked_bar_variant(prioritisation_data)
+    elif plot_type == "bar_cumulative":
+        plot_generator.generate_cumulative_bar_variant(prioritisation_data)
+    elif plot_type == "bar_non_cumulative":
+        plot_generator.generate_non_cumulative_bar_variant(prioritisation_data)

pheval/analyse/generate_summary_outputs.py

@@ -0,0 +1,186 @@
+import csv
+import itertools
+from collections import defaultdict
+from copy import deepcopy
+from pathlib import Path
+
+import pandas as pd
+
+from pheval.analyse.generate_plots import (
+    TrackPrioritisation,
+    generate_gene_plots,
+    generate_variant_plots,
+)
+from pheval.analyse.rank_stats import RankStats
+
+
+class RankComparisonGenerator:
+    """Write the run comparison of rank assignment for prioritisation."""
+
+    def __init__(self, run_comparison: defaultdict):
+        self.run_comparison = run_comparison
+
+    def _generate_dataframe(self) -> pd.DataFrame:
+        """Generate pandas dataframe."""
+        return pd.DataFrame.from_dict(self.run_comparison, orient="index")
+
+    def _calculate_rank_difference(self) -> pd.DataFrame:
+        """Calculate the rank decrease for runs - taking the first directory as a baseline."""
+        comparison_df = self._generate_dataframe()
+        print(len(comparison_df.columns))
+        comparison_df["rank_decrease"] = comparison_df.iloc[:, 3] - comparison_df.iloc[:, 2]
+        return comparison_df
+
+    def generate_gene_output(self, prefix: str) -> None:
+        """Generate the output for gene prioritisation ranks."""
+        self._generate_dataframe().to_csv(prefix + "-gene_rank_comparison.tsv", sep="\t")
+
+    def generate_variant_output(self, prefix: str) -> None:
+        """Generate the output for variant prioritisation ranks."""
+        self._generate_dataframe().to_csv(prefix + "-variant_rank_comparison.tsv", sep="\t")
+
+    def generate_gene_comparison_output(self, prefix: str) -> None:
+        """Generate the output for gene prioritisation rank comparison."""
+        self._calculate_rank_difference().to_csv(prefix + "-gene_rank_comparison.tsv", sep="\t")
+
+    def generate_variant_comparison_output(self, prefix: str) -> None:
+        """Generate the output for variant prioritisation rank comparison."""
+        self._calculate_rank_difference().to_csv(prefix + "-variant_rank_comparison.tsv", sep="\t")
+
+
+class RankStatsWriter:
+    """Write the rank stats for each run."""
+
+    def __init__(self, file: Path):
+        self.file = open(file, "w")
+        self.writer = csv.writer(self.file, delimiter="\t")
+        self.writer.writerow(
+            [
+                "results_directory_path",
+                "top",
+                "top3",
+                "top5",
+                "top10",
+                "found",
+                "total",
+                "mean_reciprocal_rank",
+                "percentage_top",
+                "percentage_top3",
+                "percentage_top5",
+                "percentage_top10",
+                "percentage_found",
+            ]
+        )
+
+    def write_row(self, directory: Path, rank_stats: RankStats) -> None:
+        """Write summary rank stats row for run."""
+        try:
+            self.writer.writerow(
+                [
+                    directory,
+                    rank_stats.top,
+                    rank_stats.top3,
+                    rank_stats.top5,
+                    rank_stats.top10,
+                    rank_stats.found,
+                    rank_stats.total,
+                    rank_stats.mean_reciprocal_rank(),
+                    rank_stats.percentage_top(),
+                    rank_stats.percentage_top3(),
+                    rank_stats.percentage_top5(),
+                    rank_stats.percentage_top10(),
+                    rank_stats.percentage_found(),
+                ]
+            )
+        except IOError:
+            print("Error writing ", self.file)
+
+    def close(self) -> None:
+        """Close file."""
+        try:
+            self.file.close()
+        except IOError:
+            print("Error closing ", self.file)
+
+
+def generate_benchmark_gene_output(
+    prioritisation_data: TrackPrioritisation, plot_type: str
+) -> None:
+    """Generate gene prioritisation outputs for benchmarking single run."""
+    RankComparisonGenerator(prioritisation_data.gene_prioritisation.ranks).generate_gene_output(
+        f"{prioritisation_data.gene_prioritisation.results_dir.name}"
+    )
+    generate_gene_plots([prioritisation_data], plot_type)
+
+
+def generate_benchmark_variant_output(
+    prioritisation_data: TrackPrioritisation, plot_type: str
+) -> None:
+    """Generate variant prioritisation outputs for benchmarking single run."""
+    RankComparisonGenerator(
+        prioritisation_data.variant_prioritisation.ranks
+    ).generate_variant_output(f"{prioritisation_data.gene_prioritisation.results_dir.name}")
+    generate_variant_plots([prioritisation_data], plot_type)
+
+
+def merge_results(result1: dict, result2: dict) -> dict:
+    """Merge two nested dictionaries containing results on commonalities."""
+    for key, val in result1.items():
+        if type(val) == dict:
+            if key in result2 and type(result2[key] == dict):
+                merge_results(result1[key], result2[key])
+        else:
+            if key in result2:
+                result1[key] = result2[key]
+
+    for key, val in result2.items():
+        if key not in result1:
+            result1[key] = val
+    return result1
+
+
+def generate_gene_rank_comparisons(comparison_ranks: [tuple]) -> None:
+    """Generate the gene rank comparison of two result directories."""
+    for pair in comparison_ranks:
+        merged_results = merge_results(
+            deepcopy(pair[0].gene_prioritisation.ranks), deepcopy(pair[1].gene_prioritisation.ranks)
+        )
+        RankComparisonGenerator(merged_results).generate_gene_comparison_output(
+            f"{pair[0].gene_prioritisation.results_dir.parents[0].name}_"
+            f"{pair[0].gene_prioritisation.results_dir.name}"
+            f"__v__{pair[1].gene_prioritisation.results_dir.parents[0].name}_"
+            f"{pair[1].gene_prioritisation.results_dir.name}"
+        )
+
+
+def generate_variant_rank_comparisons(comparison_ranks: [tuple]) -> None:
+    """Generate the variant rank comparison of two result directories."""
+    for pair in comparison_ranks:
+        merged_results = merge_results(
+            deepcopy(pair[0].variant_prioritisation.ranks),
+            deepcopy(pair[1].variant_prioritisation.ranks),
+        )
+        RankComparisonGenerator(merged_results).generate_variant_comparison_output(
+            f"{pair[0].gene_prioritisation.results_dir.parents[0].name}_"
+            f"{pair[0].variant_prioritisation.results_dir.name}"
+            f"__v__{pair[0].gene_prioritisation.results_dir.parents[0].name}_"
+            f"{pair[1].variant_prioritisation.results_dir.name}"
+        )
+
+
+def generate_benchmark_comparison_gene_output(
+    prioritisation_stats_for_runs: [TrackPrioritisation], plot_type: str
+) -> None:
+    """Generate gene prioritisation outputs for benchmarking multiple runs."""
+    generate_gene_rank_comparisons(list(itertools.combinations(prioritisation_stats_for_runs, 2)))
+    generate_gene_plots(prioritisation_stats_for_runs, plot_type)
+
+
+def generate_benchmark_comparison_variant_output(
+    prioritisation_stats_for_runs: [TrackPrioritisation], plot_type: str
+) -> None:
+    """Generate variant prioritisation outputs for benchmarking multiple runs."""
+    generate_variant_rank_comparisons(
+        list(itertools.combinations(prioritisation_stats_for_runs, 2))
+    )
+    generate_variant_plots(prioritisation_stats_for_runs, plot_type)

pheval/analyse/rank_stats.py

@@ -0,0 +1,61 @@
+from dataclasses import dataclass, field
+from statistics import mean
+
+
+@dataclass
+class RankStats:
+    """Class for keeping track of the rank stats."""
+
+    top: int = 0
+    top3: int = 0
+    top5: int = 0
+    top10: int = 0
+    found: int = 0
+    total: int = 0
+    reciprocal_ranks: list = field(default_factory=list)
+
+    def add_rank(self, rank: int) -> None:
+        """Add rank for phenopacket."""
+        self.reciprocal_ranks.append(1 / rank)
+        self.found += 1
+        if rank == 1:
+            self.top += 1
+        if rank != "" and rank <= 3:
+            self.top3 += 1
+        if rank != "" and rank <= 5:
+            self.top5 += 1
+        if rank != "" and rank <= 10:
+            self.top10 += 1
+
+    def percentage_rank(self, value: int) -> float:
+        """Return a percentage rank."""
+        return 100 * value / self.found
+
+    def percentage_top(self) -> float:
+        """Return percentage of top matches."""
+        return self.percentage_rank(self.top)
+
+    def percentage_top3(self) -> float:
+        """Return percentage of matches in the top3."""
+        return self.percentage_rank(self.top3)
+
+    def percentage_top5(self) -> float:
+        """Return percentage of matches in the top5."""
+        return self.percentage_rank(self.top5)
+
+    def percentage_top10(self) -> float:
+        """Return percentage of matches in the top10."""
+        return self.percentage_rank(self.top10)
+
+    def percentage_found(self) -> float:
+        """Return percentage of matches found."""
+        return 100 * self.found / self.total
+
+    @staticmethod
+    def percentage_difference(percentage_value_1: float, percentage_value_2: float) -> float:
+        """Return percentage difference between two percentage values"""
+        return percentage_value_1 - percentage_value_2
+
+    def mean_reciprocal_rank(self) -> float:
+        """Return the mean reciprocal rank."""
+        return mean(self.reciprocal_ranks)

pheval/cli.py

@@ -3,8 +3,16 @@ import logging
 
 import click
 
+from pheval.analyse.analysis import benchmark, benchmark_comparison
+
 from .cli_pheval import run
-from .cli_pheval_utils import scramble_phenopacket, scramble_semsim
+from .cli_pheval_utils import (
+    create_spiked_vcfs_command,
+    scramble_phenopackets_command,
+    scramble_semsim,
+    semsim_comparison,
+    update_phenopackets_command,
+)
 
 info_log = logging.getLogger("info")
 
@@ -13,11 +21,11 @@ info_log = logging.getLogger("info")
 @click.option("-v", "--verbose", count=True)
 @click.option("-q", "--quiet")
 def main(verbose=1, quiet=False) -> None:
-    """main CLI method for pheval
+    """main CLI method for PhEval
 
     Args:
-        verbose (int): _description_
-        quiet (bool): _description_
+        verbose (int, optional): Verbose flag.
+        quiet (bool, optional): Queit Flag.
     """
     if verbose >= 2:
         info_log.setLevel(level=logging.DEBUG)
@@ -34,15 +42,22 @@ def pheval():
     """pheval"""
 
 
+pheval.add_command(run)
+
+
 @click.group()
 def pheval_utils():
     """pheval_utils"""
 
 
-pheval.add_command(run)
-
 pheval_utils.add_command(scramble_semsim)
-pheval_utils.add_command(scramble_phenopacket)
+pheval_utils.add_command(semsim_comparison)
+pheval_utils.add_command(scramble_phenopackets_command)
+pheval_utils.add_command(update_phenopackets_command)
+pheval_utils.add_command(create_spiked_vcfs_command)
+pheval_utils.add_command(benchmark)
+pheval_utils.add_command(benchmark_comparison)
+
 
 if __name__ == "__main__":
     main()