pheval 0.3.9__py3-none-any.whl → 0.4.1__py3-none-any.whl

pheval/cli_pheval_utils.py

@@ -5,13 +5,9 @@ from typing import List
 
 import click
 
-from pheval.analyse.analysis import (
-    TrackInputOutputDirectories,
-    benchmark_directory,
-    benchmark_run_comparisons,
-)
-from pheval.analyse.generate_plots import generate_plots_from_benchmark_summary_tsv
-from pheval.analyse.run_data_parser import parse_run_data_text_file
+from pheval.analyse.analysis import benchmark_run_comparisons
+from pheval.analyse.generate_plots import generate_plots_from_benchmark_summary_db
+from pheval.analyse.run_data_parser import parse_run_config
 from pheval.prepare.create_noisy_phenopackets import scramble_phenopackets
 from pheval.prepare.create_spiked_vcf import spike_vcfs
 from pheval.prepare.custom_exceptions import InputError, MutuallyExclusiveOptionError
@@ -110,17 +106,29 @@ def semsim_scramble_command(
     default="noisy_phenopackets",
     type=Path,
 )
+@click.option(
+    "--local-ontology-cache",
+    "-l",
+    metavar="PATH",
+    required=False,
+    help="Path to the local ontology cache, e.g., path to the hp.obo.",
+    default=None,
+    type=Path,
+)
 def scramble_phenopackets_command(
     phenopacket_path: Path,
     phenopacket_dir: Path,
     scramble_factor: float,
     output_dir: Path,
+    local_ontology_cache: Path,
 ):
     """Generate noisy phenopackets from existing ones."""
     if phenopacket_path is None and phenopacket_dir is None:
         raise InputError("Either a phenopacket or phenopacket directory must be specified")
     else:
-        scramble_phenopackets(output_dir, phenopacket_path, phenopacket_dir, scramble_factor)
+        scramble_phenopackets(
+            output_dir, phenopacket_path, phenopacket_dir, scramble_factor, local_ontology_cache
+        )
 
 
 @click.command("semsim-comparison")
@@ -338,196 +346,19 @@ def create_spiked_vcfs_command(
 
 @click.command()
 @click.option(
-    "--directory",
-    "-d",
-    required=True,
-    metavar="PATH",
-    help="General results directory to be benchmarked, assumes contains subdirectories of pheval_gene_results/,"
-    "pheval_variant_results/ or pheval_disease_results/. ",
-    type=Path,
-)
-@click.option(
-    "--phenopacket-dir",
-    "-p",
-    required=True,
-    metavar="PATH",
-    help="Full path to directory containing input phenopackets.",
-    type=Path,
-)
-@click.option(
-    "--output-prefix",
-    "-o",
-    metavar="<str>",
-    required=True,
-    help=" Output file prefix. ",
-)
-@click.option(
-    "--score-order",
-    "-so",
-    required=True,
-    help="Ordering of results for ranking.",
-    type=click.Choice(["ascending", "descending"]),
-    default="descending",
-    show_default=True,
-)
-@click.option(
-    "--threshold",
-    "-t",
-    metavar="<float>",
-    default=float(0.0),
-    required=False,
-    help="Score threshold.",
-    type=float,
-)
-@click.option(
-    "--gene-analysis/--no-gene-analysis",
-    default=False,
-    required=False,
-    type=bool,
-    show_default=True,
-    help="Specify analysis for gene prioritisation",
-)
-@click.option(
-    "--variant-analysis/--no-variant-analysis",
-    default=False,
-    required=False,
-    type=bool,
-    show_default=True,
-    help="Specify analysis for variant prioritisation",
-)
-@click.option(
-    "--disease-analysis/--no-disease-analysis",
-    default=False,
-    required=False,
-    type=bool,
-    show_default=True,
-    help="Specify analysis for disease prioritisation",
-)
-@click.option(
-    "--plot-type",
-    "-y",
-    default="bar_stacked",
-    show_default=True,
-    type=click.Choice(["bar_stacked", "bar_cumulative", "bar_non_cumulative"]),
-    help="Bar chart type to output.",
-)
-def benchmark(
-    directory: Path,
-    phenopacket_dir: Path,
-    score_order: str,
-    output_prefix: str,
-    threshold: float,
-    gene_analysis: bool,
-    variant_analysis: bool,
-    disease_analysis: bool,
-    plot_type: str,
-):
-    """Benchmark the gene/variant/disease prioritisation performance for a single run."""
-    if not gene_analysis and not variant_analysis and not disease_analysis:
-        raise InputError("Need to specify at least one of gene/variant/disease analysis.")
-    benchmark_directory(
-        TrackInputOutputDirectories(results_dir=directory, phenopacket_dir=phenopacket_dir),
-        score_order,
-        output_prefix,
-        threshold,
-        gene_analysis,
-        variant_analysis,
-        disease_analysis,
-        plot_type,
-    )
-
-
-@click.command()
-@click.option(
-    "--run-data",
+    "--run-yaml",
     "-r",
     required=True,
     metavar="PATH",
-    help="Path to .txt file containing testdata phenopacket directory "
-    "and corresponding results directory separated by tab."
-    "Each run contained to a new line with the input testdata listed first and on the same line separated by a tab"
-    "the results directory.",
+    help="Path to yaml configuration file for benchmarking.",
     type=Path,
 )
-@click.option(
-    "--output-prefix",
-    "-o",
-    metavar="<str>",
-    required=True,
-    help=" Output file prefix. ",
-)
-@click.option(
-    "--score-order",
-    "-so",
-    required=True,
-    help="Ordering of results for ranking.",
-    type=click.Choice(["ascending", "descending"]),
-    default="descending",
-    show_default=True,
-)
-@click.option(
-    "--threshold",
-    "-t",
-    metavar="<float>",
-    default=float(0.0),
-    required=False,
-    help="Score threshold.",
-    type=float,
-)
-@click.option(
-    "--gene-analysis/--no-gene-analysis",
-    default=False,
-    required=False,
-    type=bool,
-    show_default=True,
-    help="Specify analysis for gene prioritisation",
-)
-@click.option(
-    "--variant-analysis/--no-variant-analysis",
-    default=False,
-    required=False,
-    type=bool,
-    show_default=True,
-    help="Specify analysis for variant prioritisation",
-)
-@click.option(
-    "--disease-analysis/--no-disease-analysis",
-    default=False,
-    required=False,
-    type=bool,
-    show_default=True,
-    help="Specify analysis for disease prioritisation",
-)
-@click.option(
-    "--plot-type",
-    "-y",
-    default="bar_cumulative",
-    show_default=True,
-    type=click.Choice(["bar_stacked", "bar_cumulative", "bar_non_cumulative"]),
-    help="Bar chart type to output.",
-)
-def benchmark_comparison(
-    run_data: Path,
-    score_order: str,
-    output_prefix: str,
-    threshold: float,
-    gene_analysis: bool,
-    variant_analysis: bool,
-    disease_analysis: bool,
-    plot_type: str,
+def generate_benchmark_stats(
+    run_yaml: Path,
 ):
-    """Benchmark the gene/variant/disease prioritisation performance for two runs."""
-    if not gene_analysis and not variant_analysis and not disease_analysis:
-        raise InputError("Need to specify at least one of gene/variant/disease analysis.")
+    """Benchmark the gene/variant/disease prioritisation performance for runs."""
     benchmark_run_comparisons(
-        parse_run_data_text_file(run_data),
-        score_order,
-        output_prefix,
-        threshold,
-        gene_analysis,
-        variant_analysis,
-        disease_analysis,
-        plot_type,
+        parse_run_config(run_yaml),
     )
 
 
@@ -580,69 +411,27 @@ def semsim_to_exomiserdb_command(
 
 @click.command()
 @click.option(
-    "--benchmarking-tsv",
+    "--benchmark-db",
     "-b",
     required=True,
     metavar="PATH",
-    help="Path to benchmark summary tsv output by PhEval benchmark commands.",
+    help="Path to benchmark db output by PhEval benchmark commands.",
     type=Path,
 )
 @click.option(
-    "--gene-analysis/--no-gene-analysis",
-    default=False,
-    required=False,
-    type=bool,
-    show_default=True,
-    help="Specify analysis for gene prioritisation",
-    cls=MutuallyExclusiveOptionError,
-    mutually_exclusive=["variant_analysis", "disease_analysis"],
-)
-@click.option(
-    "--variant-analysis/--no-variant-analysis",
-    default=False,
-    required=False,
-    type=bool,
-    show_default=True,
-    help="Specify analysis for variant prioritisation",
-    cls=MutuallyExclusiveOptionError,
-    mutually_exclusive=["gene_analysis", "disease_analysis"],
-)
-@click.option(
-    "--disease-analysis/--no-disease-analysis",
-    default=False,
-    required=False,
-    type=bool,
-    show_default=True,
-    help="Specify analysis for disease prioritisation",
-    cls=MutuallyExclusiveOptionError,
-    mutually_exclusive=["gene_analysis", "variant_analysis"],
-)
-@click.option(
-    "--plot-type",
-    "-y",
-    default="bar_cumulative",
-    show_default=True,
-    type=click.Choice(["bar_stacked", "bar_cumulative", "bar_non_cumulative"]),
-    help="Bar chart type to output.",
-)
-@click.option(
-    "--title",
-    "-t",
-    type=str,
-    help='Title for plot, specify the title on the CLI enclosed with ""',
+    "--run-data",
+    "-r",
+    required=True,
+    metavar="PATH",
+    help="Path to yaml configuration file for benchmarking.",
+    type=Path,
 )
 def generate_stats_plot(
-    benchmarking_tsv: Path,
-    gene_analysis: bool,
-    variant_analysis: bool,
-    disease_analysis: bool,
-    plot_type: str,
-    title: str = None,
+    benchmark_db: Path,
+    run_data: Path,
 ):
-    """Generate bar plot from benchmark stats summary tsv."""
-    generate_plots_from_benchmark_summary_tsv(
-        benchmarking_tsv, gene_analysis, variant_analysis, disease_analysis, plot_type, title
-    )
+    """Generate bar plot from benchmark db."""
+    generate_plots_from_benchmark_summary_db(benchmark_db, run_data)
 
 
 @click.command("prepare-corpus")

pheval/prepare/create_noisy_phenopackets.py

@@ -15,15 +15,20 @@ from pheval.utils.phenopacket_utils import (
 )
 
 
-def load_ontology():
+def load_ontology(local_cached_ontology: Path = None) -> ProntoImplementation:
     """
     Load the Human Phenotype Ontology (HPO).
-
+    Args:
+        local_cached_ontology(Path): Path to the local cached ontology.
     Returns:
         ProntoImplementation: An instance of ProntoImplementation containing the loaded HPO.
     """
-    resource = OntologyResource(slug="hp.obo", local=False)
-    return ProntoImplementation(resource)
+    if local_cached_ontology is None:
+        resource = OntologyResource(slug="hp.obo", local=False)
+        return ProntoImplementation(resource)
+    else:
+        resource = OntologyResource(slug=local_cached_ontology, local=True)
+        return ProntoImplementation(resource)
 
 
 class HpoRandomiser:
@@ -181,78 +186,77 @@ class HpoRandomiser:
             + self.create_random_hpo_terms(number_of_scrambled_terms)
         )
 
+    def add_noise_to_phenotypic_profile(
+        self,
+        phenopacket: Union[Phenopacket, Family],
+    ) -> Union[Phenopacket, Family]:
+        """
+        Randomise the phenotypic profile of a Phenopacket or Family.
 
-def add_noise_to_phenotypic_profile(
-    hpo_randomiser: HpoRandomiser,
-    phenopacket: Union[Phenopacket, Family],
-) -> Union[Phenopacket, Family]:
-    """
-    Randomise the phenotypic profile of a Phenopacket or Family.
-
-    Args:
-        hpo_randomiser (HpoRandomiser): An instance of HpoRandomiser used for randomisation.
-        phenopacket (Union[Phenopacket, Family]): The Phenopacket or Family to be randomised.
-
-    Returns:
-        Union[Phenopacket, Family]: The randomised Phenopacket or Family.
-    """
-    phenotypic_features = PhenopacketUtil(phenopacket).observed_phenotypic_features()
-    random_phenotypes = hpo_randomiser.randomise_hpo_terms(phenotypic_features)
-    randomised_phenopacket = PhenopacketRebuilder(phenopacket).add_randomised_hpo(random_phenotypes)
-    return randomised_phenopacket
-
+        Args:
+            phenopacket (Union[Phenopacket, Family]): The Phenopacket or Family to be randomised.
 
-def create_scrambled_phenopacket(
-    output_dir: Path, phenopacket_path: Path, scramble_factor: float
-) -> None:
-    """
-    Create a scrambled version of a Phenopacket.
+        Returns:
+            Union[Phenopacket, Family]: The randomised Phenopacket or Family.
+        """
+        phenotypic_features = PhenopacketUtil(phenopacket).observed_phenotypic_features()
+        random_phenotypes = self.randomise_hpo_terms(phenotypic_features)
+        randomised_phenopacket = PhenopacketRebuilder(phenopacket).add_randomised_hpo(
+            random_phenotypes
+        )
+        return randomised_phenopacket
 
-    Args:
-        output_dir (Path): The directory to store the output scrambled Phenopacket.
-        phenopacket_path (Path): The path to the original Phenopacket file.
-        scramble_factor (float): A factor determining the level of scrambling for phenotypic features.
-    """
-    ontology = load_ontology()
-    hpo_randomiser = HpoRandomiser(ontology, scramble_factor)
-    phenopacket = phenopacket_reader(phenopacket_path)
-    created_noisy_phenopacket = add_noise_to_phenotypic_profile(
-        hpo_randomiser,
-        phenopacket,
-    )
-    write_phenopacket(
-        created_noisy_phenopacket,
-        output_dir.joinpath(phenopacket_path.name),
-    )
-
-
-def create_scrambled_phenopackets(
-    output_dir: Path, phenopacket_dir: Path, scramble_factor: float
-) -> None:
-    """
-    Create scrambled versions of Phenopackets within a directory.
+    def create_scrambled_phenopacket(
+        self,
+        output_dir: Path,
+        phenopacket_path: Path,
+    ) -> None:
+        """
+        Create a scrambled version of a Phenopacket.
 
-    Args:
-        output_dir (Path): The directory to store the output scrambled Phenopackets.
-        phenopacket_dir (Path): The directory containing the original Phenopacket files.
-        scramble_factor (float): A factor determining the level of scrambling for phenotypic features.
-    """
-    ontology = load_ontology()
-    hpo_randomiser = HpoRandomiser(ontology, scramble_factor)
-    phenopacket_files = files_with_suffix(phenopacket_dir, ".json")
-    for phenopacket_path in phenopacket_files:
+        Args:
+            output_dir (Path): The directory to store the output scrambled Phenopacket.
+            phenopacket_path (Path): The path to the original Phenopacket file.
+        """
         phenopacket = phenopacket_reader(phenopacket_path)
-        created_noisy_phenopacket = add_noise_to_phenotypic_profile(hpo_randomiser, phenopacket)
+        created_noisy_phenopacket = self.add_noise_to_phenotypic_profile(
+            phenopacket,
+        )
         write_phenopacket(
             created_noisy_phenopacket,
-            output_dir.joinpath(
-                phenopacket_path.name,
-            ),
+            output_dir.joinpath(phenopacket_path.name),
         )
 
+    def create_scrambled_phenopackets(
+        self,
+        output_dir: Path,
+        phenopacket_dir: Path,
+    ) -> None:
+        """
+        Create scrambled versions of Phenopackets within a directory.
+
+        Args:
+            output_dir (Path): The directory to store the output scrambled Phenopackets.
+            phenopacket_dir (Path): The directory containing the original Phenopacket files.
+        """
+        phenopacket_files = files_with_suffix(phenopacket_dir, ".json")
+        for phenopacket_path in phenopacket_files:
+            phenopacket = phenopacket_reader(phenopacket_path)
+            created_noisy_phenopacket = self.add_noise_to_phenotypic_profile(phenopacket)
+            write_phenopacket(
+                created_noisy_phenopacket,
+                output_dir.joinpath(
+                    phenopacket_path.name,
+                ),
+            )
+
 
 def scramble_phenopackets(
-    output_dir: Path, phenopacket_path: Path, phenopacket_dir: Path, scramble_factor: float
+    output_dir: Path,
+    phenopacket_path: Path,
+    phenopacket_dir: Path,
+    scramble_factor: float,
+    local_cached_ontology: Path,
 ) -> None:
     """
     Create scrambled phenopackets from either a single phenopacket or a directory of phenopackets.
@@ -262,9 +266,16 @@ def scramble_phenopackets(
         phenopacket_path (Path): The path to a single Phenopacket file (if applicable).
         phenopacket_dir (Path): The directory containing multiple Phenopacket files (if applicable).
         scramble_factor (float): A factor determining the level of scrambling for phenotypic features.
+        local_cached_ontology (Path): The path to the local cached ontology.
     """
     output_dir.mkdir(exist_ok=True)
+    ontology = load_ontology(local_cached_ontology)
     if phenopacket_path is not None:
-        create_scrambled_phenopacket(output_dir, phenopacket_path, scramble_factor)
+        HpoRandomiser(ontology, scramble_factor).create_scrambled_phenopacket(
+            output_dir, phenopacket_path
+        )
     elif phenopacket_dir is not None:
-        create_scrambled_phenopackets(output_dir, phenopacket_dir, scramble_factor)
+        HpoRandomiser(ontology, scramble_factor).create_scrambled_phenopackets(
+            output_dir,
+            phenopacket_dir,
+        )

pheval-0.4.1.dist-info/METADATA

@@ -0,0 +1,113 @@
+Metadata-Version: 2.1
+Name: pheval
+Version: 0.4.1
+Summary: 
+Author: Yasemin Bridges
+Author-email: y.bridges@qmul.ac.uk
+Requires-Python: >=3.9,<4.0.0
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Dist: class-resolver (>=0.4.2)
+Requires-Dist: click (>=8.1.3)
+Requires-Dist: deprecation (>=2.1.0)
+Requires-Dist: duckdb (>=1.0.0,<2.0.0)
+Requires-Dist: google (>=3.0.0,<4.0.0)
+Requires-Dist: jaydebeapi (>=1.2.3)
+Requires-Dist: matplotlib (>=3.7.0,<4.0.0)
+Requires-Dist: oaklib (>=0.5.6)
+Requires-Dist: pandas (>=1.5.1)
+Requires-Dist: phenopackets (>=2.0.2,<3.0.0)
+Requires-Dist: plotly (>=5.13.0,<6.0.0)
+Requires-Dist: polars (>=0.19.15,<0.20.0)
+Requires-Dist: pyaml (>=21.10.1,<22.0.0)
+Requires-Dist: pyserde (>=0.9.8,<0.10.0)
+Requires-Dist: scikit-learn (>=1.4.0,<2.0.0)
+Requires-Dist: seaborn (>=0.12.2,<0.13.0)
+Requires-Dist: tqdm (>=4.64.1)
+Description-Content-Type: text/markdown
+
+# PhEval - Phenotypic Inference Evaluation Framework
+
+![PyPI](https://img.shields.io/pypi/v/pheval)
+![Build Status](https://img.shields.io/github/actions/workflow/status/monarch-initiative/pheval/pypi-publish.yml?branch=main)
+![License](https://img.shields.io/github/license/monarch-initiative/pheval)
+![Python Version](https://img.shields.io/badge/python-3.8%2B-blue)
+![Issues](https://img.shields.io/github/issues/monarch-initiative/pheval)
+
+## Overview
+
+The absence of standardised benchmarks and data standardisation for Variant and Gene Prioritisation Algorithms (VGPAs) presents a significant challenge in the field of genomic research. To address this, we developed PhEval, a novel framework designed to streamline the evaluation of VGPAs that incorporate phenotypic data. PhEval offers several key benefits:
+
+- Automated Processes: Reduces manual effort by automating various evaluation tasks, thus enhancing efficiency.
+- Standardisation: Ensures consistency and comparability in evaluation methodologies, leading to more reliable and standardised assessments.
+- Reproducibility: Facilitates reproducibility in research by providing a standardised platform, allowing for consistent validation of algorithms.
+- Comprehensive Benchmarking: Enables thorough benchmarking of algorithms, providing well-founded comparisons and deeper insights into their performance. 
+
+PhEval is a valuable tool for researchers looking to improve the accuracy and reliability of VGPA evaluations through a structured and standardised approach.
+
+For more information please see the full [documentation](https://monarch-initiative.github.io/pheval/).
+
+## Download and Installation
+
+1. Ensure you have Python 3.8 or greater installed.
+2. Install with `pip`:
+```bash
+pip install pheval
+```
+3. See list of all PhEval utility commands:
+```bash
+pheval-utils --help
+```
+
+## Usage
+
+The PhEval CLI offers a variety of commands categorised into two main types: **Runner Implementations** and **Utility Commands**. Below is an overview of each category, detailing how they can be utilised to perform various tasks within PhEval.
+
+### Runner Implementations
+
+The primary command used within PhEval is `pheval run`. This command is responsible for executing concrete VGPA runner implementations, that we sometimes term as plugins. By using pheval run, users can leverage these runner implementations to: execute the VGPA on a set of test corpora, produce tool-specific result outputs, and post-process tool-specific outputs to PhEval standardised TSV outputs.
+
+Some concrete PhEval runner implementations include the [Exomiser runner](https://github.com/monarch-initiative/pheval.exomiser) and the [Phen2Gene runner](https://github.com/monarch-initiative/pheval.phen2gene). The full list of currently implemented runners can be found [here](https://monarch-initiative.github.io/pheval/plugins/)
+
+Please read the [documentation](https://monarch-initiative.github.io/pheval/developing_a_pheval_plugin/) for a step-by-step for creating your own PhEval plugin. 
+
+### Utility Commands
+
+In addition to the main `run` command, PhEval provides a set of utility commands designed to enhance the overall functionality of the CLI. These commands can be used to set up and configure experiments, streamline data preparation, and benchmark the performance of various VGPA runner implementations. By utilising these utilities, users can optimise their experimental workflows, ensure reproducibility, and compare the efficiency and accuracy of different approaches. The utility commands offer a range of options that facilitate the customisation and fine-tuning to suit diverse research objectives.
+
+#### Example Usage
+
+To add noise to an existing corpus of phenopackets, this could be used to assess the robustness of VGPAs when less relevant or unreliable phenotype data is introduced:
+```bash
+pheval-utils scramble-phenopackets --phenopacket-dir /phenopackets --scramble-factor 0.5 --output-dir /scrambled_phenopackets_0.5
+```
+
+To update the gene symbols and identifiers to a specific namespace:
+```bash
+pheval-utils update-phenopackets --phenopacket-dir /phenopackets --output-dir /updated_phenopackets --gene-identifier ensembl_id
+```
+
+To prepare VCF files for a corpus of phenopackets, spiking in the known causative variants:
+```bash
+pheval-utils create-spiked-vcfs --phenopacket-dir /phenopackets --hg19-template-vcf /template_hg19.vcf --hg38-template-vcf /template_hg38.vcf --output-dir /vcf
+```
+
+Alternatively, you can wrap all corpus preparatory commands into a single step. Specifying `--variant-analysis`/`--gene-analysis`/`--disease-analysis` will check the phenopackets for complete records documenting the known entities. If template vcf(s) are provided this will spike VCFs with the known variant for the corpus. If a `--gene-identifier` is specified then the corpus of phenopackets is updated.
+```bash
+pheval-utils prepare-corpus \
+    --phenopacket-dir /phenopackets \
+    --variant-analysis \
+    --gene-analysis \
+    --gene-identifier ensembl_id \
+    --hg19-template-vcf /template_hg19.vcf \
+    --hg38-template-vcf /template_hg38.vcf \
+    --output-dir /vcf
+```
+
+See the [documentation](https://monarch-initiative.github.io/pheval/executing_a_benchmark/) for instructions on benchmarking and evaluating the performance of various VGPAs.
+
+