pheval 0.4.6__tar.gz → 0.5.0__tar.gz

{pheval-0.4.6 → pheval-0.5.0}/PKG-INFO

@@ -1,12 +1,11 @@
 Metadata-Version: 2.3
 Name: pheval
-Version: 0.4.6
+Version: 0.5.0
 Summary: 
 Author: Yasemin Bridges
 Author-email: y.bridges@qmul.ac.uk
-Requires-Python: >=3.9,<4.0.0
+Requires-Python: >=3.10,<4.0.0
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
@@ -22,8 +21,9 @@ Requires-Dist: oaklib (>=0.5.6)
 Requires-Dist: pandas (>=1.5.1)
 Requires-Dist: phenopackets (>=2.0.2,<3.0.0)
 Requires-Dist: plotly (>=5.13.0,<6.0.0)
-Requires-Dist: polars (>=0.19.15,<0.20.0)
+Requires-Dist: polars (>=1.23,<2.0)
 Requires-Dist: pyaml (>=21.10.1,<22.0.0)
+Requires-Dist: pyarrow (>=19.0.1,<20.0.0)
 Requires-Dist: pyserde (>=0.9.8,<0.10.0)
 Requires-Dist: scikit-learn (>=1.4.0,<2.0.0)
 Requires-Dist: seaborn (>=0.12.2,<0.13.0)

{pheval-0.4.6 → pheval-0.5.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pheval"
-version = "0.4.6"
+version = "0.5.0"
 description = ""
 authors = ["Yasemin Bridges <y.bridges@qmul.ac.uk>",
   "Julius Jacobsen <j.jacobsen@qmul.ac.uk>",
@@ -10,7 +10,7 @@ readme = "README.md"
 packages = [{include = "pheval", from = "src"}]
 
 [tool.poetry.dependencies]
-python = ">=3.9,<4.0.0"
+python = ">=3.10,<4.0.0"
 jaydebeapi = ">=1.2.3"
 tqdm = ">=4.64.1"
 pandas = ">=1.5.1"
@@ -25,14 +25,14 @@ plotly = "^5.13.0"
 seaborn = "^0.12.2"
 matplotlib = "^3.7.0"
 pyserde = "^0.9.8"
-polars = "^0.19.15"
+polars = "^1.23"
 scikit-learn = "^1.4.0"
 duckdb = "^1.0.0"
+pyarrow = "^19.0.1"
 
 [tool.poetry.dev-dependencies]
 pytest = "^7.2.0"
 coverage = "^6.5.0"
-pheval-template = "^0.1.2"
 pytest-workflow = "^2.0.1"
 
 [tool.poetry.scripts]

pheval-0.5.0/src/pheval/analyse/benchmark.py

@@ -0,0 +1,156 @@
+import time
+from pathlib import Path
+from typing import List, Tuple
+
+import duckdb
+import polars as pl
+
+from pheval.analyse.benchmark_db_manager import write_table
+from pheval.analyse.benchmark_output_type import BenchmarkOutputType, BenchmarkOutputTypeEnum
+from pheval.analyse.binary_classification_curves import compute_curves
+from pheval.analyse.binary_classification_stats import compute_confusion_matrix
+from pheval.analyse.generate_plots import generate_plots
+from pheval.analyse.generate_rank_comparisons import calculate_rank_changes
+from pheval.analyse.rank_stats import compute_rank_stats
+from pheval.analyse.run_data_parser import Config, RunConfig, parse_run_config
+from pheval.utils.logger import get_logger
+
+
+def scan_directory(run: RunConfig, benchmark_type: BenchmarkOutputType) -> pl.LazyFrame:
+    """
+    Scan a results directory containing pheval parquet standardised results and return a LazyFrame object.
+    Args:
+        run (RunConfig): RunConfig object.
+        benchmark_type (BenchmarkOutputTypeEnum): Benchmark output type.
+    Returns:
+        pl.LazyFrame: LazyFrame object containing all the results in the directory..
+    """
+    logger = get_logger()
+    logger.info(f"Analysing results in {run.results_dir.joinpath(benchmark_type.result_directory)}")
+    return (
+        pl.scan_parquet(
+            run.results_dir.joinpath(benchmark_type.result_directory),
+            include_file_paths="file_path",
+        ).with_columns(
+            pl.col("rank").cast(pl.Int64),
+            pl.col("file_path").str.extract(r"([^/\\]+)$").alias("result_file"),
+            pl.col("true_positive").fill_null(False),
+        )
+    ).filter(
+        (
+            pl.col("score") >= run.threshold
+            if run.score_order.lower() == "descending"
+            else pl.col("score") <= run.threshold
+        )
+        if run.threshold is not None
+        else True
+    )
+
+
+def process_stats(
+    runs: List[RunConfig], benchmark_type: BenchmarkOutputType
+) -> Tuple[pl.DataFrame, pl.DataFrame, pl.DataFrame]:
+    """
+    Processes stats outputs for specified runs to compare.
+    Args:
+        runs (List[RunConfig]): List of runs to benchmark.
+        benchmark_type (BenchmarkOutputTypeEnum): Benchmark output type.
+    Returns:
+        Tuple[pl.DataFrame, pl.DataFrame, pl.DataFrame, pl.DataFrame]: The stats for all runs.
+    """
+    stats, curve_results, true_positive_cases = [], [], []
+    for run in runs:
+        result_scan = scan_directory(run, benchmark_type)
+        stats.append(
+            compute_rank_stats(run.run_identifier, result_scan).join(
+                compute_confusion_matrix(run.run_identifier, result_scan), on="run_identifier"
+            )
+        )
+        curve_results.append(compute_curves(run.run_identifier, result_scan))
+        true_positive_cases.append(
+            result_scan.filter(pl.col("true_positive")).select(
+                ["result_file", *benchmark_type.columns, pl.col("rank").alias(run.run_identifier)]
+            )
+        )
+    return (
+        pl.concat(stats, how="vertical").collect(),
+        pl.concat(curve_results, how="vertical").collect(),
+        pl.concat(true_positive_cases, how="align_inner").collect(),
+    )
+
+
+def benchmark(config: Config, benchmark_type: BenchmarkOutputType) -> None:
+    """
+    Benchmark results for specified runs for a specified prioritisation type for comparison.
+    Args:
+        config (Config): Configuration for benchmarking.
+        benchmark_type (BenchmarkOutputType): Benchmark output type.
+    """
+    conn = duckdb.connect(f"{config.benchmark_name}.duckdb")
+    stats, curve_results, true_positive_cases = process_stats(config.runs, benchmark_type)
+    write_table(
+        conn, stats, f"{config.benchmark_name}_{benchmark_type.prioritisation_type_string}_summary"
+    )
+    write_table(
+        conn,
+        curve_results,
+        f"{config.benchmark_name}_{benchmark_type.prioritisation_type_string}_binary_classification_curves",
+    )
+    calculate_rank_changes(
+        conn, [run.run_identifier for run in config.runs], true_positive_cases, benchmark_type
+    )
+    generate_plots(
+        config.benchmark_name, stats, curve_results, benchmark_type, config.plot_customisation
+    )
+    conn.close()
+
+
+def benchmark_runs(benchmark_config_file: Path) -> None:
+    """
+    Benchmark results for specified runs for comparison.
+    Args:
+        benchmark_config_file (Path): Path to benchmark config file.
+    """
+    logger = get_logger()
+    start_time = time.perf_counter()
+    logger.info("Initiated benchmarking process.")
+    config = parse_run_config(benchmark_config_file)
+    gene_analysis_runs = [run for run in config.runs if run.gene_analysis]
+    variant_analysis_runs = [run for run in config.runs if run.variant_analysis]
+    disease_analysis_runs = [run for run in config.runs if run.disease_analysis]
+    if gene_analysis_runs:
+        logger.info("Initiating benchmarking for gene results.")
+        benchmark(
+            Config(
+                benchmark_name=config.benchmark_name,
+                runs=gene_analysis_runs,
+                plot_customisation=config.plot_customisation,
+            ),
+            BenchmarkOutputTypeEnum.GENE.value,
+        )
+        logger.info("Finished benchmarking for gene results.")
+    if variant_analysis_runs:
+        logger.info("Initiating benchmarking for variant results")
+        benchmark(
+            Config(
+                benchmark_name=config.benchmark_name,
+                runs=variant_analysis_runs,
+                plot_customisation=config.plot_customisation,
+            ),
+            BenchmarkOutputTypeEnum.VARIANT.value,
+        )
+        logger.info("Finished benchmarking for variant results.")
+    if disease_analysis_runs:
+        logger.info("Initiating benchmarking for disease results")
+        benchmark(
+            Config(
+                benchmark_name=config.benchmark_name,
+                runs=disease_analysis_runs,
+                plot_customisation=config.plot_customisation,
+            ),
+            BenchmarkOutputTypeEnum.DISEASE.value,
+        )
+        logger.info("Finished benchmarking for disease results.")
+    logger.info(
+        f"Finished benchmarking! Total time: {time.perf_counter() - start_time:.2f} seconds."
+    )

pheval-0.5.0/src/pheval/analyse/benchmark_db_manager.py

@@ -0,0 +1,23 @@
+import polars as pl
+from duckdb import DuckDBPyConnection
+
+from pheval.utils.logger import get_logger
+
+logger = get_logger()
+
+
+def load_table_lazy(table_name: str, conn: DuckDBPyConnection) -> pl.LazyFrame:
+    logger.info(f"Loading table {table_name}")
+    return pl.from_arrow(conn.execute(f"SELECT * FROM {table_name}").fetch_arrow_table()).lazy()
+
+
+def write_table(conn: DuckDBPyConnection, df: pl.DataFrame, table_name: str) -> None:
+    """
+    Write table to DuckDB database.
+    Args:
+        conn (DuckDBPyConnection): DuckDB connection.
+        df (pl.DataFrame): DuckDB dataframe.
+        table_name (str): Table name.
+    """
+    logger.info(f"Storing results in {table_name}.")
+    conn.execute(f"""CREATE TABLE "{table_name}" AS SELECT * FROM df""")

pheval-0.5.0/src/pheval/analyse/benchmark_output_type.py

@@ -0,0 +1,43 @@
+from enum import Enum
+from typing import List, NamedTuple
+
+
+class BenchmarkOutputType(NamedTuple):
+    """
+    Represents the structure of benchmark output types.
+
+    Attributes:
+        prioritisation_type_string (str): The type of prioritisation being performed.
+        y_label (str): The label for the y-axis in performance evaluation plots.
+        columns (List[str]): The list of column names relevant to the benchmark output.
+        result_directory (str): The directory where benchmark results are stored.
+    """
+
+    prioritisation_type_string: str
+    y_label: str
+    columns: List[str]
+    result_directory: str
+
+
+class BenchmarkOutputTypeEnum(Enum):
+    """
+    Enumeration of benchmark output types, representing different entities.
+
+    Attributes:
+        GENE (BenchmarkOutputType): Benchmark output type for gene prioritisation.
+        VARIANT (BenchmarkOutputType): Benchmark output type for variant prioritisation.
+        DISEASE (BenchmarkOutputType): Benchmark output type for disease prioritisation.
+    """
+
+    GENE = BenchmarkOutputType(
+        "gene",
+        "Disease-causing genes (%)",
+        ["gene_identifier", "gene_symbol"],
+        "pheval_gene_results",
+    )
+    VARIANT = BenchmarkOutputType(
+        "variant", "Disease-causing variants (%)", ["variant_id"], "pheval_variant_results"
+    )
+    DISEASE = BenchmarkOutputType(
+        "disease", "Known diseases (%)", ["disease_identifier"], "pheval_disease_results"
+    )

pheval-0.5.0/src/pheval/analyse/binary_classification_curves.py

@@ -0,0 +1,132 @@
+from typing import Tuple
+
+import numpy as np
+import polars as pl
+from sklearn.metrics import precision_recall_curve, roc_curve
+
+from pheval.utils.logger import get_logger
+
+
+class BinaryClassificationCurves:
+    """Class for computing and storing ROC & Precision-Recall curves in Polars."""
+
+    @staticmethod
+    def _compute_finite_bounds(result_scan: pl.LazyFrame) -> Tuple[float, float]:
+        """
+        Compute min and max finite values in the 'score' column to handle NaN and Inf values.
+        Args:
+            result_scan (pl.LazyFrame): The LazyFrame containing the results for the directory.
+
+        Returns:
+            Tuple[float, float]: The (max_finite, min_finite) values for normalising scores.
+        """
+        return (
+            result_scan.select(
+                [
+                    pl.col("score").filter(pl.col("score").is_finite()).max().alias("max_finite"),
+                    pl.col("score").filter(pl.col("score").is_finite()).min().alias("min_finite"),
+                ]
+            )
+            .collect()
+            .row(0)
+        )
+
+    @staticmethod
+    def _clean_and_extract_data(
+        result_scan: pl.LazyFrame, max_finite: float, min_finite: float
+    ) -> pl.LazyFrame:
+        """
+        Normalise the 'score' column (handling NaNs and Inf values) and extract 'true_positive' labels.
+
+        Args:
+            result_scan (pl.LazyFrame): The LazyFrame containing the results for the directory.
+            max_finite (float): The maximum finite score value.
+            min_finite (float): The minimum finite score value.
+
+        Returns:
+            pl.LazyFrame: A LazyFrame with cleaned 'score' and binary 'true_positive' columns.
+        """
+        return result_scan.with_columns(
+            [
+                pl.when(pl.col("score").is_nan())
+                .then(0.0)
+                .when(pl.col("score").is_infinite() & (pl.col("score") > 0))
+                .then(max_finite)
+                .when(pl.col("score").is_infinite() & (pl.col("score") < 0))
+                .then(min_finite)
+                .otherwise(pl.col("score"))
+                .alias("score"),
+                pl.when(pl.col("true_positive").is_null())
+                .then(0)
+                .otherwise(pl.col("true_positive").cast(pl.Int8))
+                .alias("true_positive"),
+            ]
+        )
+
+    @staticmethod
+    def _compute_roc_pr_curves(
+        run_identifier: str, labels: np.ndarray, scores: np.ndarray
+    ) -> pl.LazyFrame:
+        """
+        Compute ROC and Precision-Recall curves.
+
+        Args:
+            labels (np.ndarray): Binary ground truth labels (0 or 1).
+            scores (np.ndarray): Prediction scores.
+
+        Returns:
+            pl.LazyFrame: A LazyFrame containing the computed FPR, TPR, Precision, Recall, and Thresholds.
+        """
+        fpr, tpr, roc_thresholds = roc_curve(labels, scores, pos_label=1)
+        precision, recall, pr_thresholds = precision_recall_curve(labels, scores, pos_label=1)
+
+        return pl.LazyFrame(
+            {
+                "run_identifier": [run_identifier],
+                "fpr": [fpr.tolist()],
+                "tpr": [tpr.tolist()],
+                "threshold_roc": [roc_thresholds.tolist()],
+                "precision": [precision.tolist()],
+                "recall": [recall.tolist()],
+                "threshold_pr": [pr_thresholds.tolist()],
+            }
+        )
+
+    @classmethod
+    def process(cls, result_scan: pl.LazyFrame, run_identifier: str) -> pl.LazyFrame:
+        """
+        Process scores, extract true labels, compute ROC and Precision-Recall curves,
+        and store results in a Polars LazyFrame with NumPy arrays.
+
+        Args:
+            result_scan (pl.LazyFrame): The LazyFrame containing the results for the directory.
+            run_identifier (str): Identifier for this run.
+
+        Returns:
+            pl.LazyFrame: A LazyFrame containing ROC & PR curve data with NumPy arrays.
+        """
+        max_finite, min_finite = cls._compute_finite_bounds(result_scan)
+        cleaned_data = (
+            cls._clean_and_extract_data(result_scan, max_finite, min_finite)
+            .select(["true_positive", "score"])
+            .collect()
+        )
+        return cls._compute_roc_pr_curves(
+            run_identifier,
+            cleaned_data["true_positive"].to_numpy().flatten(),
+            cleaned_data["score"].to_numpy().flatten(),
+        )
+
+
+def compute_curves(run_identifier: str, result_scan: pl.LazyFrame) -> pl.LazyFrame:
+    """
+    Compute ROC and Precision-Recall curves.
+    Args:
+        result_scan (pl.LazyFrame): The LazyFrame containing the results for the directory.
+        run_identifier (str): Identifier for this run.
+    Returns:
+        pl.LazyFrame: LazyFrame containing the ROC & Precision-Recall curve data with NumPy arrays.
+    """
+    logger = get_logger()
+    logger.info("Calculating ROC and Precision-Recall metrics")
+    return BinaryClassificationCurves.process(result_scan, run_identifier)

pheval-0.5.0/src/pheval/analyse/binary_classification_stats.py

@@ -0,0 +1,186 @@
+from dataclasses import dataclass
+from multiprocessing.util import get_logger
+
+import polars as pl
+
+
+@dataclass(frozen=True)
+class ConfusionMatrix:
+    """
+    Define logical conditions for computing a confusion matrix using Polars expressions.
+
+    Attributes:
+        TRUE_POSITIVES (pl.Expr): Condition identifying true positive cases,
+            where `rank == 1` and `true_positive` is `True`.
+        FALSE_POSITIVES (pl.Expr): Condition identifying false positive cases,
+            where `rank == 1` and `true_positive` is `False`.
+        TRUE_NEGATIVES (pl.Expr): Condition identifying true negative cases,
+            where `rank != 1` and `true_positive` is `False`.
+        FALSE_NEGATIVES (pl.Expr): Condition identifying false negative cases,
+            where `rank != 1` and `true_positive` is `True`.
+    """
+
+    TRUE_POSITIVES = (pl.col("rank") == 1) & (pl.col("true_positive"))
+    FALSE_POSITIVES = (pl.col("rank") == 1) & (~pl.col("true_positive"))
+    TRUE_NEGATIVES = (pl.col("rank") != 1) & (~pl.col("true_positive"))
+    FALSE_NEGATIVES = (pl.col("rank") != 1) & (pl.col("true_positive"))
+
+
+@dataclass(frozen=True)
+class BinaryClassificationStats:
+    """Binary classification statistic expressions."""
+
+    SENSITIVITY = (
+        pl.when((pl.col("true_positives") + pl.col("false_negatives")) != 0)
+        .then(pl.col("true_positives") / (pl.col("true_positives") + pl.col("false_negatives")))
+        .otherwise(0.0)
+        .alias("sensitivity")
+    )
+
+    SPECIFICITY = (
+        pl.when((pl.col("true_negatives") + pl.col("false_positives")) != 0)
+        .then(pl.col("true_negatives") / (pl.col("true_negatives") + pl.col("false_positives")))
+        .otherwise(0.0)
+        .alias("specificity")
+    )
+
+    PRECISION = (
+        pl.when((pl.col("true_positives") + pl.col("false_positives")) != 0)
+        .then(pl.col("true_positives") / (pl.col("true_positives") + pl.col("false_positives")))
+        .otherwise(0.0)
+        .alias("precision")
+    )
+
+    NEGATIVE_PREDICTIVE_VALUE = (
+        pl.when((pl.col("true_negatives") + pl.col("false_negatives")) != 0)
+        .then(pl.col("true_negatives") / (pl.col("true_negatives") + pl.col("false_negatives")))
+        .otherwise(0.0)
+        .alias("negative_predictive_value")
+    )
+
+    FALSE_POSITIVE_RATE = (
+        pl.when((pl.col("false_positives") + pl.col("true_negatives")) != 0)
+        .then(pl.col("false_positives") / (pl.col("false_positives") + pl.col("true_negatives")))
+        .otherwise(0.0)
+        .alias("false_positive_rate")
+    )
+
+    FALSE_DISCOVERY_RATE = (
+        pl.when((pl.col("false_positives") + pl.col("true_positives")) != 0)
+        .then(pl.col("false_positives") / (pl.col("false_positives") + pl.col("true_positives")))
+        .otherwise(0.0)
+        .alias("false_discovery_rate")
+    )
+
+    FALSE_NEGATIVE_RATE = (
+        pl.when((pl.col("false_negatives") + pl.col("true_positives")) != 0)
+        .then(pl.col("false_negatives") / (pl.col("false_negatives") + pl.col("true_positives")))
+        .otherwise(0.0)
+        .alias("false_negative_rate")
+    )
+
+    ACCURACY = (
+        pl.when(
+            (
+                pl.col("true_positives")
+                + pl.col("false_positives")
+                + pl.col("true_negatives")
+                + pl.col("false_negatives")
+            )
+            != 0
+        )
+        .then(
+            (pl.col("true_positives") + pl.col("true_negatives"))
+            / (
+                pl.col("true_positives")
+                + pl.col("false_positives")
+                + pl.col("true_negatives")
+                + pl.col("false_negatives")
+            )
+        )
+        .otherwise(0.0)
+        .alias("accuracy")
+    )
+
+    F1_SCORE = (
+        pl.when(
+            2 * (pl.col("true_positives") + pl.col("false_positives") + pl.col("false_negatives"))
+            != 0
+        )
+        .then(
+            2
+            * pl.col("true_positives")
+            / (2 * pl.col("true_positives") + pl.col("false_positives") + pl.col("false_negatives"))
+        )
+        .otherwise(0.0)
+        .alias("f1_score")
+    )
+
+    MATTHEWS_CORRELATION_COEFFICIENT = (
+        pl.when(
+            (
+                (pl.col("true_positives") + pl.col("false_positives"))
+                * (pl.col("true_positives") + pl.col("false_negatives"))
+                * (pl.col("true_negatives") + pl.col("false_positives"))
+                * (pl.col("true_negatives") + pl.col("false_negatives"))
+            )
+            > 0
+        )
+        .then(
+            (
+                (pl.col("true_positives") * pl.col("true_negatives"))
+                - (pl.col("false_positives") * pl.col("false_negatives"))
+            )
+            / (
+                (pl.col("true_positives") + pl.col("false_positives"))
+                * (pl.col("true_positives") + pl.col("false_negatives"))
+                * (pl.col("true_negatives") + pl.col("false_positives"))
+                * (pl.col("true_negatives") + pl.col("false_negatives"))
+            ).sqrt()
+        )
+        .otherwise(0.0)
+        .alias("matthews_correlation_coefficient")
+    )
+
+
+def compute_confusion_matrix(run_identifier: str, result_scan: pl.LazyFrame) -> pl.LazyFrame:
+    """
+    Computes binary classification statistics.
+
+    Args:
+        run_identifier (str): The identifier for the run.
+        result_scan (pl.LazyFrame): The LazyFrame containing the results for the directory.
+
+    Returns:
+        pl.LazyFrame: The LazyFrame containing the binary classification statistics.
+    """
+    logger = get_logger()
+    logger.info(f"Computing binary classification statistics for {run_identifier}")
+    confusion_matrix = result_scan.select(
+        [
+            pl.lit(run_identifier).alias("run_identifier"),
+            ConfusionMatrix.TRUE_POSITIVES.sum().alias("true_positives").cast(pl.Int64),
+            ConfusionMatrix.FALSE_POSITIVES.sum().alias("false_positives").cast(pl.Int64),
+            ConfusionMatrix.TRUE_NEGATIVES.sum().alias("true_negatives").cast(pl.Int64),
+            ConfusionMatrix.FALSE_NEGATIVES.sum().alias("false_negatives").cast(pl.Int64),
+        ]
+    )
+    return confusion_matrix.select(
+        [
+            pl.col("run_identifier"),
+            pl.col("true_positives"),
+            pl.col("false_positives"),
+            pl.col("true_negatives"),
+            pl.col("false_negatives"),
+            BinaryClassificationStats.SENSITIVITY,
+            BinaryClassificationStats.SPECIFICITY,
+            BinaryClassificationStats.PRECISION,
+            BinaryClassificationStats.NEGATIVE_PREDICTIVE_VALUE,
+            BinaryClassificationStats.FALSE_POSITIVE_RATE,
+            BinaryClassificationStats.FALSE_DISCOVERY_RATE,
+            BinaryClassificationStats.FALSE_NEGATIVE_RATE,
+            BinaryClassificationStats.ACCURACY,
+            BinaryClassificationStats.F1_SCORE,
+            BinaryClassificationStats.MATTHEWS_CORRELATION_COEFFICIENT,
+        ]
+    )