pipeconcord 0.2.0__py3-none-any.whl

pipeconcord/__init__.py

@@ -0,0 +1,8 @@
+"""Semantic comparison of bioinformatics pipeline outputs."""
+
+from pipeconcord._version import __version__
+from pipeconcord.core.engine import ComparisonEngine
+from pipeconcord.core.report import ConcordanceReport
+
+__all__ = ["ComparisonEngine", "ConcordanceReport", "__version__"]
+

pipeconcord/__main__.py

@@ -0,0 +1,5 @@
+from pipeconcord.cli import main
+
+if __name__ == "__main__":
+    raise SystemExit(main())
+

pipeconcord/_version.py

@@ -0,0 +1 @@
+__version__ = "0.2.0"

pipeconcord/cli.py

@@ -0,0 +1,147 @@
+from __future__ import annotations
+
+import argparse
+import sys
+
+from pipeconcord.core.batch import run_batch
+from pipeconcord.core.engine import ComparisonEngine
+from pipeconcord.io.report_writers import (
+    batch_to_html,
+    batch_to_json,
+    batch_to_text,
+    batch_to_tsv,
+    report_to_html,
+    report_to_json,
+    report_to_text,
+    write_batch,
+    write_report,
+)
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="pipeconcord",
+        description="Generate semantic concordance reports for bioinformatics outputs.",
+    )
+    subparsers = parser.add_subparsers(dest="command")
+
+    compare_parser = subparsers.add_parser("compare", help="Compare one pair of files.")
+    add_common_options(compare_parser)
+    compare_parser.add_argument("file_a", help="First output file to compare.")
+    compare_parser.add_argument("file_b", help="Second output file to compare.")
+    compare_parser.add_argument("-o", "--output", help="Write the report to a file instead of stdout.")
+    compare_parser.add_argument("--format", choices=["html", "json", "text"], default="json", help="Output format.")
+
+    batch_parser = subparsers.add_parser("batch", help="Compare file pairs listed in a CSV/TSV manifest.")
+    add_common_options(batch_parser)
+    batch_parser.add_argument("manifest", help="CSV/TSV manifest with file_a and file_b columns.")
+    batch_parser.add_argument("--min-concordance", type=float, help="Fail if any successful comparison is below this threshold.")
+    batch_parser.add_argument("--stop-on-error", action="store_true", help="Stop on the first failed comparison.")
+    batch_parser.add_argument("-o", "--output", help="Write the batch report to a file instead of stdout.")
+    batch_parser.add_argument("--format", choices=["html", "json", "tsv", "text"], default="tsv", help="Batch output format.")
+
+    return parser
+
+
+def add_common_options(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument("-t", "--type", dest="file_type", help="Force a comparator/file type such as bam_stats, bed, counts, deg, expression, fasta, fastq, table, csv, tsv, or vcf.")
+    parser.add_argument("--key", dest="key_column", help="Column to use for row alignment.")
+    parser.add_argument("--delimiter", help="Force a delimiter for tabular files.")
+    parser.add_argument("--alpha", type=float, help="DEG adjusted p-value threshold. Default: 0.05.")
+    parser.add_argument("--lfc-threshold", type=float, help="DEG absolute log-fold-change threshold. Default: 0.0.")
+    parser.add_argument("--top-n", type=int, help="Number of top-ranked DEG genes to compare. Default: 50.")
+    parser.add_argument("--gene-column", help="Gene identifier column override for DEG/count matrices.")
+    parser.add_argument("--sample-columns", help="Counts comparator sample columns as a comma-separated list.")
+    parser.add_argument("--min-reciprocal-overlap", type=float, help="BED interval match threshold. Default: 0.0 for any overlap.")
+    parser.add_argument("--reference-fasta", help="Reference FASTA for optional VCF indel left-alignment.")
+    parser.add_argument("--logfc-column", help="DEG log-fold-change column override.")
+    parser.add_argument("--padj-column", help="DEG adjusted p-value column override.")
+    parser.add_argument("--pvalue-column", help="DEG raw p-value column override when adjusted p-values are absent.")
+
+
+def main(argv: list[str] | None = None) -> int:
+    args_list = list(sys.argv[1:] if argv is None else argv)
+    if args_list and args_list[0] not in {"compare", "batch", "-h", "--help"}:
+        args_list = ["compare", *args_list]
+    parser = build_parser()
+    args = parser.parse_args(args_list)
+    if args.command is None:
+        parser.print_help()
+        return 2
+
+    try:
+        if args.command == "batch":
+            return run_batch_command(args)
+        return run_compare_command(args)
+    except Exception as exc:
+        parser.exit(2, f"pipeconcord: error: {exc}\n")
+    return 0
+
+
+def run_compare_command(args: argparse.Namespace) -> int:
+    engine = ComparisonEngine()
+    report = engine.compare(
+        args.file_a,
+        args.file_b,
+        **comparison_kwargs(args),
+    )
+    if args.output:
+        write_report(report, args.output, fmt=args.format)
+    elif args.format == "html":
+        print(report_to_html(report))
+    elif args.format == "text":
+        print(report_to_text(report))
+    else:
+        print(report_to_json(report))
+    return 0
+
+
+def run_batch_command(args: argparse.Namespace) -> int:
+    results = run_batch(
+        args.manifest,
+        stop_on_error=args.stop_on_error,
+        default_file_type=args.file_type,
+        **comparison_kwargs(args, include_file_type=False),
+    )
+    if args.output:
+        write_batch(results, args.output, fmt=args.format)
+    elif args.format == "html":
+        print(batch_to_html(results))
+    elif args.format == "json":
+        print(batch_to_json(results))
+    elif args.format == "text":
+        print(batch_to_text(results))
+    else:
+        print(batch_to_tsv(results))
+    if args.min_concordance is not None and not 0.0 <= args.min_concordance <= 1.0:
+        raise ValueError("min-concordance must be between 0.0 and 1.0")
+    below_threshold = [
+        result
+        for result in results
+        if result.report is not None and args.min_concordance is not None and result.report.overall_concordance < args.min_concordance
+    ]
+    return 1 if any(result.error for result in results) or below_threshold else 0
+
+
+def comparison_kwargs(args: argparse.Namespace, *, include_file_type: bool = True) -> dict[str, object]:
+    kwargs: dict[str, object] = {
+        "key_column": args.key_column,
+        "delimiter": args.delimiter,
+        "alpha": args.alpha,
+        "lfc_threshold": args.lfc_threshold,
+        "top_n": args.top_n,
+        "gene_column": args.gene_column,
+        "sample_columns": args.sample_columns,
+        "min_reciprocal_overlap": args.min_reciprocal_overlap,
+        "reference_fasta": args.reference_fasta,
+        "logfc_column": args.logfc_column,
+        "padj_column": args.padj_column,
+        "pvalue_column": args.pvalue_column,
+    }
+    if include_file_type:
+        kwargs["file_type"] = args.file_type
+    return kwargs
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

pipeconcord/comparators/__init__.py

@@ -0,0 +1,25 @@
+from __future__ import annotations
+
+from pipeconcord.comparators.bam_stats import BAMStatsComparator
+from pipeconcord.comparators.bed import BEDComparator
+from pipeconcord.comparators.counts import CountsComparator
+from pipeconcord.comparators.deg import DEGComparator
+from pipeconcord.comparators.expression import ExpressionComparator
+from pipeconcord.comparators.fasta import FASTAComparator
+from pipeconcord.comparators.table import TableComparator
+from pipeconcord.comparators.vcf import VCFComparator
+from pipeconcord.core.registry import ComparatorRegistry
+
+
+def register_builtin_comparators(registry: type[ComparatorRegistry] = ComparatorRegistry) -> None:
+    registry.register(DEGComparator)
+    registry.register(ExpressionComparator)
+    registry.register(CountsComparator)
+    registry.register(BEDComparator)
+    registry.register(FASTAComparator)
+    registry.register(VCFComparator)
+    registry.register(BAMStatsComparator)
+    registry.register(TableComparator)
+
+
+__all__ = ["BAMStatsComparator", "BEDComparator", "CountsComparator", "DEGComparator", "ExpressionComparator", "FASTAComparator", "TableComparator", "VCFComparator", "register_builtin_comparators"]

pipeconcord/comparators/bam_stats.py

@@ -0,0 +1,229 @@
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from pathlib import Path
+
+from pipeconcord.comparators.base import Comparator
+from pipeconcord.core.report import ConcordanceReport
+from pipeconcord.core.utils import clamp01
+
+
+FLAGSTAT_COUNT_RE = re.compile(r"^(\d+)\s+\+\s+\d+\s+(.+)$")
+
+
+@dataclass(slots=True)
+class AlignmentStats:
+    path: str
+    total_reads: float | None = None
+    mapped_reads: float | None = None
+    duplicate_reads: float | None = None
+    paired_reads: float | None = None
+    properly_paired_reads: float | None = None
+    insert_size_average: float | None = None
+    average_length: float | None = None
+    error_rate: float | None = None
+    recognized_metrics: int = 0
+
+    @property
+    def alignment_rate(self) -> float | None:
+        return fraction(self.mapped_reads, self.total_reads)
+
+    @property
+    def duplicate_rate(self) -> float | None:
+        return fraction(self.duplicate_reads, self.total_reads)
+
+    @property
+    def proper_pair_rate(self) -> float | None:
+        return fraction(self.properly_paired_reads, self.paired_reads)
+
+
+class BAMStatsComparator(Comparator):
+    """Comparator for samtools flagstat/stats alignment summaries."""
+
+    name = "bam_stats"
+    supported_types = ("bam_stats", "bam-stats", "flagstat", "samtools-stats")
+
+    def can_handle(self, file_a: str, file_b: str, **kwargs: object) -> bool:
+        requested_type = kwargs.get("file_type")
+        if requested_type in self.supported_types:
+            return True
+        if requested_type is not None:
+            return False
+        return looks_like_bam_stats(file_a) and looks_like_bam_stats(file_b)
+
+    def compare(self, file_a: str, file_b: str, **kwargs: object) -> ConcordanceReport:
+        stats_a = parse_alignment_stats(file_a)
+        stats_b = parse_alignment_stats(file_b)
+        metrics: dict[str, float] = {}
+        scores: list[float] = []
+
+        add_ratio_metric(metrics, scores, "total_reads_ratio", stats_a.total_reads, stats_b.total_reads)
+        add_ratio_metric(metrics, scores, "mapped_reads_ratio", stats_a.mapped_reads, stats_b.mapped_reads)
+        add_rate_similarity(metrics, scores, "alignment_rate_similarity", stats_a.alignment_rate, stats_b.alignment_rate)
+        add_rate_similarity(metrics, scores, "duplicate_rate_similarity", stats_a.duplicate_rate, stats_b.duplicate_rate)
+        add_rate_similarity(metrics, scores, "proper_pair_rate_similarity", stats_a.proper_pair_rate, stats_b.proper_pair_rate)
+        add_ratio_metric(metrics, scores, "insert_size_average_ratio", stats_a.insert_size_average, stats_b.insert_size_average)
+        add_ratio_metric(metrics, scores, "average_length_ratio", stats_a.average_length, stats_b.average_length)
+        add_rate_similarity(metrics, scores, "error_rate_similarity", stats_a.error_rate, stats_b.error_rate)
+
+        if stats_a.alignment_rate is not None:
+            metrics["alignment_rate_file_a"] = stats_a.alignment_rate
+        if stats_b.alignment_rate is not None:
+            metrics["alignment_rate_file_b"] = stats_b.alignment_rate
+        if stats_a.duplicate_rate is not None:
+            metrics["duplicate_rate_file_a"] = stats_a.duplicate_rate
+        if stats_b.duplicate_rate is not None:
+            metrics["duplicate_rate_file_b"] = stats_b.duplicate_rate
+
+        details = {
+            "file_a": stats_to_dict(stats_a),
+            "file_b": stats_to_dict(stats_b),
+            "scored_metric_count": len(scores),
+        }
+        return ConcordanceReport(
+            comparator=self.__class__.__name__,
+            file_a=str(file_a),
+            file_b=str(file_b),
+            overall_concordance=clamp01(mean(scores)),
+            metrics=metrics,
+            details=details,
+            warnings=[],
+        )
+
+
+def looks_like_bam_stats(path: str) -> bool:
+    text = Path(path).read_text(encoding="utf-8", errors="replace")[:8192]
+    if text.startswith("SN\t") or "\nSN\t" in text:
+        return True
+    lowered = text.lower()
+    return " in total " in lowered and " mapped (" in lowered
+
+
+def parse_alignment_stats(path: str) -> AlignmentStats:
+    stats = AlignmentStats(path=str(path))
+    for line in Path(path).read_text(encoding="utf-8", errors="replace").splitlines():
+        stripped = line.strip()
+        if not stripped:
+            continue
+        if stripped.startswith("SN\t"):
+            parse_samtools_stats_line(stats, stripped)
+        else:
+            parse_flagstat_line(stats, stripped)
+    if stats.recognized_metrics == 0:
+        raise ValueError(f"{path!r} does not look like samtools stats or flagstat output")
+    return stats
+
+
+def parse_samtools_stats_line(stats: AlignmentStats, line: str) -> None:
+    fields = line.split("\t")
+    if len(fields) < 3:
+        return
+    key = fields[1].rstrip(":").lower()
+    value = parse_number(fields[2])
+    if value is None:
+        return
+    mapping = {
+        "raw total sequences": "total_reads",
+        "reads mapped": "mapped_reads",
+        "reads duplicated": "duplicate_reads",
+        "insert size average": "insert_size_average",
+        "average length": "average_length",
+        "error rate": "error_rate",
+    }
+    attribute = mapping.get(key)
+    if attribute is not None:
+        setattr(stats, attribute, value)
+        stats.recognized_metrics += 1
+
+
+def parse_flagstat_line(stats: AlignmentStats, line: str) -> None:
+    match = FLAGSTAT_COUNT_RE.match(line)
+    if match is None:
+        return
+    count = float(match.group(1))
+    label = match.group(2).lower()
+    if " in total " in f" {label} ":
+        stats.total_reads = count
+    elif label.startswith("mapped "):
+        stats.mapped_reads = count
+    elif label.startswith("duplicates"):
+        stats.duplicate_reads = count
+    elif label.startswith("paired in sequencing"):
+        stats.paired_reads = count
+    elif label.startswith("properly paired"):
+        stats.properly_paired_reads = count
+    else:
+        return
+    stats.recognized_metrics += 1
+
+
+def parse_number(value: str) -> float | None:
+    try:
+        return float(value.strip())
+    except ValueError:
+        return None
+
+
+def fraction(numerator: float | None, denominator: float | None) -> float | None:
+    if numerator is None or denominator is None or denominator == 0:
+        return None
+    return numerator / denominator
+
+
+def add_ratio_metric(
+    metrics: dict[str, float],
+    scores: list[float],
+    name: str,
+    left: float | None,
+    right: float | None,
+) -> None:
+    if left is None and right is None:
+        return
+    score = magnitude_ratio(left or 0.0, right or 0.0)
+    metrics[name] = score
+    scores.append(score)
+
+
+def add_rate_similarity(
+    metrics: dict[str, float],
+    scores: list[float],
+    name: str,
+    left: float | None,
+    right: float | None,
+) -> None:
+    if left is None and right is None:
+        return
+    score = 0.0 if left is None or right is None else clamp01(1.0 - abs(left - right))
+    metrics[name] = score
+    scores.append(score)
+
+
+def magnitude_ratio(left: float, right: float) -> float:
+    if left == 0 and right == 0:
+        return 1.0
+    return min(abs(left), abs(right)) / max(abs(left), abs(right))
+
+
+def mean(values: list[float]) -> float:
+    if not values:
+        return 0.0
+    return sum(values) / len(values)
+
+
+def stats_to_dict(stats: AlignmentStats) -> dict[str, float | str | None]:
+    return {
+        "path": stats.path,
+        "total_reads": stats.total_reads,
+        "mapped_reads": stats.mapped_reads,
+        "duplicate_reads": stats.duplicate_reads,
+        "paired_reads": stats.paired_reads,
+        "properly_paired_reads": stats.properly_paired_reads,
+        "insert_size_average": stats.insert_size_average,
+        "average_length": stats.average_length,
+        "error_rate": stats.error_rate,
+        "alignment_rate": stats.alignment_rate,
+        "duplicate_rate": stats.duplicate_rate,
+        "proper_pair_rate": stats.proper_pair_rate,
+    }
+

pipeconcord/comparators/base.py

@@ -0,0 +1,21 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+
+from pipeconcord.core.report import ConcordanceReport
+
+
+class Comparator(ABC):
+    """Base class for semantic output comparators."""
+
+    name = "base"
+    supported_types: tuple[str, ...] = ()
+
+    @abstractmethod
+    def can_handle(self, file_a: str, file_b: str, **kwargs: object) -> bool:
+        """Return True when this comparator can compare the two inputs."""
+
+    @abstractmethod
+    def compare(self, file_a: str, file_b: str, **kwargs: object) -> ConcordanceReport:
+        """Run comparison and return a unified concordance report."""
+