TSUMUGI 1.0.1__py3-none-any.whl

TSUMUGI/filterer.py

@@ -0,0 +1,36 @@
+from __future__ import annotations
+
+from collections.abc import Generator, Iterable, Iterator
+from itertools import groupby
+from operator import itemgetter
+
+
+def subset_columns(records: Iterator[dict[str, str]], columns: set[str]) -> Generator[dict[str, str]]:
+    """Return list[dict] keeping only the requested columns; missing keys become empty strings."""
+    return ({col: record.get(col, "") for col in columns} for record in records)
+
+
+###########################################################
+# Others
+###########################################################
+
+
+def distinct_records_with_max_effect(
+    records: Iterable[dict[str, str | float]], unique_keys: list[str]
+) -> Generator[dict[str, str | float]]:
+    """
+    Groups records by the specified keys and returns the record with the maximum
+    effect_size from each group.
+    Note: effect_size is already an absolute value.
+    """
+    # Dynamically define the key function based on unique_keys.
+    record_key_getter = itemgetter(*unique_keys)
+
+    # Pre-sort by the same key for groupby to function correctly.
+    records_sorted = sorted(records, key=record_key_getter)
+
+    for _, group in groupby(records_sorted, key=record_key_getter):
+        # Find the record with the maximum effect_size within the group.
+        # Use .get() to safely handle cases where the 'effect_size' key might be missing.
+        record_with_max_effect = max(group, key=lambda r: r.get("effect_size", -1))
+        yield record_with_max_effect

TSUMUGI/formatter.py

@@ -0,0 +1,122 @@
+from __future__ import annotations
+
+import math
+from collections import defaultdict
+from collections.abc import Generator, Iterable
+
+###########################################################
+# String to Float
+###########################################################
+
+
+def _to_float(x: str | None) -> float:
+    """Convert a string to float; empty/None becomes NaN."""
+    return float(x) if x not in (None, "") else float("nan")
+
+
+def floatinize_columns(records: Iterable[dict[str, str]], columns: list[str]) -> Generator[dict[str, str | float]]:
+    """Return a record with numeric fields coerced to float/NaN."""
+    for record in records:
+        for col in columns:
+            record[col] = _to_float(record.get(col))
+        yield record
+
+
+def abs_effect_size(
+    records: Iterable[dict[str, str | float]], effect_size_columns: list[str]
+) -> Generator[dict[str, str | float]]:
+    """Return a record with the absolute effect size and NaN replaced with 0."""
+    for record in records:
+        for col in effect_size_columns:
+            if math.isnan(record[col]):
+                record[col] = 0.0
+            record[col] = abs(record[col])
+        yield record
+
+
+###########################################################
+# Zygosity Formatting
+###########################################################
+
+
+def format_zygosity(
+    records: Iterable[dict[str, str | float]], zygosity_converter: dict[str, str]
+) -> Generator[dict[str, str | float]]:
+    """Format zygosity values to a consistent style."""
+    for record in records:
+        record["zygosity"] = zygosity_converter.get(record["zygosity"], record["zygosity"])
+        yield record
+
+
+###########################################################
+# IMPC human disease_annotations
+###########################################################
+
+
+def _select_fields_from_disease_annotations(disease_annotations: list[dict[str, str]]) -> list[dict[str, str]]:
+    """Select disorder_name and description fields from IMPC human disease_annotations."""
+
+    # Heuristic to identify the correct fields
+    prev_disorder_name = None
+    prev_description = None
+    for record in disease_annotations:
+        for key, value in record.items():
+            if "Syndrome" in value:
+                prev_disorder_name = key
+            if "<em1(IMPC)Bay>" in value:
+                prev_description = key
+        if prev_disorder_name is not None and prev_description is not None:
+            break
+
+    if prev_disorder_name is None or prev_description is None:
+        raise ValueError("Could not identify disorder_name or description fields in disease_annotations.")
+
+    # Select the fields
+    disease_annotations_selected = []
+    for record in disease_annotations:
+        disease_annotations_selected.append(
+            {"disorder_name": record[prev_disorder_name], "description": record[prev_description]}
+        )
+
+    return disease_annotations_selected
+
+
+def format_disease_annotations(disease_annotations: list[dict[str, str | float]]) -> dict[str, list[dict[str, str]]]:
+    """Format the IMPC human disease_annotations for output."""
+
+    # Select "description" and "disorder_name" fields
+    disease_annotations = _select_fields_from_disease_annotations(disease_annotations)
+
+    # Filter out records with unexpected description format: expected format is "<allele_symbol> <zygosity> <life_stage>"
+    disease_annotations = [record for record in disease_annotations if len(record["description"].split(" ")) == 3]
+
+    zygosity_converter = {"het": "Hetero", "hom": "Homo", "hem": "Hemi"}
+    life_stage_converter = {"middle": "Interval", "late": "Late", "early": "Early", "embryo": "Embryo"}
+
+    for record in disease_annotations:
+        description = record["description"]
+        description_split = description.split(" ")
+        marker_symbol = description.split("<")[0].strip()
+        zygosity = description_split[-2]
+        life_stage = description_split[-1]
+        # Apply converters
+        zygosity = zygosity_converter.get(zygosity, zygosity)
+        life_stage = life_stage_converter.get(life_stage, life_stage)
+        # Update record with new values
+        record["marker_symbol"] = marker_symbol
+        record["zygosity"] = zygosity
+        record["life_stage"] = life_stage
+        # Delete used fields
+        del record["description"]
+
+    # Using marker_symbol as the key makes it easier to join with IMPC phenotype records later
+    disease_annotations_by_gene = defaultdict(list)
+    appended_records = set()
+    for record in disease_annotations:
+        if tuple(record.items()) not in appended_records:
+            appended_records.add(tuple(record.items()))
+            marker_symbol = record["marker_symbol"]
+            del record["marker_symbol"]
+            disease_annotations_by_gene[marker_symbol].append(record)
+
+    return dict(disease_annotations_by_gene)

TSUMUGI/genewise_annotation_builder.py

@@ -0,0 +1,94 @@
+from __future__ import annotations
+
+from collections.abc import Iterator
+
+from TSUMUGI import annotator, filterer, formatter
+
+
+def build_genewise_phenotype_annotations(
+    records: Iterator[dict], ontology_terms: dict, disease_annotations_by_gene: dict
+) -> Iterator[dict]:
+    ###########################################################
+    # Preprocess data
+    ###########################################################
+
+    # --------------------------------------------------------
+    # Select columns, maintained mp term, and significant genes
+    # --------------------------------------------------------
+
+    # Floatinize columns
+    float_columns = [
+        "p_value",
+        "effect_size",
+        "female_ko_effect_p_value",
+        "female_ko_parameter_estimate",
+        "male_ko_effect_p_value",
+        "male_ko_parameter_estimate",
+    ]
+    records_formatted = formatter.floatinize_columns(records, float_columns)
+
+    # Format zygosity
+    zygosity_converter = {"heterozygote": "Hetero", "homozygote": "Homo", "hemizygote": "Hemi"}
+    records_formatted = formatter.format_zygosity(records_formatted, zygosity_converter)
+    # Take absolute value of effect size
+    effect_size_columns = ["effect_size", "female_ko_parameter_estimate", "male_ko_parameter_estimate"]
+    records_formatted = formatter.abs_effect_size(records_formatted, effect_size_columns)
+
+    # --------------------------------------------------------
+    # Annotate life stage and sexual dimorphisms
+    # --------------------------------------------------------
+
+    embryo_assays = {
+        "E9.5",
+        "E10.5",
+        "E12.5",
+        "Embryo LacZ",  # E12.5
+        "E14.5",
+        "E14.5-E15.5",
+        "E18.5",
+    }
+    # Life stage
+    records_annotated = annotator.annotate_life_stage(records_formatted, embryo_assays)
+    # Sexual dimorphism
+    records_annotated = annotator.annotate_sexual_dimorphism(records_annotated, threshold=1e-4)
+    # Human Diseases
+    records_annotated = annotator.annotate_diseases(records_annotated, disease_annotations_by_gene)
+    # Annotate Significant (True/False)
+    records_annotated = annotator.annotate_significant(records_annotated)
+
+    # --------------------------------------------------------
+    # Filter records
+    # --------------------------------------------------------
+    records_filtered = records_annotated
+
+    # Subset columns
+    to_keep_columns = {
+        "marker_symbol",
+        "marker_accession_id",
+        "mp_term_id",
+        "mp_term_name",
+        "zygosity",
+        "life_stage",
+        "sexual_dimorphism",
+        "effect_size",
+        "significant",
+        "disease_annotation",
+    }
+    records_filtered = filterer.subset_columns(records_filtered, to_keep_columns)
+
+    # Keep only records with mp_term_id in the ontology file (= not obsolete)
+    records_filtered = (record for record in records_filtered if record["mp_term_id"] in ontology_terms)
+
+    # Distinct records with max effect size
+    unique_keys = [
+        "marker_symbol",
+        "mp_term_id",
+        "zygosity",
+        "life_stage",
+        "sexual_dimorphism",
+    ]
+    records_filtered = filterer.distinct_records_with_max_effect(records_filtered, unique_keys)
+
+    genewise_phenotype_annotations = records_filtered
+
+    return genewise_phenotype_annotations

TSUMUGI/io_handler.py

@@ -0,0 +1,189 @@
+from __future__ import annotations
+
+import csv
+import gzip
+import json
+import pickle
+import sys
+from collections.abc import Iterable, Iterator
+from pathlib import Path
+
+from tqdm import tqdm
+
+from TSUMUGI import formatter
+
+
+def count_newline(file_path: str | Path, chunk_size: int = 1024 * 1024) -> int:
+    """Count newline characters in plain text or gzip-compressed file efficiently."""
+    open_func = gzip.open if Path(file_path).suffix == ".gz" else open
+    count = 0
+    with open_func(file_path, "rb") as f:
+        while chunk := f.read(chunk_size):
+            count += chunk.count(b"\n")
+    return count
+
+
+def load_csv_as_dicts(file_path: str | Path, encoding: str = "utf-8") -> Iterator[dict[str, str]]:
+    """
+    Read CSV file and return each row as a {header: value} dict
+    """
+    file_path = Path(file_path)
+
+    # Detect gzip-compressed file
+    open_func = gzip.open if file_path.suffix == ".gz" else open
+
+    # Open and read as text
+    with open_func(file_path, mode="rt", newline="", encoding=encoding) as f:
+        reader = csv.DictReader(f)
+        for row in reader:
+            yield dict(row)
+
+
+def write_pickle(obj: any, file_path: str | Path) -> None:
+    """
+    Save any Python object in pickle format
+    """
+    with open(file_path, "wb") as f:
+        pickle.dump(obj, f, protocol=pickle.HIGHEST_PROTOCOL)
+
+
+def read_pickle(file_path: str | Path) -> any:
+    """
+    Load pickle file and return as Python object
+    """
+    with open(file_path, "rb") as f:
+        return pickle.load(f)
+
+
+def write_pickle_iter(iterable: Iterator, file_path: str | Path) -> None:
+    """
+    Sequential (streaming) pickle save.
+    Write element by element from iterable (including generators) without loading into memory.
+    """
+    with open(file_path, "wb") as f:
+        for item in iterable:
+            pickle.dump(item, f, protocol=pickle.HIGHEST_PROTOCOL)
+
+
+def read_pickle_iter(file_path: str | Path) -> Iterator[any]:
+    """
+    Generator that reads pickle sequentially (yields until EOF)
+    """
+    with open(file_path, "rb") as f:
+        while True:
+            try:
+                yield pickle.load(f)
+            except EOFError:
+                break
+
+
+def parse_obo_file(file_path: str | Path) -> dict[str, dict[str, str]]:
+    """Parse ontology file (OBO format) and extract term information.
+    Returns dict with keys: id, name, is_a (parent terms), is_obsolete
+    """
+    ontology_terms = {}
+    current_term_data = None
+    with open(file_path, encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+
+            if line == "[Term]":
+                current_term_data = {}
+                continue
+
+            if line.startswith("[") and line.endswith("]") and line != "[Term]":
+                current_term_data = None
+                continue
+
+            if current_term_data is None:
+                continue
+
+            if ":" in line:
+                key, value = line.split(":", 1)
+                key = key.strip()
+                value = value.strip()
+
+                if key == "id":
+                    current_term_data["id"] = value
+                elif key == "name":
+                    current_term_data["name"] = value
+                elif key == "is_a":
+                    if "is_a" not in current_term_data:
+                        current_term_data["is_a"] = []
+                    parent_id = value.split("!")[0].strip()
+                    current_term_data["is_a"].append(parent_id)
+                elif key == "is_obsolete":
+                    current_term_data["is_obsolete"] = value.lower() == "true"
+
+            if line == "" and current_term_data and "id" in current_term_data:
+                if not current_term_data.get("is_obsolete", False):
+                    ontology_terms[current_term_data["id"]] = current_term_data
+                current_term_data = None
+
+    return ontology_terms
+
+
+def parse_impc_phenodigm(file_path: str | Path) -> dict[str, list[dict[str, str]]]:
+    with open(Path(file_path)) as f:
+        disease_annotations_by_gene: dict[str, list[dict[str, str]]] = formatter.format_disease_annotations(
+            list(csv.DictReader(f))
+        )
+    return disease_annotations_by_gene
+
+
+def read_jsonl(path_jsonl: str | Path | None) -> Iterator[dict]:
+    """
+    Stream JSONL (.jsonl or .jsonl.gz). If path_jsonl is None or "-", read from stdin.
+    Keeps the file open during iteration (no 'I/O operation on closed file').
+    """
+    # stdin
+    if path_jsonl is None or str(path_jsonl) == "-" or path_jsonl == sys.stdin:
+        for line in sys.stdin:
+            if line.strip():
+                yield json.loads(line)
+        return
+
+    # file / gzip
+    p = Path(path_jsonl)
+    open_func = gzip.open if p.suffix == ".gz" else open
+
+    with open_func(p, "rt", encoding="utf-8") as f:
+        for line in f:
+            if line.strip():
+                yield json.loads(line)
+
+
+def write_jsonl(records: Iterable[dict], path_jsonl: str | Path | None) -> None:
+    """
+    Write an iterable of records as JSONL (.jsonl or .jsonl.gz).
+
+    If the filename ends with .gz, use gzip compression (level=9).
+    """
+    p = Path(path_jsonl)
+
+    def open_text_file(path: Path, mode: str, encoding: str):
+        return open(path, mode, encoding=encoding)
+
+    def open_gzip_file(path: Path, mode: str, encoding: str):
+        return gzip.open(path, mode, encoding=encoding, compresslevel=9)
+
+    open_func = open_gzip_file if p.suffix == ".gz" else open_text_file
+
+    message = f"Writing JSONL to {path_jsonl}"
+    with open_func(p, "wt", encoding="utf-8") as f:
+        for record in tqdm(records, desc=message):
+            json.dump(record, f, ensure_ascii=False)
+            f.write("\n")
+
+
+def write_jsonl_to_stdout(record: dict) -> None:
+    """Write record as JSONL and suppress BrokenPipeError cleanly."""
+    try:
+        json.dump(record, sys.stdout, ensure_ascii=False)
+        sys.stdout.write("\n")
+    except BrokenPipeError:
+        try:
+            sys.stdout.close()
+        except Exception:
+            pass
+        sys.exit(0)