parseet 0.2.0__py3-none-any.whl

parseet/core/utils/check_order.py

@@ -0,0 +1,426 @@
+from collections import defaultdict
+from pathlib import Path
+import logging
+import re
+import uuid
+
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# Constants
+# ---------------------------------------------------------------------------
+
+FILENAME_PATTERN = re.compile(
+    r"^(?P<index>\d+)_(?P<sample>.+)_(?P<column>C18|HILIC)_(?P<mode>n|p|np)\.raw$"
+)
+
+CONTROL_SAMPLES: frozenset[str] = frozenset({"Blank", "QC", "FM", "FMA", "FMB"})
+
+
+# ---------------------------------------------------------------------------
+# Result helper
+# ---------------------------------------------------------------------------
+
+class CheckResult:
+    """Accumulates errors, warnings, and informational messages."""
+
+    def __init__(self):
+        self.errors:   list[str] = []
+        self.warnings: list[str] = []
+        self.ok:       list[str] = []
+
+    def error(self, msg: str) -> None:
+        self.errors.append(msg)
+        logger.error(msg)
+
+    def warn(self, msg: str) -> None:
+        self.warnings.append(msg)
+        logger.warning(msg)
+
+    def good(self, msg: str) -> None:
+        self.ok.append(msg)
+        logger.info(msg)
+
+    @property
+    def has_errors(self) -> bool:
+        return bool(self.errors)
+
+
+# ---------------------------------------------------------------------------
+# Filename helpers
+# ---------------------------------------------------------------------------
+
+def _numeric_prefix(path: Path) -> int:
+    """Return the leading integer of a filename, or inf if unparseable."""
+    try:
+        return int(path.name.split("_", 1)[0])
+    except ValueError:
+        return float("inf")
+
+
+def _parse_filename(filename: str) -> dict | None:
+    """Return the named groups from FILENAME_PATTERN, or None on no match."""
+    m = FILENAME_PATTERN.match(filename)
+    return m.groupdict() if m else None
+
+
+def _build_filename(index: int, sample: str, column: str, mode: str, leading_zeros: int) -> str:
+    return f"{index:0{leading_zeros}d}_{sample.replace('_', '-')}_{column}_{mode}.raw"
+
+
+# ---------------------------------------------------------------------------
+# Sub-functions
+# ---------------------------------------------------------------------------
+
+def _collect_raw_files(folder: Path, result: CheckResult) -> list[Path]:
+    """
+    Glob all .raw files in *folder*.
+
+    Returns an empty list (and registers an error) when none are found.
+    """
+    raw_files = sorted(folder.glob("*.raw"))
+    if not raw_files:
+        result.error(f"No .raw files found in '{folder}'")
+    else:
+        result.good(f"Found {len(raw_files)} .raw file(s) in '{folder}'")
+    return raw_files
+
+
+def _check_time_vs_alpha_order(
+    raw_files: list[Path],
+    result: CheckResult,
+) -> tuple[list[Path], list[Path], bool]:
+    """
+    Compare alphabetical (numeric-prefix) order against last-modified order.
+
+    Returns
+    -------
+    alpha_order, time_order:
+        The two sorted file lists.
+    already_correct:
+        True when orders match, indexing starts at 01, and all names have
+        exactly 3 underscores.
+    """
+    alpha_order = sorted(raw_files, key=_numeric_prefix)
+    time_order  = sorted(raw_files, key=lambda f: f.stat().st_mtime)
+
+    already_correct = (
+        alpha_order == time_order
+        and alpha_order[0].stem.startswith("01")
+        and all(f.name.count("_") == 3 for f in alpha_order)
+    )
+
+    if already_correct:
+        result.good("Alphabetical order matches last-modified time order")
+    else:
+        result.warn("Alphabetical order does NOT match last-modified time order")
+
+    return alpha_order, time_order, already_correct
+
+
+def _parse_and_build_rename_map(
+    time_order: list[Path],
+    result: CheckResult,
+) -> tuple[dict[str, str], list[str], dict[str, list[Path]]]:
+    """
+    Parse every file in *time_order* and build the rename map.
+
+    Returns
+    -------
+    rename_map:
+        ``{old_name: new_name}`` for every successfully parsed file.
+    samples:
+        Ordered list of sample names (one per parsed file).
+    sample_to_files:
+        Mapping of sample name → list of Path objects.
+
+    Raises
+    ------
+    ValueError
+        If any filename does not match FILENAME_PATTERN.
+    """
+    rename_map:      dict[str, str]        = {}
+    samples:         list[str]             = []
+    sample_to_files: dict[str, list[Path]] = defaultdict(list)
+    invalid_files:   list[str]             = []
+
+    for idx, file in enumerate(time_order, start=1):
+        parsed = _parse_filename(file.name)
+        if not parsed:
+            invalid_files.append(file.name)
+            continue
+
+        sample = parsed["sample"].replace("_", "-")
+        leading_zeros = len(str(len(time_order)))
+        new_name = _build_filename(idx, sample, parsed["column"], parsed["mode"], leading_zeros)
+
+        rename_map[file.name] = new_name
+        samples.append(sample)
+        sample_to_files[sample].append(file)
+        logger.debug("Parsed '%s' → sample='%s' new_name='%s'", file.name, sample, new_name)
+
+    if invalid_files:
+        raise ValueError("Invalid filenames:\n" + "\n".join(invalid_files))
+
+    # Warn about duplicate sample names
+    seen: set[str] = set()
+    for sample in samples:
+        if samples.count(sample) > 1 and sample not in seen:
+            seen.add(sample)
+            count = samples.count(sample)
+            result.warn(
+                f"Duplicate sample name '{sample}' appears {count} time(s) "
+                "in raw files — may cause renaming and validation issues"
+            )
+
+    return rename_map, samples, sample_to_files
+
+
+def _validate_against_samplesheet(
+    samplesheet,          # pd.DataFrame | None
+    samples: list[str],
+    sample_to_files: dict[str, list[Path]],
+    ignore_id: bool,
+    result: CheckResult,
+) -> None:
+    """
+    Cross-check parsed raw-file sample names against the samplesheet.
+
+    Errors are registered for:
+    - samples present in raw files but absent from the samplesheet
+      (``ID``-prefixed samples are treated separately and can be suppressed
+      with *ignore_id*)
+    - samples present in the samplesheet but absent from raw files
+
+    Duplicate sample names in the samplesheet are registered as warnings.
+    """
+    if samplesheet is None:
+        logger.debug("No samplesheet provided — skipping cross-validation")
+        return
+
+    sheet_names_series = samplesheet["Sample name"].astype(str).str.strip()
+    sheet_samples = set(sheet_names_series)
+
+    # Warn about samplesheet duplicates
+    if len(sheet_samples) != len(sheet_names_series):
+        dupes = sheet_names_series[sheet_names_series.duplicated(keep=False)]
+        for name in sorted(set(dupes)):
+            count = dupes.tolist().count(name)
+            result.warn(
+                f"Duplicate samplesheet entry: '{name}' appears {count} time(s)"
+            )
+
+    # Raw samples, excluding known controls
+    raw_samples = {
+        s for s in samples
+        if s not in CONTROL_SAMPLES and not s.startswith("QC")
+    }
+
+    missing_in_sheet = raw_samples - sheet_samples
+    missing_in_raw   = sheet_samples - raw_samples
+
+    if missing_in_sheet:
+        id_samples    = {s for s in missing_in_sheet if s.startswith("ID")}
+        other_samples = missing_in_sheet - id_samples
+
+        if id_samples:
+            if ignore_id:
+                result.good(
+                    f"Ignoring {len(id_samples)} 'ID'-prefixed sample(s) "
+                    "missing from samplesheet (--ignore-id active)"
+                )
+            else:
+                for s in sorted(id_samples):
+                    files = ", ".join(f.name for f in sample_to_files[s])
+                    result.error(
+                        f"Sample '{s}' starts with 'ID' and is missing from "
+                        f"the samplesheet — files: {files}. "
+                        "Use --ignore-id to skip."
+                    )
+
+        for s in sorted(other_samples):
+            files = ", ".join(f.name for f in sample_to_files[s])
+            result.error(
+                f"Sample '{s}' is present in raw files but missing from "
+                f"the samplesheet — files: {files}"
+            )
+
+    if missing_in_raw:
+        for s in sorted(missing_in_raw):
+            result.error(
+                f"Sample '{s}' is present in the samplesheet but has no "
+                "matching .raw file"
+            )
+
+    if not missing_in_sheet and not missing_in_raw:
+        result.good("All samples in raw files are present in the samplesheet")
+
+
+def _log_rename_proposals(rename_map: dict[str, str], result: CheckResult) -> None:
+    """Register info messages for every rename that would change a filename."""
+    changes = [(old, new) for old, new in rename_map.items() if old != new]
+    if not changes:
+        result.good("No renames needed — all filenames already correct")
+        return
+    for old, new in changes:
+        result.good(f"Proposed rename: '{old}' → '{new}'")
+
+
+def _apply_renames(folder: Path, rename_map: dict[str, str], result: CheckResult) -> None:
+    """
+    Atomically rename files using a two-phase temp-name strategy to avoid
+    collisions when names are swapped.
+    """
+    temp_map: dict[str, str] = {}
+
+    # Phase 1: move every changed file to a unique temp name
+    for old_name, new_name in rename_map.items():
+        if old_name == new_name:
+            continue
+        old_path  = folder / old_name
+        temp_name = f".tmp_{uuid.uuid4().hex}_{old_name}"
+        old_path.rename(folder / temp_name)
+        temp_map[temp_name] = new_name
+        logger.debug("Phase-1 rename: '%s' → '%s'", old_name, temp_name)
+
+    # Phase 2: move each temp file to its final name
+    for temp_name, final_name in temp_map.items():
+        (folder / temp_name).rename(folder / final_name)
+        logger.debug("Phase-2 rename: '%s' → '%s'", temp_name, final_name)
+
+    result.good(f"Renamed {len(temp_map)} file(s)")
+
+
+def _write_rename_log(
+    folder: Path,
+    rename_map: dict[str, str],
+    output_txt: str,
+    result: CheckResult,
+) -> None:
+    """Write the old→new mapping to a tab-separated text file."""
+    output_path = folder / output_txt
+    with open(output_path, "w") as fh:
+        for old_name, new_name in rename_map.items():
+            fh.write(f"{old_name}\t{new_name}\n")
+    result.good(f"Rename log written to: '{output_path}'")
+
+
+# ---------------------------------------------------------------------------
+# Public entry point
+# ---------------------------------------------------------------------------
+
+def check_raw_file_order(
+    folder_path,
+    samplesheet=None,
+    output_txt: str = "correct_time_order.txt",
+    dry_run: bool = True,
+    ignore_id: bool = False,
+) -> tuple[dict[str, str], list[str], list[str], list[str]]:
+    """
+    Validate and (optionally) fix the numeric ordering of .raw files.
+
+    Compares the alphabetical (numeric-prefix) order of ``.raw`` files
+    against their last-modified timestamps, cross-validates sample names
+    against the samplesheet, and optionally renames files in place using a
+    collision-safe two-phase strategy.
+
+    Parameters
+    ----------
+    folder_path:
+        Directory containing the ``.raw`` files.
+    samplesheet:
+        Optional DataFrame with a ``"Sample name"`` column. When provided,
+        sample names extracted from filenames are cross-checked against it.
+    output_txt:
+        Name of the tab-separated rename-log file written inside
+        *folder_path*.
+    dry_run:
+        When ``True`` (default), propose renames without applying them.
+    ignore_id:
+        When ``True``, suppress errors for samples whose names start with
+        ``"ID"`` but are absent from the samplesheet.
+
+    Returns
+    -------
+    rename_map:
+        ``{old_filename: new_filename}`` for every parsed .raw file.
+    errors:
+        Blocking error messages collected during validation.
+    warnings:
+        Non-blocking warning messages.
+    ok:
+        Passed-check messages.
+
+    Raises
+    ------
+    ValueError
+        If any filename does not match the expected pattern.
+    """
+    result = CheckResult()
+    folder = Path(folder_path)
+    logger.info("Checking raw file order in '%s'", folder)
+
+    # 1. Collect .raw files
+    raw_files = _collect_raw_files(folder, result)
+    if not raw_files:
+        return {}, result.errors, result.warnings, result.ok
+
+    # 2. Compare alphabetical vs time order
+    _alpha_order, time_order, already_correct = _check_time_vs_alpha_order(
+        raw_files, result
+    )
+
+    # 3. Parse files and build rename map (raises on invalid filenames)
+    rename_map, samples, sample_to_files = _parse_and_build_rename_map(
+        time_order, result
+    )
+
+    # 4. Cross-validate against samplesheet (always, regardless of order)
+    _validate_against_samplesheet(
+        samplesheet, samples, sample_to_files, ignore_id, result
+    )
+
+    # 5. Stop here if the order is already correct
+    if already_correct:
+        logger.info("Order is already correct — nothing to rename")
+        return rename_map, result.errors, result.warnings, result.ok
+
+    # 6. Log proposed renames
+    _log_rename_proposals(rename_map, result)
+
+    # 7. Apply renames or report dry-run status
+    if result.has_errors:
+        result.warn("Skipping rename step due to validation errors")
+    elif not dry_run:
+        _apply_renames(folder, rename_map, result)
+    else:
+        result.good("Dry run enabled — no files were renamed")
+
+    # 8. Write rename log
+    _write_rename_log(folder, rename_map, output_txt, result)
+
+    logger.info(
+        "Done — %d error(s), %d warning(s)",
+        len(result.errors), len(result.warnings),
+    )
+
+    return rename_map, result.errors, result.warnings, result.ok
+
+if __name__ == "__main__":
+    import pandas as pd
+    from parse_samplesheet import parse_samplesheet
+    from setup_logger import configure_logging
+    configure_logging(level=logging.INFO)
+
+    df = pd.read_excel("test_files/LC-MS_metabolomics_PLASMA.xlsx", header=None)
+    sample_df, _, _, _, _ = parse_samplesheet(df, )
+    rename_map, errors, warnings, ok = check_raw_file_order(
+        "test_files/raw_files",
+        samplesheet=sample_df,
+        output_txt="proposed_renames.txt",
+        dry_run=True,
+        ignore_id=True,
+    )
+    #print("Errors:", errors)
+    #print("Warnings:", warnings)
+    #print("OK messages:", ok)