batchlens-bio 0.1.0__py3-none-any.whl

batchlens/__init__.py

@@ -0,0 +1,3 @@
+"""Metadata-only experimental-design audits. No expression correction."""
+
+__version__ = "0.1.0"

batchlens/__main__.py

@@ -0,0 +1,3 @@
+from batchlens.cli import main
+
+raise SystemExit(main())

batchlens/audit.py

@@ -0,0 +1,232 @@
+"""Evidence-backed findings over experimental units and declared contrasts."""
+
+from typing import Any
+
+from batchlens import __version__
+from batchlens.config import StudySpec
+from batchlens.design import analyse_design
+from batchlens.metadata import Inputs, deidentify
+
+LIMITATION = (
+    "Metadata and the declared additive model only. This audit does not establish power, "
+    "independence, randomization, absence of unmeasured confounding, or causality."
+)
+
+
+def audit(inputs: Inputs, spec: StudySpec) -> dict[str, Any]:
+    inputs = deidentify(inputs, spec)
+    samples, observations, assays = inputs.samples, inputs.observations, inputs.assays
+    findings: list[dict[str, Any]] = []
+
+    def finding(rule: str, severity: str, message: str, refs: list[str], next_step: str) -> None:
+        findings.append(
+            {
+                "rule_id": rule,
+                "rule_version": "1.0",
+                "severity": severity,
+                "scope": "declared study",
+                "message": message,
+                "evidence_refs": refs,
+                "limitations": LIMITATION,
+                "suggested_next_step": next_step,
+            }
+        )
+
+    units: list[dict[str, Any]] = [
+        {
+            "unit_alias": unit,
+            "samples": len(group),
+            "sample_aliases": sorted(group[spec.sample_id]),
+            "target_levels": sorted(set(group[spec.target])),
+        }
+        for unit, group in samples.groupby(spec.unit_id, sort=True)
+    ]
+    unsupported: list[str] = []
+    if any(unit["samples"] > 1 for unit in units):
+        finding(
+            "BL-UNIT-001",
+            "info",
+            "Repeated samples from an experimental unit are present.",
+            ["/units"],
+            "Verify the declared experimental unit and repeated-sampling structure.",
+        )
+    if spec.design_mode == "independent" and any(unit["samples"] != 1 for unit in units):
+        unsupported.append("independent mode requires exactly one sample per experimental unit")
+    if spec.design_mode == "paired":
+        expected = set(spec.variables[spec.target].levels or [])
+        for _, group in samples.groupby(spec.unit_id):
+            if len(group) != 2 or set(group[spec.target]) != expected:
+                unsupported.append(
+                    "paired mode needs one sample at each target level for every unit"
+                )
+                break
+
+    target_counts = []
+    for level in spec.variables[spec.target].levels or []:
+        subset = samples[samples[spec.target] == level]
+        count = int(subset[spec.unit_id].nunique())
+        target_counts.append({"target_level": level, "samples": len(subset), "units": count})
+        if count < 2:
+            finding(
+                "BL-UNIT-002",
+                "warning",
+                f"Target level {level!r} has fewer than two units.",
+                ["/target_counts"],
+                "Review biological replication; this is not a power analysis.",
+            )
+    coverage = []
+    for name, var in spec.variables.items():
+        if var.role != "batch":
+            continue
+        for target in spec.variables[spec.target].levels or []:
+            for batch in var.levels or []:
+                subset = samples[(samples[spec.target] == target) & (samples[name] == batch)]
+                coverage.append(
+                    {
+                        "batch_variable": name,
+                        "batch_level": batch,
+                        "target_level": target,
+                        "samples": len(subset),
+                        "units": int(subset[spec.unit_id].nunique()),
+                    }
+                )
+    if any(row["samples"] == 0 for row in coverage):
+        finding(
+            "BL-COVER-001",
+            "warning",
+            "Some target-by-batch combinations have no samples.",
+            ["/coverage"],
+            "Review coverage. An empty combination alone is not non-estimability.",
+        )
+
+    observation_coverage = []
+    if observations is not None:
+        lookup = samples.set_index(spec.sample_id)
+        observed = set(observations[spec.sample_id])
+        absent = sorted(set(samples[spec.sample_id]) - observed)
+        if absent:
+            finding(
+                "BL-COVER-002",
+                "warning",
+                f"{len(absent)} samples have no linked observations.",
+                ["/counts"],
+                "Confirm whether the observation export is intentionally partial.",
+            )
+        for column in ["cell_type", "region"]:
+            if column not in observations:
+                continue
+            for label, group in observations.groupby(column, sort=True):
+                for target in spec.variables[spec.target].levels or []:
+                    selected = group[group[spec.sample_id].map(lookup[spec.target]) == target]
+                    observation_coverage.append(
+                        {
+                            "annotation": column,
+                            "label": label,
+                            "target_level": target,
+                            "observations": len(selected),
+                            "samples": int(selected[spec.sample_id].nunique()),
+                            "units": int(
+                                selected[spec.sample_id].map(lookup[spec.unit_id]).nunique()
+                            ),
+                        }
+                    )
+
+    assay_summary: dict[str, Any] = {"links": 0, "assays": 0, "multi_batch_samples": []}
+    if assays is not None:
+        assay_summary.update(links=len(assays), assays=int(assays.assay_id.nunique()))
+        for name, var in spec.variables.items():
+            if var.role != "batch" or name not in assays:
+                continue
+            for sample, group in assays.groupby(spec.sample_id, sort=True):
+                if group[name].nunique() > 1:
+                    assay_summary["multi_batch_samples"].append(
+                        {
+                            "sample_alias": sample,
+                            "batch_variable": name,
+                            "batch_levels": sorted(set(group[name])),
+                        }
+                    )
+        for column in ["slide_id", "section_id"]:
+            if column in assays:
+                assay_summary[column + "_count"] = int(assays[column].nunique())
+        if assay_summary["multi_batch_samples"]:
+            unsupported.append(
+                "some samples span multiple batches; sample-level batch is not unique"
+            )
+
+    matrix = None
+    if unsupported:
+        comparisons = [{**c.model_dump(), "status": "NOT_ASSESSED"} for c in spec.contrasts]
+        finding(
+            "BL-SUPPORT-001",
+            "warning",
+            "; ".join(unsupported),
+            ["/not_assessed"],
+            "Review the analysis unit/model explicitly; no samples are automatically dropped.",
+        )
+    else:
+        matrix, comparisons = analyse_design(samples, spec)
+        if matrix["rank"] < matrix["n_columns"]:
+            finding(
+                "BL-DESIGN-001",
+                "warning",
+                "The declared design matrix is rank deficient.",
+                ["/matrix_diagnostics/dependencies"],
+                "Inspect the dependencies and each contrast separately; do not drop batch blindly.",
+            )
+        if matrix["numerically_sensitive"]:
+            finding(
+                "BL-NUMERIC-001",
+                "warning",
+                "The design is nearly singular and sensitive to tolerance.",
+                ["/matrix_diagnostics"],
+                "Review covariate redundancy and the numeric diagnostics.",
+            )
+        for index, comparison in enumerate(comparisons):
+            if comparison["status"] == "NON_ESTIMABLE":
+                finding(
+                    "BL-CONTRAST-001",
+                    "critical",
+                    f"{comparison['id']}: the requested contrast is not estimable in this model.",
+                    [f"/contrasts/{index}", "/matrix_diagnostics/dependencies"],
+                    "Discuss crossed sampling or a revised scientific question; correction cannot "
+                    "supply missing design information.",
+                )
+
+    findings.sort(
+        key=lambda f: (
+            {"critical": 0, "warning": 1, "info": 2}[f["severity"]],
+            f["rule_id"],
+            f["message"],
+        )
+    )
+    return {
+        "schema_version": "1.0",
+        "tool_version": __version__,
+        "ruleset_version": "1.0",
+        "run_status": "completed",
+        "declared_design": spec.model_dump(),
+        "counts": {
+            "experimental_units": len(units),
+            "samples": len(samples),
+            "observations": 0 if observations is None else len(observations),
+        },
+        "units": units,
+        "target_counts": target_counts,
+        "coverage": coverage,
+        "observation_coverage": observation_coverage,
+        "assay_summary": assay_summary,
+        "matrix_diagnostics": matrix,
+        "contrasts": comparisons,
+        "findings": findings,
+        "not_assessed": unsupported,
+        "limitations": LIMITATION,
+        "sharing_notice": "IDs use aliases. Labels and design values may still identify people; "
+        "review every output before sharing. No observation-level data are exported.",
+    }
+
+
+def policy_exit(result: dict[str, Any], fail_on: str) -> int:
+    levels = {"none": 99, "critical": 2, "warning": 1}
+    severity = {"info": 0, "warning": 1, "critical": 2}
+    return 3 if any(severity[f["severity"]] >= levels[fail_on] for f in result["findings"]) else 0

batchlens/cli.py

@@ -0,0 +1,104 @@
+"""Command-line interface. Exit 3 means findings, not execution failure."""
+
+import argparse
+import hashlib
+import json
+import sys
+from importlib import resources
+from pathlib import Path
+
+from batchlens import __version__
+from batchlens.audit import audit, policy_exit
+from batchlens.config import InputError, read_spec
+from batchlens.metadata import load_inputs
+from batchlens.reporting import write_bundle
+
+CASES = [
+    "balanced",
+    "confounded-time",
+    "partial-overlap",
+    "redundant-nuisance",
+    "paired",
+    "spatial-replicates",
+    "mixed-assays",
+]
+
+
+def parser() -> argparse.ArgumentParser:
+    result = argparse.ArgumentParser(
+        prog="batchlens",
+        description="Audit experimental units and declared batch-confounded contrasts.",
+        epilog="Exit codes: 0 completed; 1 internal failure; 2 input/output error; "
+        "3 findings threshold. "
+        "No expression correction. See docs/input-schema.md and docs/interpretation.md.",
+    )
+    result.add_argument("--version", action="version", version=f"batchlens {__version__}")
+    sub = result.add_subparsers(dest="command", required=True)
+    for name in ["validate", "audit", "demo"]:
+        command = sub.add_parser(name)
+        if name == "demo":
+            command.add_argument("--case", choices=CASES, default="confounded-time")
+        else:
+            command.add_argument(
+                "--samples", required=True, type=Path, help="CSV/TSV; unique sample ID"
+            )
+            command.add_argument(
+                "--design", required=True, type=Path, help="Strict YAML model contract"
+            )
+            command.add_argument("--observations", type=Path, help="Optional cells/spots metadata")
+            command.add_argument("--assays", type=Path, help="Optional sample-to-assay links")
+        if name != "validate":
+            command.add_argument(
+                "--out", required=True, type=Path, help="New directory; parent must exist"
+            )
+            command.add_argument(
+                "--fail-on", choices=["critical", "warning", "none"], default="critical"
+            )
+    return result
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = parser().parse_args(argv)
+    try:
+        dataset = None
+        if args.command == "demo":
+            folder = Path(str(resources.files("batchlens").joinpath("resources/demo", args.case)))
+            args.samples, args.design = folder / "samples.tsv", folder / "design.yaml"
+            args.observations = (
+                folder / "observations.tsv" if (folder / "observations.tsv").is_file() else None
+            )
+            args.assays = folder / "assays.tsv" if (folder / "assays.tsv").is_file() else None
+            dataset = {"kind": "SYNTHETIC", "title": args.case}
+        spec = read_spec(args.design)
+        inputs = load_inputs(args.samples, spec, args.observations, args.assays)
+        if args.command == "validate":
+            print("Metadata valid. Model support and contrast estimability have not been assessed.")
+            return 0
+        result = audit(inputs, spec)
+        inputs.provenance["design"] = {
+            "sha256": hashlib.sha256(args.design.read_bytes()).hexdigest(),
+            "canonical_sha256": hashlib.sha256(
+                json.dumps(spec.model_dump(), sort_keys=True, ensure_ascii=False).encode()
+            ).hexdigest(),
+        }
+        write_bundle(result, inputs.provenance, args.out, dataset)
+        print(
+            f"BatchLens audit completed: {result['counts']['experimental_units']} "
+            "experimental units; "
+            f"{result['counts']['samples']} samples"
+        )
+        for comparison in result["contrasts"]:
+            print(f"{comparison['id']}: {comparison['status']}")
+        print(f"Report: {args.out / 'report.html'}")
+        print(f"JSON: {args.out / 'result.json'}")
+        return policy_exit(result, args.fail_on)
+    except (InputError, OSError) as exc:
+        print(f"Input/output error: {exc}", file=sys.stderr)
+        return 2
+    except Exception as exc:
+        print(
+            f"Internal error ({type(exc).__name__}). No successful audit is claimed. "
+            "Please report a minimal, de-identified reproducer.",
+            file=sys.stderr,
+        )
+        return 1

batchlens/config.py

@@ -0,0 +1,132 @@
+"""A bounded, declarative model contract; no formula evaluation."""
+
+from pathlib import Path
+from typing import Any, Literal, Self
+
+import yaml
+from pydantic import BaseModel, ConfigDict, Field, ValidationError, model_validator
+
+
+class InputError(ValueError):
+    """An actionable input problem, distinct from a scientific finding."""
+
+
+class StrictModel(BaseModel):
+    model_config = ConfigDict(extra="forbid", strict=True)
+
+
+class Variable(StrictModel):
+    role: Literal["target", "batch", "covariate"]
+    type: Literal["categorical", "numeric"]
+    levels: list[str] | None = None
+    reference: str | None = None
+
+    @model_validator(mode="after")
+    def validate_type(self) -> Self:
+        if self.type == "categorical":
+            if not self.levels or len(set(self.levels)) != len(self.levels):
+                raise ValueError("Categorical levels must be a nonempty unique list of strings")
+            if any(not x.strip() or x != x.strip() for x in self.levels):
+                raise ValueError("Levels must be nonempty and have no surrounding whitespace")
+            if self.reference not in self.levels:
+                raise ValueError("Categorical reference must belong to levels")
+        elif self.levels is not None or self.reference is not None:
+            raise ValueError("Numeric variables cannot declare levels or reference")
+        return self
+
+
+class Contrast(StrictModel):
+    id: str = Field(min_length=1, max_length=200)
+    numerator: str
+    denominator: str
+
+
+class StudySpec(StrictModel):
+    schema_version: Literal["1.0"]
+    design_mode: Literal["independent", "paired"]
+    sample_id: str
+    unit_id: str
+    variables: dict[str, Variable]
+    target: str
+    adjust_for: list[str]
+    contrasts: list[Contrast] = Field(min_length=1, max_length=100)
+
+    @model_validator(mode="after")
+    def validate_model(self) -> Self:
+        names = [self.sample_id, self.unit_id, *self.variables]
+        if any(not n.strip() or n != n.strip() for n in names):
+            raise ValueError("Column names must be nonempty with no surrounding whitespace")
+        if self.sample_id == self.unit_id or {self.sample_id, self.unit_id} & self.variables.keys():
+            raise ValueError("ID columns and model variables must have distinct names")
+        reserved = {"observation_id", "assay_id", "slide_id", "section_id", "cell_type", "region"}
+        if {self.sample_id, self.unit_id} & reserved:
+            raise ValueError(
+                "Sample/unit ID column names cannot reuse optional-table interface names"
+            )
+        if {"observation_id", "assay_id"} & self.variables.keys():
+            raise ValueError("Observation and assay identity columns cannot be model variables")
+        target = self.variables.get(self.target)
+        if target is None or target.role != "target" or target.type != "categorical":
+            raise ValueError("target must name a categorical variable with role target")
+        if sum(v.role == "target" for v in self.variables.values()) != 1:
+            raise ValueError("Exactly one target variable is supported")
+        if len(target.levels or []) < 2:
+            raise ValueError("target needs at least two levels")
+        if len(set(self.adjust_for)) != len(self.adjust_for):
+            raise ValueError("adjust_for must not contain duplicates")
+        if set(self.adjust_for) != set(self.variables) - {self.target}:
+            raise ValueError(
+                "adjust_for must contain every declared non-target variable exactly once"
+            )
+        if not any(v.role == "batch" for v in self.variables.values()):
+            raise ValueError("Declare at least one batch variable")
+        if any(v.role == "batch" and v.type != "categorical" for v in self.variables.values()):
+            raise ValueError("Batch variables must be categorical in v0.1")
+        if self.design_mode == "paired" and len(target.levels or []) != 2:
+            raise ValueError("paired mode requires exactly two target levels")
+        if len({c.id for c in self.contrasts}) != len(self.contrasts):
+            raise ValueError("Contrast IDs must be unique")
+        for contrast in self.contrasts:
+            if contrast.numerator == contrast.denominator:
+                raise ValueError("Contrast numerator and denominator must differ")
+            if not {contrast.numerator, contrast.denominator} <= set(target.levels or []):
+                raise ValueError("Contrast levels must be declared target levels")
+        return self
+
+
+class UniqueSafeLoader(yaml.SafeLoader):
+    """Reject duplicate keys instead of silently changing the declared design."""
+
+
+def _mapping(loader: UniqueSafeLoader, node: yaml.MappingNode) -> dict[Any, Any]:
+    result: dict[Any, Any] = {}
+    for key_node, value_node in node.value:
+        key = loader.construct_object(key_node, deep=True)
+        if not isinstance(key, str):
+            raise InputError("YAML mapping keys must be strings")
+        if key in result:
+            raise InputError(f"Duplicate YAML key at line {key_node.start_mark.line + 1}")
+        result[key] = loader.construct_object(value_node, deep=True)
+    return result
+
+
+UniqueSafeLoader.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, _mapping)
+
+
+def read_spec(path: Path) -> StudySpec:
+    try:
+        raw = path.read_text(encoding="utf-8-sig")
+        if len(raw) > 1_000_000:
+            raise InputError("Design configuration exceeds 1 MB")
+        # Aliases are unnecessary here and can create recursive/expanding documents.
+        if any(isinstance(t, (yaml.AliasToken, yaml.AnchorToken)) for t in yaml.scan(raw)):
+            raise InputError("YAML anchors and aliases are unsupported; use explicit values")
+        return StudySpec.model_validate(yaml.load(raw, Loader=UniqueSafeLoader))
+    except ValidationError as exc:
+        details = "; ".join(
+            f"{'.'.join(map(str, e['loc'])) or 'design'}: {e['msg']}"
+            for e in exc.errors(include_input=False, include_url=False)
+        )
+        raise InputError(details) from exc
+    except (OSError, UnicodeError, yaml.YAMLError) as exc:
+        raise InputError(f"Cannot read design ({type(exc).__name__}); check path and YAML") from exc

batchlens/design.py

@@ -0,0 +1,105 @@
+"""Linear-model estimability, without fitting outcomes or interpreting causality."""
+
+from typing import Any
+
+import numpy as np
+import pandas as pd
+
+from batchlens.config import InputError, StudySpec
+
+
+def analyse_design(samples: pd.DataFrame, spec: StudySpec) -> tuple[dict[str, Any], list[dict]]:
+    n_columns = 1 + sum(
+        len(v.levels or []) - 1 if v.type == "categorical" else 1 for v in spec.variables.values()
+    )
+    if spec.design_mode == "paired":
+        n_columns += int(samples[spec.unit_id].nunique()) - 1
+    if n_columns > 256 or len(samples) > 100_000:
+        raise InputError("Design exceeds v0.1 limit: 100,000 samples or 256 encoded columns")
+    vectors = [np.ones(len(samples), dtype=float)]
+    columns: list[dict[str, Any]] = [{"name": "Intercept", "kind": "intercept"}]
+    target_indices: dict[str, int] = {}
+    terms = [spec.target, *spec.adjust_for]
+    if spec.design_mode == "paired":
+        terms.append(spec.unit_id)
+    for term in terms:
+        var = spec.variables.get(term)
+        if var is None or var.type == "categorical":
+            levels = sorted(set(samples[term])) if var is None else (var.levels or [])
+            reference = levels[0] if var is None else var.reference
+            for level in levels:
+                if level == reference:
+                    continue
+                if term == spec.target:
+                    target_indices[level] = len(vectors)
+                vectors.append((samples[term] == level).to_numpy(dtype=float))
+                columns.append(
+                    {"name": term, "kind": "categorical", "level": level, "reference": reference}
+                )
+        else:
+            values = samples[term].to_numpy(dtype=float)
+            # Rescale before centering to avoid overflow for large finite input values.
+            magnitude = float(np.max(np.abs(values))) or 1.0
+            scaled = values / magnitude
+            center = float(scaled.mean())
+            scale = float(np.std(scaled)) or 1.0
+            vectors.append((scaled - center) / scale)
+            columns.append(
+                {
+                    "name": term,
+                    "kind": "numeric",
+                    "magnitude": magnitude,
+                    "scaled_center": center,
+                    "scaled_std": scale,
+                }
+            )
+    if len(vectors) > 256 or len(samples) > 100_000:
+        raise InputError("Design exceeds v0.1 limit: 100,000 samples or 256 encoded columns")
+    matrix = np.column_stack(vectors)
+    n, p = matrix.shape
+    _, singular, vt = np.linalg.svd(matrix, full_matrices=n < p)
+    tolerance = float(np.finfo(float).eps * max(n, p) * singular[0])
+    rank = int(np.count_nonzero(singular > tolerance))
+    basis = vt[:rank]
+    dependencies = []
+    for vector in vt[rank:]:
+        vector = vector / np.max(np.abs(vector))
+        first = next((value for value in vector if abs(value) > 1e-10), 1)
+        if first < 0:
+            vector = -vector
+        dependencies.append([round(float(v), 12) for v in vector])
+    condition = float(singular[0] / singular[rank - 1]) if rank else None
+    sensitive = condition is not None and condition > 1e8
+    comparisons = []
+    for contrast in spec.contrasts:
+        vector = np.zeros(p)
+        for level, sign in [(contrast.numerator, 1), (contrast.denominator, -1)]:
+            if level in target_indices:
+                vector[target_indices[level]] += sign
+        residual = float(
+            np.linalg.norm(vector - basis.T @ (basis @ vector)) / max(1, np.linalg.norm(vector))
+        )
+        comparisons.append(
+            {
+                **contrast.model_dump(),
+                "status": "ESTIMABLE" if residual <= 1e-10 else "NON_ESTIMABLE",
+                "vector": vector.tolist(),
+                "row_space_residual": residual,
+                "tolerance": 1e-10,
+            }
+        )
+    return {
+        "n_rows": n,
+        "n_columns": p,
+        "rank": rank,
+        "row_residual_df": n - rank,
+        "columns": columns,
+        "encoded_rows": matrix.tolist(),
+        "sample_aliases": samples[spec.sample_id].tolist(),
+        "singular_values": singular.tolist(),
+        "rank_tolerance": tolerance,
+        "condition_on_nonzero_subspace": condition,
+        "numerically_sensitive": sensitive,
+        "dependencies": dependencies,
+        "assumptions": "Additive fixed-effects design; estimability is not power or causality.",
+    }, comparisons