apb2 0.1.0__py3-none-any.whl

apb2/__init__.py

@@ -0,0 +1 @@
+"""Rules-driven proteomics vendor-table conversion."""

apb2/annotation/application/policies.py

@@ -0,0 +1,306 @@
+"""Runtime behaviors for attaching annotation and selecting observations."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass, replace
+from typing import Protocol
+
+import polars as pl
+
+from apb2.annotation.data.model import (
+    AnnotationError,
+    AnnotationFileOrigin,
+    AnnotationMatches,
+    AnnotationOrigin,
+    AnnotationResult,
+    LevelAnnotationMatch,
+    LevelAnnotationReport,
+)
+from apb2.parserV2.parse_quant.data.layer_columns import observation_labels
+from apb2.parserV2.parse_quant.data.parsed import (
+    FinalLayerTable,
+    JsonValue,
+    ObsFinal,
+    ParsedLevel,
+    ParsedLevelName,
+    ParsedLevels,
+)
+
+
+class ObservationSelection(Protocol):
+    """Compute one validated Boolean selection per observation."""
+
+    def validate(self, match: LevelAnnotationMatch, /) -> None:
+        """Reject evidence from which this selection cannot be computed."""
+        ...
+
+    def selected_rows(self, match: LevelAnnotationMatch, /) -> pl.Series:
+        """Return one non-null Boolean per observation."""
+        ...
+
+
+class AnnotationApplication(Protocol):
+    """Validate and apply one observation-retention behavior."""
+
+    def validate(self, matches: AnnotationMatches, /) -> None:
+        """Reject matching evidence that cannot produce this application."""
+        ...
+
+    def apply(self, parsed: ParsedLevels, matches: AnnotationMatches, /) -> AnnotationResult:
+        """Attach annotation and return a new storage-neutral result."""
+        ...
+
+
+class MatchedAnnotationSelection:
+    """Select observations matched to any annotation row."""
+
+    __slots__ = ()
+
+    def validate(self, match: LevelAnnotationMatch, /) -> None:
+        del match
+
+    def selected_rows(self, match: LevelAnnotationMatch, /) -> pl.Series:
+        return match.matched_rows
+
+
+@dataclass(frozen=True, slots=True)
+class BooleanAnnotationSelection:
+    """Select matched observations whose annotation field is true."""
+
+    column: str
+
+    def validate(self, match: LevelAnnotationMatch, /) -> None:
+        if self.column not in match.aligned.columns:
+            raise AnnotationError(
+                f"annotation selection column {self.column!r} is absent; "
+                f"available={match.aligned.columns}"
+            )
+        values = match.aligned.get_column(self.column)
+        if values.dtype != pl.Boolean:
+            raise AnnotationError(
+                f"annotation selection column {self.column!r} must be Boolean, got {values.dtype}"
+            )
+        if values.filter(match.matched_rows).null_count():
+            raise AnnotationError(
+                f"annotation selection column {self.column!r} contains null matched values"
+            )
+
+    def selected_rows(self, match: LevelAnnotationMatch, /) -> pl.Series:
+        return match.aligned.get_column(self.column).fill_null(False)
+
+
+@dataclass(frozen=True, slots=True)
+class AllAnnotationSelections:
+    """Intersect several independently reusable observation selections."""
+
+    selections: tuple[ObservationSelection, ...]
+
+    def validate(self, match: LevelAnnotationMatch, /) -> None:
+        for selection in self.selections:
+            selection.validate(match)
+
+    def selected_rows(self, match: LevelAnnotationMatch, /) -> pl.Series:
+        selected = pl.Series("selected", [True] * len(match.matched_rows), dtype=pl.Boolean)
+        for selection in self.selections:
+            selected = selected & selection.selected_rows(match)
+        return selected.rename("selected")
+
+
+class KeepUnmatchedAnnotation:
+    """Attach null metadata to unmatched observations and retain every observation."""
+
+    __slots__ = ()
+
+    def validate(self, matches: AnnotationMatches, /) -> None:
+        _require_any_match(matches)
+
+    def apply(self, parsed: ParsedLevels, matches: AnnotationMatches, /) -> AnnotationResult:
+        return _apply(parsed, matches, selections=None)
+
+
+class RequireCompleteAnnotation:
+    """Require every observation to match before attaching annotation."""
+
+    __slots__ = ()
+
+    def validate(self, matches: AnnotationMatches, /) -> None:
+        _require_any_match(matches)
+        incomplete = {
+            name: match.coverage
+            for name, match in matches.levels.items()
+            if match.coverage.quant_only_count
+        }
+        if incomplete:
+            details = "; ".join(
+                f"{name}: {coverage.quant_only_count} unmatched "
+                f"{list(coverage.quant_only_examples)}, near_misses={dict(coverage.near_misses)}"
+                for name, coverage in incomplete.items()
+            )
+            raise AnnotationError(f"complete sample annotation required; {details}")
+
+    def apply(self, parsed: ParsedLevels, matches: AnnotationMatches, /) -> AnnotationResult:
+        return _apply(parsed, matches, selections=None)
+
+
+@dataclass(frozen=True, slots=True)
+class SelectAnnotatedObservations:
+    """Attach annotation and consistently retain only selected observations."""
+
+    selection: ObservationSelection
+
+    def validate(self, matches: AnnotationMatches, /) -> None:
+        _require_any_match(matches)
+        for match in matches.levels.values():
+            self.selection.validate(match)
+
+    def apply(self, parsed: ParsedLevels, matches: AnnotationMatches, /) -> AnnotationResult:
+        selections: dict[ParsedLevelName, pl.Series] = {
+            name: self.selection.selected_rows(match) for name, match in matches.levels.items()
+        }
+        return _apply(parsed, matches, selections=selections)
+
+
+def record_annotation_provenance(
+    result: AnnotationResult,
+    convention: str,
+    origin: AnnotationOrigin,
+    /,
+    *,
+    metadata: Mapping[str, JsonValue] | None = None,
+) -> AnnotationResult:
+    """Record tool-owned source provenance and each level's annotation report."""
+    if convention in {"parse", "roles", "storage"}:
+        raise AnnotationError(f"annotation convention uses reserved APB section {convention!r}")
+    record: dict[str, JsonValue] = dict(metadata or {})
+    record["schema_version"] = "2"
+    if "source" not in record:
+        record["source"] = (
+            {"path": str(origin.path)} if isinstance(origin, AnnotationFileOrigin) else None
+        )
+    root_tool = _metadata_section(result.parsed.metadata, convention)
+    provenance = _metadata_section(root_tool, "provenance")
+    provenance["annotation"] = record
+    for name, level in result.parsed.levels.items():
+        report = _report_json(result.reports[name])
+        tool = _metadata_section(level.metadata, convention)
+        tool["annotation"] = report
+    return result
+
+
+def _metadata_section(metadata: dict[str, JsonValue], name: str) -> dict[str, JsonValue]:
+    value = metadata.setdefault(name, {})
+    if not isinstance(value, dict):
+        raise AnnotationError(f"APB metadata section {name!r} must be an object")
+    return value
+
+
+def _apply(
+    parsed: ParsedLevels,
+    matches: AnnotationMatches,
+    selections: Mapping[ParsedLevelName, pl.Series] | None,
+) -> AnnotationResult:
+    levels: dict[ParsedLevelName, ParsedLevel] = {}
+    reports: dict[ParsedLevelName, LevelAnnotationReport] = {}
+    for name, level in parsed.levels.items():
+        match = matches.levels[name]
+        selected = (
+            pl.Series("selected", [True] * level.obs.frame.height, dtype=pl.Boolean)
+            if selections is None
+            else selections[name]
+        )
+        levels[name] = _annotated_level(level, match, selected)
+        reports[name] = LevelAnnotationReport(
+            coverage=match.coverage,
+            corrections=match.corrections,
+            columns_added=tuple(match.aligned.columns),
+        )
+    return AnnotationResult(
+        parsed=ParsedLevels(
+            levels=levels,
+            uns=dict(parsed.uns),
+            metadata=dict(parsed.metadata),
+            annotation_tables=dict(parsed.annotation_tables),
+            feature_relations=dict(parsed.feature_relations),
+        ),
+        reports=reports,
+    )
+
+
+def _require_any_match(matches: AnnotationMatches) -> None:
+    empty = [
+        name
+        for name, match in matches.levels.items()
+        if match.coverage.matched_observation_count == 0
+    ]
+    if empty:
+        raise AnnotationError(
+            f"sample annotation matched no observations for level(s) {empty}; "
+            "no dataset-bound annotation was constructed"
+        )
+
+
+def _annotated_level(
+    level: ParsedLevel,
+    match: LevelAnnotationMatch,
+    selected: pl.Series,
+) -> ParsedLevel:
+    annotated_obs = level.obs.frame.hstack(match.aligned.get_columns()).filter(selected)
+    kept = [index for index, value in enumerate(selected) if value]
+    return replace(
+        level,
+        obs=ObsFinal(frame=annotated_obs, key_columns=level.obs.key_columns),
+        uns=dict(level.uns),
+        layers={name: _subset_layer(layer, kept) for name, layer in level.layers.items()},
+        obsm={name: frame.filter(selected) for name, frame in level.obsm.items()},
+        varm=dict(level.varm),
+        obsp={name: _subset_pairwise(frame, kept) for name, frame in level.obsp.items()},
+        varp=dict(level.varp),
+        metadata=dict(level.metadata),
+    )
+
+
+def _subset_layer(layer: FinalLayerTable, kept: list[int]) -> FinalLayerTable:
+    value_columns = layer.values.columns
+    selected_names = [value_columns[index] for index in kept]
+    values = layer.values.select(selected_names)
+    replacement = observation_labels(len(kept), ())
+    values = values.rename(dict(zip(selected_names, replacement, strict=True)))
+    return replace(layer, values=values)
+
+
+def _subset_pairwise(frame: pl.DataFrame, kept: list[int]) -> pl.DataFrame:
+    mapping = {old: new for new, old in enumerate(kept)}
+    if not mapping or frame.is_empty():
+        return frame.head(0)
+    return frame.filter(pl.col("row").is_in(kept) & pl.col("column").is_in(kept)).with_columns(
+        pl.col("row").replace_strict(mapping),
+        pl.col("column").replace_strict(mapping),
+    )
+
+
+def _report_json(report: LevelAnnotationReport) -> dict[str, JsonValue]:
+    coverage = report.coverage
+    near_misses: dict[str, JsonValue] = {
+        key: dict(candidates) for key, candidates in coverage.near_misses.items()
+    }
+    corrections: dict[str, JsonValue] = {
+        correction.observed: {
+            "observed": correction.observed,
+            "expected": correction.expected,
+            "score": correction.score,
+        }
+        for correction in report.corrections
+    }
+    return {
+        "observation_count": coverage.observation_count,
+        "annotation_count": coverage.annotation_count,
+        "matched_observation_count": coverage.matched_observation_count,
+        "quant_only_count": coverage.quant_only_count,
+        "annotation_only_count": coverage.annotation_only_count,
+        "quant_only_examples": list(coverage.quant_only_examples),
+        "annotation_only_examples": list(coverage.annotation_only_examples),
+        "near_misses": near_misses,
+        "corrections": corrections,
+        "columns_added": list(report.columns_added),
+    }

apb2/annotation/compiler.py

@@ -0,0 +1,76 @@
+"""Compile one delimited annotation source into a source-bound parser."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Literal
+
+import polars as pl
+
+from apb2.annotation.application.policies import (
+    AllAnnotationSelections,
+    AnnotationApplication,
+    BooleanAnnotationSelection,
+    KeepUnmatchedAnnotation,
+    MatchedAnnotationSelection,
+    ObservationSelection,
+    RequireCompleteAnnotation,
+    SelectAnnotatedObservations,
+)
+from apb2.annotation.contracts import AnnotationParser
+from apb2.annotation.data.model import AnnotationError
+from apb2.annotation.prolfquapp import ProlfquappAnnotationParser, prolfquapp_signature
+from apb2.annotation.sdrf import SdrfAnnotationParser, SdrfSource, sdrf_signature
+from apb2.annotation.source.load import load_annotation_file, load_annotation_frame
+
+
+class AnnotationCompiler:
+    """User-configured parser for SDRF and generic delimited observation annotations."""
+
+    __slots__ = ("_application",)
+
+    def __init__(
+        self,
+        unmatched: Literal["keep", "error", "drop"] = "keep",
+        include: str | None = None,
+    ) -> None:
+        """Choose what happens to observations the annotation does not cover.
+
+        Args:
+            unmatched: ``keep`` them with null metadata, raise (``error``), or ``drop`` them.
+            include: A Boolean annotation column that further selects observations; requires
+                ``unmatched="drop"``.
+
+        Raises:
+            AnnotationError: ``include`` is given without ``unmatched="drop"``.
+        """
+        self._application = _application(unmatched, include)
+
+    def compile(self, source: Path | pl.DataFrame) -> AnnotationParser:
+        """Load once, verify the tabular convention, and return its bound parser.
+
+        SDRF headers take precedence; other tables must carry a prolfquapp observation key.
+        """
+        loaded = (
+            load_annotation_file(source)
+            if isinstance(source, Path)
+            else load_annotation_frame(source)
+        )
+        if sdrf_signature(loaded):
+            return SdrfAnnotationParser(source=SdrfSource(loaded), application=self._application)
+        if not prolfquapp_signature(loaded):
+            raise AnnotationError("annotation table has no supported observation key")
+        return ProlfquappAnnotationParser(source=loaded, application=self._application)
+
+
+def _application(
+    unmatched: Literal["keep", "error", "drop"], include: str | None
+) -> AnnotationApplication:
+    if unmatched == "drop":
+        selections: list[ObservationSelection] = [MatchedAnnotationSelection()]
+        if include is not None:
+            selections.append(BooleanAnnotationSelection(include))
+        return SelectAnnotatedObservations(AllAnnotationSelections(tuple(selections)))
+    if include is not None:
+        raise AnnotationError("include requires unmatched='drop'")
+    return KeepUnmatchedAnnotation() if unmatched == "keep" else RequireCompleteAnnotation()

apb2/annotation/contracts.py

@@ -0,0 +1,29 @@
+"""Small public capabilities returned by sample-annotation compilation."""
+
+from __future__ import annotations
+
+from typing import Protocol
+
+from apb2.annotation.data.model import AnnotationMatches, AnnotationResult
+from apb2.parserV2.parse_quant.data.parsed import ParsedLevels
+
+
+class Annotation(Protocol):
+    """A validated sample annotation bound to exactly one parsed dataset."""
+
+    @property
+    def matches(self) -> AnnotationMatches:
+        """Return completed per-level matching evidence."""
+        ...
+
+    def annotate(self) -> AnnotationResult:
+        """Apply this annotation to the dataset against which it was validated."""
+        ...
+
+
+class AnnotationParser(Protocol):
+    """A convention parser bound to one already loaded annotation source."""
+
+    def parse(self, parsed: ParsedLevels, /) -> Annotation:
+        """Validate and match the source, then construct a dataset-bound annotation."""
+        ...

apb2/annotation/data/model.py

@@ -0,0 +1,112 @@
+"""Innermost values exchanged by the sample-annotation workflow."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass
+from pathlib import Path
+
+import polars as pl
+
+from apb2.parserV2.parse_quant.data.parsed import ParsedLevelName, ParsedLevels
+
+
+class AnnotationError(ValueError):
+    """A sample annotation cannot be recognized, parsed, matched, or applied."""
+
+
+@dataclass(frozen=True, slots=True)
+class AnnotationFileOrigin:
+    """The resolved file from which an annotation was loaded."""
+
+    path: Path
+
+
+@dataclass(frozen=True, slots=True)
+class InMemoryAnnotationOrigin:
+    """Marker for a programmatically supplied Polars annotation."""
+
+
+type AnnotationOrigin = AnnotationFileOrigin | InMemoryAnnotationOrigin
+
+IN_MEMORY_ANNOTATION = InMemoryAnnotationOrigin()
+
+
+@dataclass(frozen=True, slots=True)
+class LoadedAnnotationSource:
+    """One physically decoded tabular annotation source.
+
+    ``headers`` holds the verbatim header text aligned with ``frame.columns``. Repeated headers
+    keep their text there while the frame columns stay unique.
+    """
+
+    frame: pl.DataFrame
+    origin: AnnotationOrigin
+    headers: tuple[str, ...]
+
+
+@dataclass(frozen=True, slots=True)
+class AnnotationTable:
+    """Validated sample rows and every identifier column accepted for matching."""
+
+    frame: pl.DataFrame
+    key_columns: tuple[str, ...]
+    alias_columns: tuple[str, ...]
+    origin: AnnotationOrigin
+
+
+@dataclass(frozen=True, slots=True)
+class AnnotationCoverage:
+    """Counts and bounded mismatch evidence for one quantification level."""
+
+    observation_count: int
+    annotation_count: int
+    matched_observation_count: int
+    quant_only_count: int
+    annotation_only_count: int
+    quant_only_examples: tuple[str, ...]
+    annotation_only_examples: tuple[str, ...]
+    near_misses: Mapping[str, tuple[tuple[str, float], ...]]
+
+
+@dataclass(frozen=True, slots=True)
+class KeyCorrection:
+    """One accepted fuzzy observation-to-annotation identifier correction."""
+
+    observed: str
+    expected: str
+    score: float
+
+
+@dataclass(frozen=True, slots=True)
+class LevelAnnotationMatch:
+    """Annotation aligned to one observation axis plus its matching evidence."""
+
+    aligned: pl.DataFrame
+    matched_rows: pl.Series
+    coverage: AnnotationCoverage
+    corrections: tuple[KeyCorrection, ...]
+
+
+@dataclass(frozen=True, slots=True)
+class AnnotationMatches:
+    """Independent matching evidence for every parsed quantification level."""
+
+    levels: Mapping[ParsedLevelName, LevelAnnotationMatch]
+
+
+@dataclass(frozen=True, slots=True)
+class LevelAnnotationReport:
+    """Persistable evidence produced by applying one level's annotation."""
+
+    coverage: AnnotationCoverage
+    corrections: tuple[KeyCorrection, ...]
+    columns_added: tuple[str, ...]
+
+
+@dataclass(frozen=True, slots=True)
+class AnnotationResult:
+    """A newly annotated storage-neutral result and its per-level reports."""
+
+    parsed: ParsedLevels
+    reports: Mapping[ParsedLevelName, LevelAnnotationReport]